NousResearch/atropos
1
0
Fork 0
mirror of https://github.com/NousResearch/atropos.git synced 2026-04-19 12:57:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[pre-commit.ci] auto fixes from pre-commit.com hooks

Browse source 
for more information, see https://pre-commit.ci
This commit is contained in:
pre-commit-ci[bot] 2025-12-24 10:48:20 +00:00
parent ef9c0c3699
commit afab28dfa9
37 changed files with 4868 additions and 4052 deletions
Download patch file Download diff file Expand all files Collapse all files

3
environments/eval_environments/ifeval_instructions/__init__.py
View file

@ -4,7 +4,6 @@ IFEval Instruction Checking Module
Ported from lighteval (google/IFEval) for standalone use in Atropos.
"""
from .instructions_registry import INSTRUCTION_DICT, INSTRUCTION_CONFLICTS
from .instructions_registry import INSTRUCTION_CONFLICTS, INSTRUCTION_DICT
__all__ = ["INSTRUCTION_DICT", "INSTRUCTION_CONFLICTS"]

154
environments/eval_environments/ifeval_instructions/instructions.py
View file

@ -27,13 +27,13 @@ import string
# Check if langdetect is available
try:
import langdetect
LANGDETECT_AVAILABLE = True
except ImportError:
LANGDETECT_AVAILABLE = False
from . import instructions_utils as instructions_util
logger = logging.getLogger(__name__)
_LANGUAGES = instructions_util.LANGUAGE_CODES
@ -150,7 +150,8 @@ class ResponseLanguageChecker(Instruction):
if self._language is None:
self._language = random.choice(list(_LANGUAGES.keys()))
self._description_pattern = (
"Your ENTIRE response should be in {language} language, no other " + "language is allowed."
"Your ENTIRE response should be in {language} language, no other "
+ "language is allowed."
)
return self._description_pattern.format(language=_LANGUAGES[self._language])
@ -218,7 +219,9 @@ class NumberOfSentences(Instruction):
else:
self._comparison_relation = relation
self._description_pattern = "Your response should contain {relation} {num_sentences} sentences."
self._description_pattern = (
"Your response should contain {relation} {num_sentences} sentences."
)
return self._description_pattern.format(
relation=self._comparison_relation,
num_sentences=self._num_sentences_threshold,
@ -356,8 +359,12 @@ class ConstrainedResponseChecker(Instruction):
"""Build the instruction description."""
# A sequence of string(s) representing the options of the expected response.
self._constrained_responses = _CONSTRAINED_RESPONSE_OPTIONS
self._description_pattern = "Answer with one of the following options: {response_options}"
return self._description_pattern.format(response_options=self._constrained_responses)
self._description_pattern = (
"Answer with one of the following options: {response_options}"
)
return self._description_pattern.format(
response_options=self._constrained_responses
)
def get_instruction_args(self):
"""Returns the keyward args of `build_description`."""
@ -401,7 +408,8 @@ class ConstrainedStartChecker(Instruction):
if self._starter is None:
self._starter = random.choice(_STARTER_OPTIONS)
self._description_pattern = (
"During the conversation, when it is your turn, " + "please always start with {starter}"
"During the conversation, when it is your turn, "
+ "please always start with {starter}"
)
return self._description_pattern.format(starter=self._starter)
@ -424,7 +432,9 @@ class ConstrainedStartChecker(Instruction):
contained in `instruction_args`; otherwise, False.
"""
response_pattern = r"^\s*" + self._starter + r".*$"
response_with_constrained_start = re.search(response_pattern, value, flags=re.MULTILINE)
response_with_constrained_start = re.search(
response_pattern, value, flags=re.MULTILINE
)
return True if response_with_constrained_start else False
@ -498,7 +508,11 @@ class SectionChecker(Instruction):
Returns:
A string representing the instruction description.
"""
self._section_spliter = section_spliter.strip() if isinstance(section_spliter, str) else section_spliter
self._section_spliter = (
section_spliter.strip()
if isinstance(section_spliter, str)
else section_spliter
)
if self._section_spliter is None:
self._section_spliter = random.choice(_SECTION_SPLITER)
@ -515,7 +529,9 @@ class SectionChecker(Instruction):
+ "[content of section 2]"
)
return self._description_pattern.format(num_sections=self._num_sections, section_spliter=self._section_spliter)
return self._description_pattern.format(
num_sections=self._num_sections, section_spliter=self._section_spliter
)
def get_instruction_args(self):
"""Returns the keyward args of `build_description`."""
@ -564,7 +580,8 @@ class ParagraphChecker(Instruction):
self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS)
self._description_pattern = (
"There should be {num_paragraphs} paragraphs. " + "Paragraphs are separated with the markdown divider: ***"
"There should be {num_paragraphs} paragraphs. "
+ "Paragraphs are separated with the markdown divider: ***"
)
return self._description_pattern.format(num_paragraphs=self._num_paragraphs)
@ -615,13 +632,16 @@ class PostscriptChecker(Instruction):
A string representing the instruction description.
"""
self._postscript_marker = (
postscript_marker.strip() if isinstance(postscript_marker, str) else postscript_marker
postscript_marker.strip()
if isinstance(postscript_marker, str)
else postscript_marker
)
if self._postscript_marker is None:
self._postscript_marker = random.choice(_POSTSCRIPT_MARKER)
self._description_pattern = (
"At the end of your response, please explicitly add a postscript " + "starting with {postscript}"
"At the end of your response, please explicitly add a postscript "
+ "starting with {postscript}"
)
return self._description_pattern.format(postscript=self._postscript_marker)
@ -672,7 +692,9 @@ class RephraseChecker(Instruction):
A string representing the instruction description.
"""
if not self.is_change(original_message):
raise ValueError(f"Message {original_message} does not contain changes in the form of *change me*.")
raise ValueError(
f"Message {original_message} does not contain changes in the form of *change me*."
)
self._reference_without_change = original_message
self._description = (
@ -702,7 +724,9 @@ class RephraseChecker(Instruction):
in between two asterisks such as *change me*; otherwise, False.
"""
if not self.is_change(value):
raise ValueError(f"value {value} does not contain changes in the form of *change me*.")
raise ValueError(
f"value {value} does not contain changes in the form of *change me*."
)
response_without_changes = self.strip_changes(value)
reference_without_changes = self.strip_changes(self._reference_without_change)
@ -732,7 +756,9 @@ class KeywordChecker(Instruction):
A string representing the instruction description.
"""
if not keywords:
self._keywords = instructions_util.generate_keywords(num_keywords=_NUM_KEYWORDS)
self._keywords = instructions_util.generate_keywords(
num_keywords=_NUM_KEYWORDS
)
else:
self._keywords = keywords
self._keywords = sorted(self._keywords)
@ -799,7 +825,8 @@ class KeywordFrequencyChecker(Instruction):
self._comparison_relation = relation
self._description_pattern = (
"In your response, the word {keyword} should appear {relation} " + "{frequency} times."
"In your response, the word {keyword} should appear {relation} "
+ "{frequency} times."
)
return self._description_pattern.format(
@ -850,7 +877,9 @@ class NumberOfWords(Instruction):
"""
self._num_words = num_words
if self._num_words is None or self._num_words < 0:
self._num_words = random.randint(_NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT)
self._num_words = random.randint(
_NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT
)
if relation is None:
self._comparison_relation = random.choice(_COMPARISON_RELATION)
@ -863,7 +892,9 @@ class NumberOfWords(Instruction):
self._description_pattern = "Answer with {relation} {num_words} words."
return self._description_pattern.format(relation=self._comparison_relation, num_words=self._num_words)
return self._description_pattern.format(
relation=self._comparison_relation, num_words=self._num_words
)
def get_instruction_args(self):
"""Returns the keyward args of `build_description`."""
@ -887,9 +918,7 @@ class JsonFormat(Instruction):
"""Check the Json format."""
def build_description(self):
self._description_pattern = (
"Entire output should be wrapped in JSON format. You can use markdown ticks such as ```."
)
self._description_pattern = "Entire output should be wrapped in JSON format. You can use markdown ticks such as ```."
return self._description_pattern
def get_instruction_args(self):
@ -920,7 +949,9 @@ class JsonFormat(Instruction):
class ParagraphFirstWordCheck(Instruction):
"""Check the paragraph and the first word of the nth paragraph."""
def build_description(self, num_paragraphs=None, nth_paragraph=None, first_word=None):
def build_description(
self, num_paragraphs=None, nth_paragraph=None, first_word=None
):
r"""Build the instruction description.
Args:
@ -939,7 +970,11 @@ class ParagraphFirstWordCheck(Instruction):
self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS)
self._nth_paragraph = nth_paragraph
if self._nth_paragraph is None or self._nth_paragraph <= 0 or self._nth_paragraph > self._num_paragraphs:
if (
self._nth_paragraph is None
or self._nth_paragraph <= 0
or self._nth_paragraph > self._num_paragraphs
):
self._nth_paragraph = random.randint(1, self._num_paragraphs + 1)
self._first_word = first_word
@ -1040,9 +1075,13 @@ class KeySentenceChecker(Instruction):
else:
self._num_sentences = num_sentences
self._description_pattern = "Include {num_sentences} of the following sentences {key_sentences}"
self._description_pattern = (
"Include {num_sentences} of the following sentences {key_sentences}"
)
return self._description_pattern.format(num_sentences=self._num_sentences, key_sentences=self._key_sentences)
return self._description_pattern.format(
num_sentences=self._num_sentences, key_sentences=self._key_sentences
)
def get_instruction_args(self):
"""Returns the keyward args of `build_description`."""
@ -1080,11 +1119,15 @@ class ForbiddenWords(Instruction):
A string representing the instruction description.
"""
if not forbidden_words:
self._forbidden_words = instructions_util.generate_keywords(num_keywords=_NUM_KEYWORDS)
self._forbidden_words = instructions_util.generate_keywords(
num_keywords=_NUM_KEYWORDS
)
else:
self._forbidden_words = list(set(forbidden_words))
self._forbidden_words = sorted(self._forbidden_words)
self._description_pattern = "Do not include keywords {forbidden_words} in the response."
self._description_pattern = (
"Do not include keywords {forbidden_words} in the response."
)
return self._description_pattern.format(forbidden_words=self._forbidden_words)
@ -1133,7 +1176,9 @@ class RephraseParagraph(Instruction):
+ "to 'ran'."
)
return self._description.format(original_paragraph=original_paragraph, low=self._low, high=self._high)
return self._description.format(
original_paragraph=original_paragraph, low=self._low, high=self._high
)
def get_instruction_args(self):
"""Returns the keyward args of `build_description`."""
@ -1197,7 +1242,10 @@ class TwoResponsesChecker(Instruction):
return False
else:
valid_responses.append(response)
return len(valid_responses) == 2 and valid_responses[0].strip() != valid_responses[1].strip()
return (
len(valid_responses) == 2
and valid_responses[0].strip() != valid_responses[1].strip()
)
class RepeatPromptThenAnswer(Instruction):
@ -1249,12 +1297,12 @@ class EndChecker(Instruction):
Returns:
A string representing the instruction description.
"""
self._end_phrase = end_phrase.strip() if isinstance(end_phrase, str) else end_phrase
self._end_phrase = (
end_phrase.strip() if isinstance(end_phrase, str) else end_phrase
)
if self._end_phrase is None:
self._end_phrase = random.choice(_ENDING_OPTIONS)
self._description_pattern = (
"Finish your response with this exact phrase {ender}. No other words should follow this phrase."
)
self._description_pattern = "Finish your response with this exact phrase {ender}. No other words should follow this phrase."
return self._description_pattern.format(ender=self._end_phrase)
def get_instruction_args(self):
@ -1276,9 +1324,7 @@ class TitleChecker(Instruction):
def build_description(self):
"""Build the instruction description."""
self._description_pattern = (
"Your answer must contain a title, wrapped in double angular brackets, such as <<poem of joy>>."
)
self._description_pattern = "Your answer must contain a title, wrapped in double angular brackets, such as <<poem of joy>>."
return self._description_pattern
def get_instruction_args(self):
@ -1319,7 +1365,12 @@ class LetterFrequencyChecker(Instruction):
Returns:
A string representing the instruction description.
"""
if not letter or len(letter) > 1 or ord(letter.lower()) < 97 or ord(letter.lower()) > 122:
if (
not letter
or len(letter) > 1
or ord(letter.lower()) < 97
or ord(letter.lower()) > 122
):
self._letter = random.choice(list(string.ascii_letters))
else:
self._letter = letter.strip()
@ -1339,9 +1390,7 @@ class LetterFrequencyChecker(Instruction):
else:
self._comparison_relation = let_relation
self._description_pattern = (
"In your response, the letter {letter} should appear {let_relation} {let_frequency} times."
)
self._description_pattern = "In your response, the letter {letter} should appear {let_relation} {let_frequency} times."
return self._description_pattern.format(
letter=self._letter,
@ -1377,7 +1426,9 @@ class CapitalLettersEnglishChecker(Instruction):
def build_description(self):
"""Build the instruction description."""
self._description_pattern = "Your entire response should be in English, and in all capital letters."
self._description_pattern = (
"Your entire response should be in English, and in all capital letters."
)
return self._description_pattern
def get_instruction_args(self):
@ -1408,9 +1459,7 @@ class LowercaseLettersEnglishChecker(Instruction):
def build_description(self):
"""Build the instruction description."""
self._description_pattern = (
"Your entire response should be in English, and in all lowercase letters. No capital letters are allowed."
)
self._description_pattern = "Your entire response should be in English, and in all lowercase letters. No capital letters are allowed."
return self._description_pattern
def get_instruction_args(self):
@ -1441,7 +1490,9 @@ class CommaChecker(Instruction):
def build_description(self):
"""Build the instruction description."""
self._description_pattern = "In your entire response, refrain from the use of any commas."
self._description_pattern = (
"In your entire response, refrain from the use of any commas."
)
return self._description_pattern
def get_instruction_args(self):
@ -1488,11 +1539,11 @@ class CapitalWordFrequencyChecker(Instruction):
f"{_COMPARISON_RELATION}, but {capital_relation} is given."
)
self._description_pattern = (
"In your response, words with all capital letters should appear {relation} {frequency} times."
)
self._description_pattern = "In your response, words with all capital letters should appear {relation} {frequency} times."
return self._description_pattern.format(frequency=self._frequency, relation=self._comparison_relation)
return self._description_pattern.format(
frequency=self._frequency, relation=self._comparison_relation
)
def get_instruction_args(self):
"""Returns the keyword args of build description."""
@ -1524,7 +1575,9 @@ class QuotationChecker(Instruction):
def build_description(self):
"""Build the instruction description."""
self._description_pattern = "Wrap your entire response with double quotation marks."
self._description_pattern = (
"Wrap your entire response with double quotation marks."
)
return self._description_pattern
def get_instruction_args(self):
@ -1539,4 +1592,3 @@ class QuotationChecker(Instruction):
"""Checks if the response is wrapped with double quotation marks."""
value = value.strip()
return len(value) > 1 and value[0] == '"' and value[-1] == '"'

29
environments/eval_environments/ifeval_instructions/instructions_registry.py
View file

@ -19,7 +19,6 @@ Ported from lighteval for standalone use in Atropos.
from . import instructions
_KEYWORD = "keywords:"
_LANGUAGE = "language:"
@ -73,7 +72,8 @@ INSTRUCTION_CONFLICTS = {
_KEYWORD + "frequency": {_KEYWORD + "frequency"},
_KEYWORD + "forbidden_words": {_KEYWORD + "forbidden_words"},
_KEYWORD + "letter_frequency": {_KEYWORD + "letter_frequency"},
_LANGUAGE + "response_language": {
_LANGUAGE
+ "response_language": {
_LANGUAGE + "response_language",
_FORMAT + "multiple_sections",
_KEYWORD + "existence",
@ -84,14 +84,16 @@ INSTRUCTION_CONFLICTS = {
_CHANGE_CASES + "english_lowercase",
},
_LENGTH + "number_sentences": {_LENGTH + "number_sentences"},
_LENGTH + "number_paragraphs": {
_LENGTH
+ "number_paragraphs": {
_LENGTH + "number_paragraphs",
_LENGTH + "nth_paragraph_first_word",
_LENGTH + "number_sentences",
_LENGTH + "nth_paragraph_first_word",
},
_LENGTH + "number_words": {_LENGTH + "number_words"},
_LENGTH + "nth_paragraph_first_word": {
_LENGTH
+ "nth_paragraph_first_word": {
_LENGTH + "nth_paragraph_first_word",
_LENGTH + "number_paragraphs",
},
@ -100,16 +102,19 @@ INSTRUCTION_CONFLICTS = {
_FORMAT + "number_bullet_lists": {_FORMAT + "number_bullet_lists"},
_FORMAT + "constrained_response": set(INSTRUCTION_DICT.keys()),
_FORMAT + "number_highlighted_sections": {_FORMAT + "number_highlighted_sections"},
_FORMAT + "multiple_sections": {
_FORMAT
+ "multiple_sections": {
_FORMAT + "multiple_sections",
_LANGUAGE + "response_language",
_FORMAT + "number_highlighted_sections",
},
_FORMAT + "json_format": set(INSTRUCTION_DICT.keys()).difference(
_FORMAT
+ "json_format": set(INSTRUCTION_DICT.keys()).difference(
{_KEYWORD + "forbidden_words", _KEYWORD + "existence"}
),
_FORMAT + "title": {_FORMAT + "title"},
_COMBINATION + "two_responses": set(INSTRUCTION_DICT.keys()).difference(
_COMBINATION
+ "two_responses": set(INSTRUCTION_DICT.keys()).difference(
{
_KEYWORD + "forbidden_words",
_KEYWORD + "existence",
@ -118,17 +123,20 @@ INSTRUCTION_CONFLICTS = {
_PUNCTUATION + "no_comma",
}
),
_COMBINATION + "repeat_prompt": set(INSTRUCTION_DICT.keys()).difference(
_COMBINATION
+ "repeat_prompt": set(INSTRUCTION_DICT.keys()).difference(
{_KEYWORD + "existence", _FORMAT + "title", _PUNCTUATION + "no_comma"}
),
_STARTEND + "end_checker": {_STARTEND + "end_checker"},
_CHANGE_CASES + "capital_word_frequency": {
_CHANGE_CASES
+ "capital_word_frequency": {
_CHANGE_CASES + "capital_word_frequency",
_CHANGE_CASES + "english_lowercase",
_CHANGE_CASES + "english_capital",
},
_CHANGE_CASES + "english_capital": {_CHANGE_CASES + "english_capital"},
_CHANGE_CASES + "english_lowercase": {
_CHANGE_CASES
+ "english_lowercase": {
_CHANGE_CASES + "english_lowercase",
_CHANGE_CASES + "english_capital",
},
@ -153,4 +161,3 @@ def conflict_make(conflicts):
conflicts[k].add(key)
conflicts[key].add(key)
return conflicts

1
environments/eval_environments/ifeval_instructions/instructions_utils.py
View file

@ -1709,4 +1709,3 @@ def generate_keywords(num_keywords):
list[str]: List of randomly selected keywords
"""
return random.sample(WORD_LIST, k=num_keywords)