NousResearch/atropos
1
0
Fork 0
mirror of https://github.com/NousResearch/atropos.git synced 2026-04-23 16:54:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[pre-commit.ci] auto fixes from pre-commit.com hooks

Browse source 
for more information, see https://pre-commit.ci
This commit is contained in:
pre-commit-ci[bot] 2026-01-12 23:29:15 +00:00
parent 22884d2bf7
commit d84e3c70b7
16 changed files with 270 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

25
environments/eval_environments/vlmblind_environment.py
View file

@ -65,20 +65,29 @@ class VLMBlind(EvalBase):
content = []
if image_base64:
content.append({
"type": "image_url",
"image_url": {"url": f"data:image/png;base64,{image_base64}"},
})
content.append(
{
"type": "image_url",
"image_url": {"url": f"data:image/png;base64,{image_base64}"},
}
)
content.append({"type": "text", "text": prompt})
return [{"role": "user", "content": content}]
def extract_and_score(self, response: str, answer: str, task: str) -> Tuple[bool, str]:
def extract_and_score(
self, response: str, answer: str, task: str
) -> Tuple[bool, str]:
"""Task-specific answer extraction and scoring."""
response_lower = response.lower().strip()
answer_lower = str(answer).lower().strip()
if task in ["Subway Connections", "Nested Squares", "Line Plot Intersections", "Circled Letter"]:
if task in [
"Subway Connections",
"Nested Squares",
"Line Plot Intersections",
"Circled Letter",
]:
match = re.search(r"\{([^}]+)\}", response)
if match:
extracted = match.group(1).strip().lower()
@ -140,7 +149,9 @@ class VLMBlind(EvalBase):
sample = {
"id": data_item.get("index", data_item.get("id", "")),
"question": data_item.get("prompt", data_item.get("question", ""))[:200],
"question": data_item.get("prompt", data_item.get("question", ""))[
:200
],
"task": task,
"answer": answer,
"prediction": extracted,