open-thought/reasoning-gym
1
0
Fork 0
mirror of https://github.com/open-thought/reasoning-gym.git synced 2026-04-22 16:49:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add ArcAgiDataset class, fix score_entry() metadata params

Browse source
This commit is contained in:
Andreas Koepf 2025-02-08 23:18:18 +01:00
parent 2ad0965fdc
commit 4e49806d22
20 changed files with 194 additions and 93 deletions
Download patch file Download diff file Expand all files Collapse all files

5
tests/test_simple_integration.py
View file

@ -73,7 +73,7 @@ def test_verify_answer():
dataset = SimpleIntegrationDataset(config)
for i in range(len(dataset)):
item = dataset[i]
score = dataset.score_answer(item["answer"], item["metadata"])
score = dataset.score_answer(item["answer"], item)
assert score == 1.0
@ -113,5 +113,6 @@ def test_score_answer_cases():
]
for answer, metadata, expected in test_cases:
score = dataset.score_answer(answer, metadata)
dummy_entry = {"metadata": metadata}
score = dataset.score_answer(answer=answer, entry=dummy_entry)
assert score == expected, f"Failed case: {answer} | Expected {expected}, got {score}"