open-thought/reasoning-gym
1
0
Fork 0
mirror of https://github.com/open-thought/reasoning-gym.git synced 2026-04-22 16:49:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add ArcAgiDataset class, fix score_entry() metadata params

Browse source
This commit is contained in:
Andreas Koepf 2025-02-08 23:18:18 +01:00
parent 2ad0965fdc
commit 4e49806d22
20 changed files with 194 additions and 93 deletions
Download patch file Download diff file Expand all files Collapse all files

5
tests/test_intermediate_integration.py
View file

@ -100,7 +100,7 @@ def test_verify_answer():
dataset = IntermediateIntegrationDataset(config)
for i in range(len(dataset)):
item = dataset[i]
score = dataset.score_answer(item["answer"], item["metadata"])
score = dataset.score_answer(answer=item["answer"], entry=item)
assert score == 1.0
@ -140,5 +140,6 @@ def test_score_answer_cases():
]
for answer, metadata, expected in test_cases:
score = dataset.score_answer(answer, metadata)
dummy_entry = {"metadata": metadata}
score = dataset.score_answer(answer, entry=dummy_entry)
assert score == expected, f"Failed case: {answer} | Expected {expected}, got {score}"