diff --git a/reasoning_gym/algorithmic/spell_backward.py b/reasoning_gym/algorithmic/spell_backward.py index 59b163ee..d1837521 100644 --- a/reasoning_gym/algorithmic/spell_backward.py +++ b/reasoning_gym/algorithmic/spell_backward.py @@ -3,7 +3,7 @@ import re from dataclasses import dataclass from random import Random -from typing import Optional +from typing import Any, Dict, Optional from ..data import read_data_file from ..factory import ProceduralDataset, register_dataset @@ -49,5 +49,18 @@ class SpellBackwardDataset(ProceduralDataset): "metadata": {"word": word, "word_len": len(word)}, } + def score_answer(self, answer: Optional[str], entry: Dict[str, Any]) -> float: + reward = 0 + expected_answer = entry["answer"] + if answer is not None: + try: + if expected_answer.lower() == answer.lower(): + reward = 1.0 + else: + reward = 0.05 + except: + reward = 0.01 + return reward + register_dataset("spell_backward", SpellBackwardDataset, SpellBackwardConfig)