From 3a9eedaa87680fddd3abf48123d2ef616d5b262f Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Fri, 31 Jan 2025 06:42:25 +0000 Subject: [PATCH] added countdown score answer impl --- reasoning_gym/games/countdown.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/reasoning_gym/games/countdown.py b/reasoning_gym/games/countdown.py index 4721844d..87d1a793 100644 --- a/reasoning_gym/games/countdown.py +++ b/reasoning_gym/games/countdown.py @@ -1,9 +1,10 @@ from dataclasses import dataclass from random import Random -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Dict, Any import sympy from sympy import Symbol, symbols +from sympy.parsing.sympy_parser import parse_expr from ..factory import ProceduralDataset, register_dataset @@ -157,6 +158,23 @@ class CountdownDataset(ProceduralDataset): continue raise ValueError(f"Failed to generate valid expression after {max_attempts} attempts") + + def score_answer(self, answer: Optional[str], metadata: Dict[str, Any]) -> float: + """Determine if the solution provided solves the problem""" + reward = 0.0 + if answer is not None: + try: + user_answer = int(parse_expr(answer)) + solved = user_answer == metadata["target"] + if solved: + reward = 1.0 + elif (len(answer.strip()) > 0): # encourage partial solutions + reward = 0.05 + else: + reward = 0.01 + except: + reward = 0.01 + return reward # Register the dataset