mirror of
https://github.com/open-thought/reasoning-gym.git
synced 2026-04-19 12:58:07 +00:00
added custom score answer func
This commit is contained in:
parent
73910f0f61
commit
e80f44d75a
1 changed files with 29 additions and 1 deletions
|
|
@ -3,7 +3,7 @@
|
|||
import re
|
||||
from dataclasses import dataclass
|
||||
from random import Random
|
||||
from typing import Optional
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from ..data import read_data_file
|
||||
from ..factory import ProceduralDataset, register_dataset
|
||||
|
|
@ -92,5 +92,33 @@ class SentenceReorderingDataset(ProceduralDataset):
|
|||
"metadata": {"word_count": word_count},
|
||||
}
|
||||
|
||||
def score_answer(self, answer: Optional[str], entry: Dict[str, Any]) -> float:
|
||||
reward = 0
|
||||
expected_answer = entry["answer"]
|
||||
if answer is not None:
|
||||
try:
|
||||
if expected_answer == answer:
|
||||
return 1.0
|
||||
goal_words = expected_answer.split()
|
||||
answer_words = answer.split()
|
||||
if len(goal_words) == len(answer_words):
|
||||
credit = [1 if goal_word.lower() == answer_word.lower() else 0 for goal_word, answer_word in zip(goal_words, answer_words)]
|
||||
reward = sum(credit) / len(credit)
|
||||
else:
|
||||
reward = 0.05
|
||||
except:
|
||||
reward = 0.01
|
||||
return reward
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
register_dataset("sentence_reordering", SentenceReorderingDataset, SentenceReorderingConfig)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue