open-thought/reasoning-gym
1
0
Fork 0
mirror of https://github.com/open-thought/reasoning-gym.git synced 2026-04-25 17:10:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

pinned provider to nebius

Browse source
This commit is contained in:
joesharratt1229 2025-02-24 05:01:22 +00:00
parent 80eff8acb6
commit 1b0f774974
1 changed files with 1 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

12
eval/r1/eval.py
View file

@ -52,16 +52,6 @@ class OpenRouterEvaluator:
json.dump(metrics, f, indent=2)
return metrics
def prepare_messages(self, prompt: str) -> list[dict[str, str]]:
return {
"model": self.model,
"messages": [
{"role": self.config.developer_role, "content": self.config.developer_prompt},
{"role": "user", "content": prompt},
],
"provider": {"order": ["Nebius"], "allow_fallbacks": False},
}
async def get_model_response(self, session: aiohttp.ClientSession, prompt: str) -> str:
payload = {
"model": self.model,
@ -69,6 +59,7 @@ class OpenRouterEvaluator:
{"role": self.config.developer_role, "content": self.config.developer_prompt},
{"role": "user", "content": prompt},
],
"provider": {"order": ["Nebius"], "allow_fallbacks": False},
}
async for attempt in AsyncRetrying(
@ -98,7 +89,6 @@ class OpenRouterEvaluator:
response = await self.get_model_response(session, entry["question"])
model_answer = extract_answer(response)
score = dataset.score_answer(answer=model_answer, entry=entry)
print(f"Question: {entry['question']}")
return {
"question": entry["question"],