fix sonnet eval_dir (#216)

* fix eval_dir

* add logging
This commit is contained in:
vncntt 2025-02-26 00:37:09 -08:00 committed by GitHub
parent 48f082663a
commit 29179f783e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 8 additions and 1 deletions

View file

@ -15,6 +15,13 @@ from tenacity import AsyncRetrying, retry_if_exception_type, stop_after_attempt,
import reasoning_gym
from reasoning_gym.utils import extract_answer
# Configure logging
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
handlers=[logging.StreamHandler()],
)
class OpenRouterEvaluator:
def __init__(self, model: str, config: EvalConfig):

View file

@ -25,7 +25,7 @@ datasets:
- word_ladder
- word_sequence_reversal
- word_sorting
eval_dir: eval/r1
eval_dir: eval/sonnet-3.5
dataset_size: 50
dataset_seed: 45
developer_role: system