# Config used for evaluating inter-domain generalisation experiment models on ARC test data

# Models evaluated on this config:
# Qwen/Qwen2.5-3B-Instruct (original model)
# inter_algebra_qwen_3b_500 (original + 500 GRPO steps on algebra RG data)
# inter_games_qwen_3b_500 (original + 500 GRPO steps on games RG data)

model_path: ../models/inter_games_qwen_3b_500  # Change to the model to be evaluated

max_tokens: 2048  # From max_response_length in training config
top_p: 0.9  # From rollout top_p
temperature: 0.6  # Lower temperature for more focused responses

developer_prompt: DeepSeekZero
developer_role: system

output_dir: results
save_metadata: true
save_full_results: true
eval_repeats: 3

categories:
  - category: arc
    datasets:
      - dataset: arc_1d
        size: 100
        seed: 42
      - dataset: arc_agi
        size: 100
        seed: 42
      - dataset: rearc
        size: 100
        seed: 42