inter-domain generalisation evaluation configs (#424)

* add inter-domain generalisation eval config for algebra

* add algorithmic eval cfg

* vllm infer

* add arithmetic eval cfg

* add geometry eval cfg

* add arc cfg

* add games eval cfg

* add cognition eval cfg

* add graphs eval cfg

training/evaluations/inter_generalisation/algebra.yaml

@@ -0,0 +1,41 @@
+# Config used for evaluating inter-domain generalisation experiment models on algebra test data
+
+# Models evaluated on this config:
+# Qwen/Qwen2.5-3B-Instruct (original model)
+# inter_algorithmic_qwen_3b_500 (original + 500 GRPO steps on algorithmic RG data)
+# ../models/inter_algorithmic_qwen_3b_500
+model_path: Qwen/Qwen2.5-3B-Instruct  # Change to the model to be evaluated
+
+max_tokens: 2048  # From max_response_length in training config
+top_p: 0.9  # From rollout top_p
+temperature: 0.6  # Lower temperature for more focused responses
+
+developer_prompt: DeepSeekZero
+developer_role: system
+
+output_dir: results
+save_metadata: true
+save_full_results: true
+eval_repeats: 3
+
+categories:
+  - category: algebra
+    datasets:
+      - dataset: complex_arithmetic
+        size: 100
+        seed: 42
+      - dataset: intermediate_integration
+        size: 100
+        seed: 42
+      - dataset: polynomial_equations
+        size: 100
+        seed: 42
+      - dataset: polynomial_multiplication
+        size: 100
+        seed: 42
+      - dataset: simple_equations
+        size: 100
+        seed: 42
+      - dataset: simple_integration
+        size: 100
+        seed: 42

training/evaluations/inter_generalisation/algorithmic.yaml

@@ -0,0 +1,70 @@
+# Config used for evaluating inter-domain generalisation experiment models on algorithmic test data
+
+# Models evaluated on this config:
+# Qwen/Qwen2.5-3B-Instruct (original model)
+# inter_algebra_qwen_3b_500 (original + 500 GRPO steps on algebra RG data)
+
+model_path: ../models/inter_algebra_qwen_3b_500  # Change to the model to be evaluated
+
+max_tokens: 2048  # From max_response_length in training config
+top_p: 0.9  # From rollout top_p
+temperature: 0.6  # Lower temperature for more focused responses
+
+developer_prompt: DeepSeekZero
+developer_role: system
+
+output_dir: results
+save_metadata: true
+save_full_results: true
+eval_repeats: 3
+
+categories:
+  - category: algorithmic
+    datasets:
+      - dataset: ab
+        size: 100
+        seed: 42
+      - dataset: base_conversion
+        size: 100
+        seed: 42
+      - dataset: binary_alternation
+        size: 100
+        seed: 42
+        params:
+          p_solvable: 0.9
+      - dataset: binary_matrix
+        size: 100
+        seed: 42
+        params:
+          min_n: 2
+          max_n: 6
+      - dataset: caesar_cipher
+        size: 100
+        seed: 42
+        params:
+          max_words: 10
+      - dataset: cryptarithm
+        size: 100
+        seed: 42
+      - dataset: isomorphic_strings
+        size: 100
+        seed: 42
+        params:
+          max_string_length: 8
+      - dataset: jugs
+        size: 100
+        seed: 42
+        params:
+          difficulty: 6
+      - dataset: rotate_matrix
+        size: 100
+        seed: 42
+        params:
+          min_n: 2
+          max_n: 6
+      - dataset: string_manipulation
+        size: 100
+        seed: 42
+        params:
+          max_string_length: 15
+          max_num_rules: 6

training/evaluations/inter_generalisation/arc.yaml

@@ -0,0 +1,32 @@
+# Config used for evaluating inter-domain generalisation experiment models on ARC test data
+
+# Models evaluated on this config:
+# Qwen/Qwen2.5-3B-Instruct (original model)
+# inter_algebra_qwen_3b_500 (original + 500 GRPO steps on algebra RG data)
+
+model_path: ../models/inter_algebra_qwen_3b_500  # Change to the model to be evaluated
+
+max_tokens: 2048  # From max_response_length in training config
+top_p: 0.9  # From rollout top_p
+temperature: 0.6  # Lower temperature for more focused responses
+
+developer_prompt: DeepSeekZero
+developer_role: system
+
+output_dir: results
+save_metadata: true
+save_full_results: true
+eval_repeats: 3
+
+categories:
+  - category: arc
+    datasets:
+      - dataset: arc_1d
+        size: 100
+        seed: 42
+      - dataset: arc_agi
+        size: 100
+        seed: 42
+      - dataset: rearc
+        size: 100
+        seed: 42

training/evaluations/inter_generalisation/arithmetic.yaml

@@ -0,0 +1,68 @@
+# Config used for evaluating inter-domain generalisation experiment models on arithmetic test data
+
+# Models evaluated on this config:
+# Qwen/Qwen2.5-3B-Instruct (original model)
+# inter_algorithmic_qwen_3b_500 (original + 500 GRPO steps on algorithmic RG data)
+
+model_path: ../models/inter_algorithmic_qwen_3b_500  # Change to the model to be evaluated
+
+max_tokens: 2048  # From max_response_length in training config
+top_p: 0.9  # From rollout top_p
+temperature: 0.6  # Lower temperature for more focused responses
+
+developer_prompt: DeepSeekZero
+developer_role: system
+
+output_dir: results
+save_metadata: true
+save_full_results: true
+eval_repeats: 3
+
+categories:
+  - category: arithmetic
+    datasets:
+      - dataset: basic_arithmetic
+        size: 100
+        seed: 42
+      - dataset: bitwise_arithmetic
+        size: 100
+        seed: 42
+      - dataset: calendar_arithmetic
+        size: 100
+        seed: 42
+      - dataset: chain_sum
+        size: 100
+        seed: 42
+      - dataset: count_bits
+        size: 100
+        seed: 42
+      - dataset: decimal_arithmetic
+        size: 100
+        seed: 42
+      - dataset: decimal_chain_sum
+        size: 100
+        seed: 42
+      - dataset: dice
+        size: 100
+        seed: 42
+      - dataset: fraction_simplification
+        size: 100
+        seed: 42
+      - dataset: gcd
+        size: 100
+        seed: 42
+      - dataset: lcm
+        size: 100
+        seed: 42
+      - dataset: power_function
+        size: 100
+        seed: 42
+      - dataset: prime_factorization
+        size: 100
+        seed: 42
+      - dataset: products
+        size: 100
+        seed: 42
+      - dataset: time_intervals
+        size: 100
+        seed: 42

training/evaluations/inter_generalisation/cognition.yaml

@@ -0,0 +1,44 @@
+# Config used for evaluating inter-domain generalisation experiment models on cognition test data
+
+# Models evaluated on this config:
+# Qwen/Qwen2.5-3B-Instruct (original model)
+# inter_logic_qwen_3b_400 (original + 400 GRPO steps on logic RG data)
+
+model_path: ../models/inter_logic_qwen_3b_400  # Change to the model to be evaluated
+
+max_tokens: 2048  # From max_response_length in training config
+top_p: 0.9  # From rollout top_p
+temperature: 0.6  # Lower temperature for more focused responses
+
+developer_prompt: DeepSeekZero
+developer_role: system
+
+output_dir: results
+save_metadata: true
+save_full_results: true
+eval_repeats: 3
+
+categories:
+  - category: cognition
+    datasets:
+      - dataset: color_cube_rotation
+        size: 100
+        seed: 42
+      - dataset: figlet_font
+        size: 100
+        seed: 42
+      - dataset: modulo_grid
+        size: 100
+        seed: 42
+      - dataset: needle_haystack
+        size: 100
+        seed: 42
+      - dataset: number_sequence
+        size: 100
+        seed: 42
+      - dataset: rectangle_count
+        size: 100
+        seed: 42
+      - dataset: rubiks_cube
+        size: 100
+        seed: 42

training/evaluations/inter_generalisation/games.yaml

@@ -0,0 +1,48 @@
+# Config used for evaluating inter-domain generalisation experiment models on games test data
+
+# Models evaluated on this config:
+# Qwen/Qwen2.5-3B-Instruct (original model)
+# inter_algebra_qwen_3b_500 (original + 500 GRPO steps on algebra RG data)
+# inter_logic_qwen_3b_400 (original + 400 GRPO steps on logic RG data)
+
+model_path: ../models/inter_logic_qwen_3b_400  # Change to the model to be evaluated
+
+max_tokens: 2048  # From max_response_length in training config
+top_p: 0.9  # From rollout top_p
+temperature: 0.6  # Lower temperature for more focused responses
+
+developer_prompt: DeepSeekZero
+developer_role: system
+
+output_dir: results
+save_metadata: true
+save_full_results: true
+eval_repeats: 3
+
+categories:
+  - category: games
+    datasets:
+      - dataset: knight_swap
+        size: 100
+        seed: 42
+      - dataset: mahjong_puzzle
+        size: 100
+        seed: 42
+      - dataset: maze
+        size: 100
+        seed: 42
+      - dataset: mini_sudoku
+        size: 100
+        seed: 42
+      - dataset: n_queens
+        size: 100
+        seed: 42
+      - dataset: rush_hour
+        size: 100
+        seed: 42
+      - dataset: sokoban
+        size: 100
+        seed: 42
+      - dataset: tsumego
+        size: 100
+        seed: 42

training/evaluations/inter_generalisation/geometry.yaml

@@ -0,0 +1,29 @@
+# Config used for evaluating inter-domain generalisation experiment models on geometry test data
+
+# Models evaluated on this config:
+# Qwen/Qwen2.5-3B-Instruct (original model)
+# inter_algorithmic_qwen_3b_500 (original + 500 GRPO steps on algorithmic RG data)
+
+model_path: ../models/inter_algorithmic_qwen_3b_500  # Change to the model to be evaluated
+
+max_tokens: 2048  # From max_response_length in training config
+top_p: 0.9  # From rollout top_p
+temperature: 0.6  # Lower temperature for more focused responses
+
+developer_prompt: DeepSeekZero
+developer_role: system
+
+output_dir: results
+save_metadata: true
+save_full_results: true
+eval_repeats: 3
+
+categories:
+  - category: geometry
+    datasets:
+      - dataset: advanced_geometry
+        size: 100
+        seed: 42
+      - dataset: simple_geometry
+        size: 100
+        seed: 42

training/evaluations/inter_generalisation/graphs.yaml

@@ -0,0 +1,38 @@
+# Config used for evaluating inter-domain generalisation experiment models on graphs test data
+
+# Models evaluated on this config:
+# Qwen/Qwen2.5-3B-Instruct (original model)
+# inter_logic_qwen_3b_400 (original + 400 GRPO steps on logic RG data)
+
+model_path: ../models/inter_logic_qwen_3b_400  # Change to the model to be evaluated
+
+max_tokens: 2048  # From max_response_length in training config
+top_p: 0.9  # From rollout top_p
+temperature: 0.6  # Lower temperature for more focused responses
+
+developer_prompt: DeepSeekZero
+developer_role: system
+
+output_dir: results
+save_metadata: true
+save_full_results: true
+eval_repeats: 3
+
+categories:
+  - category: graphs
+    datasets:
+      - dataset: course_schedule
+        size: 100
+        seed: 42
+      - dataset: family_relationships
+        size: 100
+        seed: 42
+      - dataset: largest_island
+        size: 100
+        seed: 42
+      - dataset: quantum_lock
+        size: 100
+        seed: 42
+      - dataset: shortest_path
+        size: 100
+        seed: 42