diff --git a/reasoning_gym/algorithmic/base_conversion.py b/reasoning_gym/algorithmic/base_conversion.py index f3b9d6a1..15607ed4 100644 --- a/reasoning_gym/algorithmic/base_conversion.py +++ b/reasoning_gym/algorithmic/base_conversion.py @@ -125,8 +125,8 @@ class BaseConversionCurriculum(BaseCurriculum): self._define_attributes( RangeAttributeDefinition( name="base", - levels=[9, 18, 27, 36], - default_level=0, + levels=[2, 9, 18, 27, 36], + default_level=1, description="The base of the number system", attr_type=AttributeType.APPEND, min_value=2, diff --git a/reasoning_gym/coaching/base_curriculum.py b/reasoning_gym/coaching/base_curriculum.py index 22c031a1..5b7463c4 100644 --- a/reasoning_gym/coaching/base_curriculum.py +++ b/reasoning_gym/coaching/base_curriculum.py @@ -106,3 +106,73 @@ class BaseCurriculum: self.set_attr_level(attr_name, current_level - 1) return True return False + + def get_max_level(self) -> int: + """ + Get the maximum level currently set across all attributes. + + Returns: + int: The maximum level currently set across all attributes + """ + if not self._attributes: + return 0 + + return max(self.get_attr_level(attr_name) for attr_name in self._attributes) + + def set_global_level(self, level: int) -> None: + """ + Set all attributes to the specified level. + If the level exceeds the number of defined levels for an attribute, + use the highest defined level for that attribute. + + Args: + level: The level to set for all attributes + """ + for attr_name, attr in self._attributes.items(): + # Use the highest defined level if the requested level exceeds available levels + attr_level = min(level, len(attr.levels) - 1) + self.set_attr_level(attr_name, attr_level) + + def increment_global_level(self) -> bool: + """ + Increment the level of all attributes by one from the current maximum level. + + Returns: + bool: True if at least one attribute's level was incremented, False otherwise + """ + current_max = self.get_max_level() + target_level = current_max + 1 + + # Check if any attribute can be incremented + can_increment = any( + self.get_attr_level(attr_name) < len(self._attributes[attr_name].levels) - 1 + for attr_name in self._attributes + ) + + if can_increment: + for attr_name, attr in self._attributes.items(): + # Only increment if the attribute is not already at its maximum level + if self.get_attr_level(attr_name) < len(attr.levels) - 1: + # Don't exceed the attribute's maximum level + new_level = min(target_level, len(attr.levels) - 1) + self.set_attr_level(attr_name, new_level) + return True + return False + + def decrement_global_level(self) -> bool: + """ + Decrement the level of all attributes by one from the current maximum level. + + Returns: + bool: True if at least one attribute's level was decremented, False otherwise + """ + current_max = self.get_max_level() + + if current_max > 0: + target_level = current_max - 1 + for attr_name in self._attributes: + # Only decrement if the attribute is at the current maximum level + if self.get_attr_level(attr_name) == current_max: + self.set_attr_level(attr_name, target_level) + return True + return False diff --git a/scripts/debug_curricula.py b/scripts/debug_curricula.py new file mode 100755 index 00000000..5e2f76cc --- /dev/null +++ b/scripts/debug_curricula.py @@ -0,0 +1,182 @@ +#!/usr/bin/env -S PYTHONHASHSEED=1 python3 +"""Generate a markdown document showing curriculum progression for all datasets""" + +import argparse +import textwrap +from copy import deepcopy +from pathlib import Path +from typing import Optional + +from tqdm import tqdm + +from reasoning_gym.factory import CURRICULA, DATASETS, create_curriculum, create_dataset + + +def generate_curricula_doc( + num_examples: int = 1, show_config: bool = False, dataset_names: Optional[list[str]] = None +) -> str: + """Generate markdown content showing curriculum progression + + Args: + num_examples: Number of examples to generate per difficulty level + show_config: Whether to show the effective dataset configuration + dataset_names: Optional list of specific dataset names to generate documentation for + """ + + # Start with header + content = ["# Reasoning Gym Curriculum Progression\n"] + content.append("This document shows how tasks change as curriculum difficulty increases for each dataset.\n\n") + + # Get datasets with curricula + all_datasets_with_curricula = sorted([name for name in DATASETS.keys() if name in CURRICULA]) + + # Filter to specific datasets if provided + if dataset_names: + # Validate all requested datasets + for name in dataset_names: + if name not in CURRICULA: + raise ValueError(f"Dataset '{name}' does not have a curriculum") + datasets_with_curricula = dataset_names + else: + datasets_with_curricula = all_datasets_with_curricula + + # Add index + content.append("## Available Curricula\n") + for name in datasets_with_curricula: + # Create anchor link + anchor = name.replace(" ", "-").lower() + content.append(f"- [{name}](#{anchor})\n") + content.append("\n") + + # Add examples for each dataset with curriculum + content.append("## Curriculum Progression Examples\n") + + # Create progress bar for datasets + for name in tqdm(datasets_with_curricula, desc="Processing datasets"): + # Add dataset header with anchor + content.append(f"### {name}\n") + + # Get curriculum and dataset class + curriculum = create_curriculum(name) + dataset_cls, config_cls = DATASETS[name] + + # Get dataset class docstring if available + try: + dataset = create_dataset(name, seed=42) + if dataset.__class__.__doc__: + doc = textwrap.dedent(dataset.__class__.__doc__.strip()) + content.append(f"{doc}\n\n") + except Exception as e: + content.append(f"*Error loading dataset: {str(e)}*\n\n") + continue + + # Show curriculum attributes + content.append("#### Curriculum Attributes\n") + for attr_name, attr in curriculum.attributes.items(): + content.append(f"- **{attr_name}**: {attr.description}\n") + content.append(f" - Levels: {attr.levels}\n") + content.append("\n") + + # Show progression with all attributes increasing simultaneously + content.append(f"#### Overall Difficulty Progression\n") + + # Find the maximum number of levels across all attributes + max_levels = max(len(attr.levels) for attr in curriculum.attributes.values()) + + # Show examples at each difficulty level + for level in tqdm(range(max_levels), desc=f"Dataset: {name}, Overall Difficulty", leave=False): + try: + # Reset curriculum to defaults + curriculum = create_curriculum(name) + + # Set all attributes to this level using the global level function + curriculum.set_global_level(level) + + # Generate config with this level + config = curriculum.generate_configuration({"seed": 42 + level}) + + # Create dataset with this config + dataset = dataset_cls(config=config) + + # Show level and example + content.append(f"##### Difficulty Level {level}\n") + + # Show the current values for each attribute + content.append("Attribute values:\n") + for attr_name, attr in curriculum.attributes.items(): + attr_level = min(level, len(attr.levels) - 1) + content.append(f"- {attr_name}: {attr.levels[attr_level]}\n") + + # Show the effective configuration if requested + if show_config: + content.append("\nEffective configuration:\n") + for key, value in vars(config).items(): + if key != "seed" and key != "size": + content.append(f"- {key}: {value}\n") + + # Generate multiple examples + for ex_idx in range(num_examples): + # Get example + example = dataset[ex_idx] + + content.append(f"\n```\n") + if num_examples > 1: + content.append(f"Example {ex_idx + 1}:\n") + content.append(f"Question: {example['question']}\n") + content.append(f"Answer: {example['answer']}\n") + if example.get("metadata"): + content.append(f"Metadata: {example['metadata']}\n") + content.append("```\n") + except Exception as e: + content.append(f"##### Difficulty Level {level}\n") + content.append(f"*Error generating example: {str(e)}*\n\n") + + content.append("\n") + + return "".join(content) + + +def main(): + """Generate curricula markdown file""" + # Parse command line arguments + parser = argparse.ArgumentParser(description="Generate curriculum documentation") + parser.add_argument("--examples", type=int, default=3, help="Number of examples to generate per difficulty level") + parser.add_argument("--show-config", action="store_true", help="Show the effective dataset configuration") + parser.add_argument( + "--output", type=str, default="CURRICULA.md", help="Output file path (relative to project root)" + ) + parser.add_argument( + "--dataset", type=str, help="Generate documentation for specific datasets (comma-separated list)" + ) + args = parser.parse_args() + + # Ensure scripts directory exists + script_dir = Path(__file__).parent + if not script_dir.exists(): + script_dir.mkdir(parents=True) + + print(f"Generating curricula documentation...") + print(f"Number of examples per level: {args.examples}") + print(f"Show configuration: {args.show_config}") + + # Parse dataset names if provided + dataset_names = None + if args.dataset: + dataset_names = [name.strip() for name in args.dataset.split(",")] + print(f"Generating documentation for datasets: {', '.join(dataset_names)}") + + curricula_path = script_dir.parent / args.output + + curricula_content = generate_curricula_doc( + num_examples=args.examples, show_config=args.show_config, dataset_names=dataset_names + ) + + with open(curricula_path, "w") as f: + f.write(curricula_content) + f.write("\n") + + print(f"Generated curricula documentation at {curricula_path}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_base_conversion.py b/tests/test_base_conversion.py index 1e0ebe55..054257d0 100644 --- a/tests/test_base_conversion.py +++ b/tests/test_base_conversion.py @@ -177,18 +177,18 @@ def test_base_conversion__curriculum(): base_cfg: BaseConversionConfig = curriculum.generate_configuration(base_value) assert base_cfg.seed == 1 assert base_cfg.size == 150 - assert base_cfg.min_base == 9 and base_cfg.max_base == 9 + assert base_cfg.min_base == 2 and base_cfg.max_base == 9 assert base_cfg.min_value == 1000 and base_cfg.max_value == 1000 # test incrementing attribute levels curriculum.increment_attr_level("base") curriculum.increment_attr_level("value") increased_cfg = curriculum.generate_configuration(base_value) - assert increased_cfg.min_base == 9 and increased_cfg.max_base == 18 + assert increased_cfg.min_base == 2 and increased_cfg.max_base == 18 assert increased_cfg.min_value == 1000 and increased_cfg.max_value == 10000 # test decrementing attribute level for base again curriculum.decrement_attr_level("base") partially_decreased_cfg = curriculum.generate_configuration(base_value) - assert partially_decreased_cfg.min_base == 9 and partially_decreased_cfg.max_base == 9 + assert partially_decreased_cfg.min_base == 2 and partially_decreased_cfg.max_base == 9 assert partially_decreased_cfg.min_value == 1000 and partially_decreased_cfg.max_value == 10000