mirror of
https://github.com/open-thought/reasoning-gym.git
synced 2026-04-19 12:58:07 +00:00
* feat: Add initial server structure with configuration, registry, and middleware * feat: Add chain_sum dataset to experiment registry test * fix: Update test_registry to use DatasetSpec for composite config validation * refactor: Update Pydantic config to use json_schema_extra and ConfigDict * feat: Add Pydantic models for API request/response data * feat: Implement basic experiment management endpoints with tests * feat: Implement composite configuration endpoints for experiments * fix: Add missing DatasetConfigUpdate import in server.py * refactor: Update dataset config update method to properly merge config updates * fix: Correctly retrieve current dataset config in composite endpoint * feat: Add basic CLI structure with experiments and config commands * feat: Add initial CLI tool with basic experiment management commands * refactor: Reorganize CLI package structure and fix import paths * refactor: Implement initial CLI commands for experiment management * feat: Implement HTTP client for Reasoning Gym server in RGC CLI tool * fix: Move print statements inside try block to resolve SyntaxError * fix: Resolve SyntaxError in edit_config function by adding missing except block * feat: Add default app instance in server module for easier uvicorn startup * docs: Add README.md with server and RGC tool documentation * remove unused files * refactor: Remove unsupported type annotation in registry.py * refactor: Move ExperimentRegistry to coaching module and add Experiment class * fix: Add missing CompositeDataset import in test_registry.py * refactor: Implement lazy ASGI app creation for server initialization * feat: Add health check command to RGC CLI for server connection * feat: Add version tracking support to CompositeDataset * feat: Add DatasetVersionManager for tracking dataset versions * feat: Add entry_id metadata and score_answer_with_id method to CompositeDataset * feat: Add entry_id metadata combining version and index * fix: Resolve undefined variable by storing version_id before use * test: Add comprehensive unit tests for score_answer_with_id() function * test: Add comprehensive version tracking test for dataset config updates * feat: Validate dataset weights are positive in CompositeDataset initialization * feat: Add weight update and normalization methods to CompositeDataset * refactor: Centralize weight normalization in CompositeDataset and allow zero-weight datasets * feat: Add negative weight validation to CompositeDataset constructor * feat: Add duplicate dataset name check in CompositeDataset and update test * refactor: Move duplicate dataset name check inside dataset iteration loop * refactor: Update CompositeDataset weight management to use config as source of truth * refactor: Move duplicate dataset name check to CompositeConfig.validate() * test: Update composite dataset weight test assertions and validation * feat: Add methods to add and remove datasets in CompositeDataset * refactor: Remove weight normalization and use unnormalized weights directly * refactor: Remove redundant total weight check in update_dataset_weights * feat: Add batch generation and scoring endpoints to server * fix: Import BatchEntry in server.py to resolve undefined name error * refactor: Update ReasoningGymDataset to use server for batch generation and scoring * fix: Add missing List and Dict type imports * feat: Add get_batch() and score_outputs() methods to RGClient * test: Add unit tests for generate_batch and score_outputs endpoints * refactor: Add DatasetVersionManager to Experiment class and CompositeDataset constructor * feat: Add validation for base_index and batch_size in generate_batch endpoint * refactor: Remove unused BatchRequest type from imports * refactor: Convert models to use Pydantic exclusively * test: Update scoring endpoint tests to use correct request model format * refactor: Rename ScoreItem to AnswerItem and update related code * feat: Update scoring endpoint to return ordered ScoringResponse with scores and entry_ids * fix: Add missing ScoringResponse import in server.py * move verl ppo sample with server into own file * refactor: Use Pydantic models for get_batch() and score_outputs() in RGClient * refactor: Update client methods to use Pydantic models for type safety * refactor: Use Pydantic models for experiment and dataset config operations * refactor: Clean up duplicate methods and improve error handling in main.py * first bits of rg server use for verl * refactor: Optimize scoring with single HTTP request in _score_output * fix: Correct experiment creation with ExperimentCreate object * grpo tests with server
76 lines
2.3 KiB
Python
76 lines
2.3 KiB
Python
"""Version manager for tracking dataset versions."""
|
|
|
|
from typing import Dict, Optional, Tuple
|
|
|
|
from .dataset import ProceduralDataset
|
|
|
|
|
|
class DatasetVersionManager:
|
|
"""Manages versioned ProceduralDataset instances and their configurations."""
|
|
|
|
def __init__(self):
|
|
"""Initialize the version manager."""
|
|
self.current_version = 0
|
|
# version_id -> (dataset_name, dataset_instance)
|
|
self.datasets: Dict[int, Tuple[str, ProceduralDataset]] = {}
|
|
|
|
def register_dataset(self, name: str, dataset: ProceduralDataset) -> int:
|
|
"""
|
|
Register a new dataset version.
|
|
|
|
Args:
|
|
name: Name/identifier of the dataset type
|
|
dataset: Instance of ProceduralDataset
|
|
|
|
Returns:
|
|
version_id: Unique identifier for this dataset version
|
|
"""
|
|
self.current_version += 1
|
|
self.datasets[self.current_version] = (name, dataset)
|
|
return self.current_version
|
|
|
|
def get_dataset(self, version_id: int) -> Optional[Tuple[str, ProceduralDataset]]:
|
|
"""
|
|
Retrieve a dataset by its version ID.
|
|
|
|
Args:
|
|
version_id: The version identifier
|
|
|
|
Returns:
|
|
Tuple of (dataset_name, dataset_instance) if found, None otherwise
|
|
"""
|
|
return self.datasets.get(version_id)
|
|
|
|
def get_entry(self, version_id: int, index: int) -> Dict[str, any]:
|
|
"""
|
|
Get a specific entry from a versioned dataset.
|
|
|
|
Args:
|
|
version_id: The version identifier
|
|
index: Index of the entry to retrieve
|
|
|
|
Returns:
|
|
The dataset entry
|
|
|
|
Raises:
|
|
KeyError: If version_id is not found
|
|
"""
|
|
if version_id not in self.datasets:
|
|
raise KeyError(f"Dataset version {version_id} not found")
|
|
|
|
_, dataset = self.datasets[version_id]
|
|
return dataset[index]
|
|
|
|
def cleanup_old_versions(self, keep_latest: int = 10):
|
|
"""
|
|
Remove old dataset versions to free memory.
|
|
|
|
Args:
|
|
keep_latest: Number of most recent versions to keep
|
|
"""
|
|
if len(self.datasets) <= keep_latest:
|
|
return
|
|
|
|
versions_to_remove = sorted(self.datasets.keys())[:-keep_latest]
|
|
for version in versions_to_remove:
|
|
del self.datasets[version]
|