diff --git a/README.md b/README.md index 90088113..1aa191cc 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,30 @@ We are building a python library of procedural dataset generators and algorithmi The goal is to generate virtually infinite data with adjustable complexity. +### Set up for development +1. Clone the project +``` +git clone https://github.com/open-thought/reasoning-gym.git +``` +2. Create a virtual environment(Here we use conda) +``` +conda create --name reasoning_gym python=3.12 -y +conda activate reasoning_gym +``` +3. Link project and install dependencies +``` +pip install -e . +``` +4. Install development dependencies +``` +pip install -r requirements-dev.txt +``` + +>NOTE: To consume the APIs in reasoning_gym, just install from pip using the following +``` +pip install reasoning-gym +``` + ### How to instantiate a task dataset? Example: @@ -35,9 +59,11 @@ Available dataset names (which can be used with `create_dataset()`): 'base_conversion', 'caesar_cipher', 'letter_counting', +'letter_jumble', 'number_filtering', 'number_sorting', -'word_reversal', +'spell_backward', +'word_sequence_reversal', 'basic_arithmetic', 'chain_sum', 'fraction_simplification', @@ -53,7 +79,7 @@ Available dataset names (which can be used with `create_dataset()`): 'sudoku', 'family_relationships', 'propositional_logic', -'syllogism' +'syllogism', ``` ### Task Overview @@ -82,7 +108,8 @@ Available dataset names (which can be used with `create_dataset()`): - `NumberSortingDataset`: Sort lists of numbers in ascending or descending order - `LetterJumbleDataset`: Unscramble words that have had their letters randomly jumbled - `SentenceReorderingDataset`: Reorder sentence after words in it have been randomly shuffled -- `WordReversalDataset`: Reverse word order in text spans +- `SpellBackwardDataset`: Spell individual words backward (e.g. "sun" -> "nus") +- `WordSequenceReversalDataset`: Reverse word order in text spans #### Cognition Tasks diff --git a/reasoning_gym/algorithmic/__init__.py b/reasoning_gym/algorithmic/__init__.py index 5eb551f0..cc3b598f 100644 --- a/reasoning_gym/algorithmic/__init__.py +++ b/reasoning_gym/algorithmic/__init__.py @@ -12,10 +12,13 @@ from .letter_counting import LetterCountingConfig, LetterCountingDataset from .letter_jumble import LetterJumbleConfig, LetterJumbleDataset from .number_filtering import NumberFilteringConfig, NumberFilteringDataset from .number_sorting import NumberSortingConfig, NumberSortingDataset -from .word_reversal import WordReversalConfig, WordReversalDataset from .sentence_reordering import SentenceReorderingConfig, SentenceReorderingDataset +from .spell_backward import SpellBackwardConfig, SpellBackwardDataset +from .word_sequence_reversal import WordSequenceReversalConfig, WordSequenceReversalDataset __all__ = [ + "SpellBackwardConfig", + "SpellBackwardDataset", "BaseConversionConfig", "BaseConversionDataset", "CaesarCipherConfig", @@ -28,8 +31,8 @@ __all__ = [ "NumberFilteringDataset", "NumberSortingConfig", "NumberSortingDataset", - "WordReversalConfig", - "WordReversalDataset", "SentenceReorderingConfig", "SentenceReorderingDataset", + "WordSequenceReversalConfig", + "WordSequenceReversalDataset", ] diff --git a/reasoning_gym/algorithmic/spell_backward.py b/reasoning_gym/algorithmic/spell_backward.py new file mode 100644 index 00000000..59b163ee --- /dev/null +++ b/reasoning_gym/algorithmic/spell_backward.py @@ -0,0 +1,53 @@ +"""Spell backward task generator""" + +import re +from dataclasses import dataclass +from random import Random +from typing import Optional + +from ..data import read_data_file +from ..factory import ProceduralDataset, register_dataset + + +@dataclass +class SpellBackwardConfig: + """Configuration for spelling words backward task generation""" + + min_word_len: int = 3 # Minimum word length + seed: Optional[int] = None + size: int = 500 # Virtual dataset size + + def validate(self) -> None: + """Validate configuration parameters""" + assert self.min_word_len > 0, "min_word_len must be positive" + + +class SpellBackwardDataset(ProceduralDataset): + """Generates tasks to spell words backward""" + + def __init__(self, config: SpellBackwardConfig): + super().__init__(config=config, seed=config.seed, size=config.size) + + # Load and preprocess text + text = read_data_file("in_the_year_2889.txt") + # Extract words and clean them to contain only alphanumeric characters + self.words = [ + word for word in re.findall(r"\b\w+\b", text) if word.isalnum() and len(word) >= config.min_word_len + ] + + def __getitem__(self, idx: int) -> dict: + """Generate a single spell backward task""" + rng = Random(self.seed + idx) + + # Select random word + word = rng.choice(self.words) + answer = word[::-1] + + return { + "question": f"Spell this word backward (example: sun -> nus): {word}", + "answer": answer, + "metadata": {"word": word, "word_len": len(word)}, + } + + +register_dataset("spell_backward", SpellBackwardDataset, SpellBackwardConfig) diff --git a/reasoning_gym/algorithmic/word_reversal.py b/reasoning_gym/algorithmic/word_sequence_reversal.py similarity index 81% rename from reasoning_gym/algorithmic/word_reversal.py rename to reasoning_gym/algorithmic/word_sequence_reversal.py index b08b459d..5ad94ea7 100644 --- a/reasoning_gym/algorithmic/word_reversal.py +++ b/reasoning_gym/algorithmic/word_sequence_reversal.py @@ -10,8 +10,8 @@ from ..factory import ProceduralDataset, register_dataset @dataclass -class WordReversalConfig: - """Configuration for word reversal task generation""" +class WordSequenceReversalConfig: + """Configuration for word sequence reversal task generation""" min_words: int = 3 # Minimum words in list max_words: int = 8 # Maximum words in list @@ -24,10 +24,10 @@ class WordReversalConfig: assert self.max_words >= self.min_words, "max_words must be >= min_words" -class WordReversalDataset(ProceduralDataset): - """Generates word reversal tasks from text spans""" +class WordSequenceReversalDataset(ProceduralDataset): + """Generates word sequence reversal tasks from text spans""" - def __init__(self, config: WordReversalConfig): + def __init__(self, config: WordSequenceReversalConfig): super().__init__(config=config, seed=config.seed, size=config.size) # Load and preprocess text @@ -55,4 +55,4 @@ class WordReversalDataset(ProceduralDataset): } -register_dataset("word_reversal", WordReversalDataset, WordReversalConfig) +register_dataset("word_sequence_reversal", WordSequenceReversalDataset, WordSequenceReversalConfig) diff --git a/tests/test_spell_backward.py b/tests/test_spell_backward.py new file mode 100644 index 00000000..2db86c62 --- /dev/null +++ b/tests/test_spell_backward.py @@ -0,0 +1,59 @@ +"""Tests for spell backward task generation""" + +import pytest + +from reasoning_gym.algorithmic.spell_backward import SpellBackwardConfig, SpellBackwardDataset + + +def test_spell_backward_config_validation(): + """Test that invalid configs raise appropriate errors""" + with pytest.raises(AssertionError): + config = SpellBackwardConfig(min_word_len=0) + config.validate() + + +def test_spell_backward_dataset_deterministic(): + """Test that dataset generates same items with same seed""" + config = SpellBackwardConfig(seed=42, size=10) + dataset1 = SpellBackwardDataset(config) + dataset2 = SpellBackwardDataset(config) + + for i in range(len(dataset1)): + assert dataset1[i] == dataset2[i] + + +def test_spell_backward_dataset_items(): + """Test basic properties of generated items""" + config = SpellBackwardConfig(min_word_len=3, size=10, seed=42) + dataset = SpellBackwardDataset(config) + + for i in range(len(dataset)): + item = dataset[i] + # Check item structure + assert isinstance(item, dict) + assert "question" in item + assert "answer" in item + assert "metadata" in item + + # Check metadata + assert "word" in item["metadata"] + assert "word_len" in item["metadata"] + + # Verify word length constraint + word = item["metadata"]["word"] + assert len(word) >= config.min_word_len + + # Verify answer is correct + assert item["answer"] == word[::-1] + + +def test_spell_backward_dataset_iteration(): + """Test that iteration respects dataset size""" + config = SpellBackwardConfig(size=5, seed=42) + dataset = SpellBackwardDataset(config) + + items = list(dataset) + assert len(items) == config.size + + # Test multiple iterations yield same items + assert items == list(dataset) diff --git a/tests/test_word_reversal.py b/tests/test_word_sequence_reversal.py similarity index 61% rename from tests/test_word_reversal.py rename to tests/test_word_sequence_reversal.py index eec4bef1..a117bfba 100644 --- a/tests/test_word_reversal.py +++ b/tests/test_word_sequence_reversal.py @@ -1,35 +1,33 @@ -"""Tests for word reversal task generation""" - import pytest -from reasoning_gym.algorithmic.word_reversal import WordReversalConfig, WordReversalDataset +from reasoning_gym.algorithmic.word_sequence_reversal import WordSequenceReversalConfig, WordSequenceReversalDataset -def test_word_reversal_config_validation(): +def test_word_sequence_reversal_config_validation(): """Test that invalid configs raise appropriate errors""" with pytest.raises(AssertionError): - config = WordReversalConfig(min_words=0) + config = WordSequenceReversalConfig(min_words=0) config.validate() with pytest.raises(AssertionError): - config = WordReversalConfig(min_words=10, max_words=5) + config = WordSequenceReversalConfig(min_words=10, max_words=5) config.validate() -def test_word_reversal_dataset_deterministic(): +def test_word_sequence_reversal_dataset_deterministic(): """Test that dataset generates same items with same seed""" - config = WordReversalConfig(seed=42, size=10) - dataset1 = WordReversalDataset(config) - dataset2 = WordReversalDataset(config) + config = WordSequenceReversalConfig(seed=42, size=10) + dataset1 = WordSequenceReversalDataset(config) + dataset2 = WordSequenceReversalDataset(config) for i in range(len(dataset1)): assert dataset1[i] == dataset2[i] -def test_word_reversal_dataset_items(): +def test_word_sequence_reversal_dataset_items(): """Test basic properties of generated items""" - config = WordReversalConfig(min_words=3, max_words=6, size=10, seed=42) - dataset = WordReversalDataset(config) + config = WordSequenceReversalConfig(min_words=3, max_words=6, size=10, seed=42) + dataset = WordSequenceReversalDataset(config) for i in range(len(dataset)): item = dataset[i] @@ -54,10 +52,10 @@ def test_word_reversal_dataset_items(): assert answer_words == list(reversed(question_words)) -def test_word_reversal_dataset_iteration(): +def test_word_sequence_reversal_dataset_iteration(): """Test that iteration respects dataset size""" - config = WordReversalConfig(size=5, seed=42) - dataset = WordReversalDataset(config) + config = WordSequenceReversalConfig(size=5, seed=42) + dataset = WordSequenceReversalDataset(config) items = list(dataset) assert len(items) == config.size @@ -66,10 +64,10 @@ def test_word_reversal_dataset_iteration(): assert items == list(dataset) -def test_word_reversal_text_preprocessing(): +def test_word_sequence_reversal_text_preprocessing(): """Test that text preprocessing handles edge cases""" - config = WordReversalConfig(size=1, seed=42) - dataset = WordReversalDataset(config) + config = WordSequenceReversalConfig(size=1, seed=42) + dataset = WordSequenceReversalDataset(config) # Verify words were extracted from text assert len(dataset.words) > 0