From ea1c7795effb9039955d387da822c6bb2713f2b2 Mon Sep 17 00:00:00 2001 From: abdulhakeem Date: Sat, 25 Jan 2025 13:13:06 -0600 Subject: [PATCH 01/15] Add set up instructions --- README.md | 20 ++++++++++++++++++++ poetry.lock | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 poetry.lock diff --git a/README.md b/README.md index dae6975a..d81bb85b 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,26 @@ We are building a python library of procedural dataset generators and algorithmi The goal is to generate virtually infinite data with adjustable complexity. +### Set up +1. Clone the project +``` +git clone https://github.com/open-thought/reasoning-gym.git +``` +2. Create a virtual environment(Here we use conda) +``` +conda create --name reasoning_gym python=3.12 -y +conda activate reasoning_gym +``` +3. Run the `requirements-dev.txt` file +``` +pip install -r requirements.txt +``` +4. Install and run poetry to bring in other dependencies(`sympy` etc.) +``` +conda install -c conda-forge poetry +poetry +``` + ### How to instantiate a task dataset? Example: diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 00000000..4a8f94fe --- /dev/null +++ b/poetry.lock @@ -0,0 +1,42 @@ +# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand. + +[[package]] +name = "mpmath" +version = "1.3.0" +description = "Python library for arbitrary-precision floating-point arithmetic" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, + {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, +] + +[package.extras] +develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] +docs = ["sphinx"] +gmpy = ["gmpy2 (>=2.1.0a4)"] +tests = ["pytest (>=4.6)"] + +[[package]] +name = "sympy" +version = "1.13.3" +description = "Computer algebra system (CAS) in Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "sympy-1.13.3-py3-none-any.whl", hash = "sha256:54612cf55a62755ee71824ce692986f23c88ffa77207b30c1368eda4a7060f73"}, + {file = "sympy-1.13.3.tar.gz", hash = "sha256:b27fd2c6530e0ab39e275fc9b683895367e51d5da91baa8d3d64db2565fec4d9"}, +] + +[package.dependencies] +mpmath = ">=1.1.0,<1.4" + +[package.extras] +dev = ["hypothesis (>=6.70.0)", "pytest (>=7.1.0)"] + +[metadata] +lock-version = "2.1" +python-versions = ">=3.12" +content-hash = "bd48874a1c70818eccccc0c334cc40282d1e1c8bcc1d1369ccf3aa42489007d5" From 53139e38b74ed37a4f860251ef25951ff95ec998 Mon Sep 17 00:00:00 2001 From: abdulhakeem Date: Sat, 25 Jan 2025 13:19:08 -0600 Subject: [PATCH 02/15] Remove poetry.lock --- .gitignore | 3 +++ poetry.lock | 42 ------------------------------------------ 2 files changed, 3 insertions(+), 42 deletions(-) delete mode 100644 poetry.lock diff --git a/.gitignore b/.gitignore index be4071bb..e2ea009e 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,6 @@ ENV/ .coverage htmlcov/ .pytest_cache/ + +# Poetry lock file +poetry.lock diff --git a/poetry.lock b/poetry.lock deleted file mode 100644 index 4a8f94fe..00000000 --- a/poetry.lock +++ /dev/null @@ -1,42 +0,0 @@ -# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand. - -[[package]] -name = "mpmath" -version = "1.3.0" -description = "Python library for arbitrary-precision floating-point arithmetic" -optional = false -python-versions = "*" -groups = ["main"] -files = [ - {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, - {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, -] - -[package.extras] -develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] -docs = ["sphinx"] -gmpy = ["gmpy2 (>=2.1.0a4)"] -tests = ["pytest (>=4.6)"] - -[[package]] -name = "sympy" -version = "1.13.3" -description = "Computer algebra system (CAS) in Python" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "sympy-1.13.3-py3-none-any.whl", hash = "sha256:54612cf55a62755ee71824ce692986f23c88ffa77207b30c1368eda4a7060f73"}, - {file = "sympy-1.13.3.tar.gz", hash = "sha256:b27fd2c6530e0ab39e275fc9b683895367e51d5da91baa8d3d64db2565fec4d9"}, -] - -[package.dependencies] -mpmath = ">=1.1.0,<1.4" - -[package.extras] -dev = ["hypothesis (>=6.70.0)", "pytest (>=7.1.0)"] - -[metadata] -lock-version = "2.1" -python-versions = ">=3.12" -content-hash = "bd48874a1c70818eccccc0c334cc40282d1e1c8bcc1d1369ccf3aa42489007d5" From 01da24978e0445f9f73366e24016a981a0c0e4c2 Mon Sep 17 00:00:00 2001 From: abdulhakeem Date: Sat, 25 Jan 2025 13:22:46 -0600 Subject: [PATCH 03/15] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d81bb85b..d44d081b 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ pip install -r requirements.txt 4. Install and run poetry to bring in other dependencies(`sympy` etc.) ``` conda install -c conda-forge poetry -poetry +poetry install ``` ### How to instantiate a task dataset? From 4a5938c40905720243fbf3dd4ec04c50d70c8908 Mon Sep 17 00:00:00 2001 From: abdulhakeem Date: Sat, 25 Jan 2025 17:56:00 -0600 Subject: [PATCH 04/15] Address feedbacks --- README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d44d081b..36c5afdb 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ We are building a python library of procedural dataset generators and algorithmi The goal is to generate virtually infinite data with adjustable complexity. -### Set up +### Set up for development 1. Clone the project ``` git clone https://github.com/open-thought/reasoning-gym.git @@ -14,14 +14,18 @@ git clone https://github.com/open-thought/reasoning-gym.git conda create --name reasoning_gym python=3.12 -y conda activate reasoning_gym ``` -3. Run the `requirements-dev.txt` file +3. Link project and install dependencies ``` -pip install -r requirements.txt +pip install -e . ``` -4. Install and run poetry to bring in other dependencies(`sympy` etc.) +4. Install development dependencies ``` -conda install -c conda-forge poetry -poetry install +pip install -r requirements-dev.txt +``` + +>NOTE: To consume the APIs in reasoning_gym, just install from pip using the following +``` +pip install reasoning-gym ``` ### How to instantiate a task dataset? From 3fed58378dce2baa0ad347a3be3903b09bb6312c Mon Sep 17 00:00:00 2001 From: abdulhakeem Date: Sat, 25 Jan 2025 18:06:31 -0600 Subject: [PATCH 05/15] Remove poetry from gitignore --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index e2ea009e..be4071bb 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,3 @@ ENV/ .coverage htmlcov/ .pytest_cache/ - -# Poetry lock file -poetry.lock From 65d5a5786b041d01f50b35a5e8d0255b3a0aa9ec Mon Sep 17 00:00:00 2001 From: "Andreas Koepf (aider)" Date: Sun, 26 Jan 2025 11:40:47 +0100 Subject: [PATCH 06/15] feat: Add SpellBackwardDataset with word reversal and length filtering --- README.md | 3 +- reasoning_gym/algorithmic/word_reversal.py | 41 ++++++++++++++++ tests/test_word_reversal.py | 54 ++++++++++++++++++++++ 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 36c5afdb..06b71bfb 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,8 @@ Available dataset names (which can be used with `create_dataset()`): - `LetterCountingDataset`: Count letter occurrences in text spans - `NumberFilteringDataset`: Filter numbers based on comparison with threshold - `NumberSortingDataset`: Sort lists of numbers in ascending or descending order -- `LetterJumbleDataset`: Unscramble words that have had their letters randomly jumbled +- `LetterJumbleDataset`: Unscramble words that have had their letters randomly jumbled +- `SpellBackwardDataset`: Spell individual words backward (e.g. "sun" -> "nus") - `WordReversalDataset`: Reverse word order in text spans #### Cognition Tasks diff --git a/reasoning_gym/algorithmic/word_reversal.py b/reasoning_gym/algorithmic/word_reversal.py index b08b459d..25e77ac8 100644 --- a/reasoning_gym/algorithmic/word_reversal.py +++ b/reasoning_gym/algorithmic/word_reversal.py @@ -9,6 +9,19 @@ from ..data import read_data_file from ..factory import ProceduralDataset, register_dataset +@dataclass +class SpellBackwardConfig: + """Configuration for spelling words backward task generation""" + + min_word_len: int = 3 # Minimum word length + seed: Optional[int] = None + size: int = 500 # Virtual dataset size + + def validate(self) -> None: + """Validate configuration parameters""" + assert self.min_word_len > 0, "min_word_len must be positive" + + @dataclass class WordReversalConfig: """Configuration for word reversal task generation""" @@ -55,4 +68,32 @@ class WordReversalDataset(ProceduralDataset): } +class SpellBackwardDataset(ProceduralDataset): + """Generates tasks to spell words backward""" + + def __init__(self, config: SpellBackwardConfig): + super().__init__(config=config, seed=config.seed, size=config.size) + + # Load and preprocess text + text = read_data_file("in_the_year_2889.txt") + # Extract words and clean them to contain only alphanumeric characters + self.words = [word for word in re.findall(r"\b\w+\b", text) + if word.isalnum() and len(word) >= config.min_word_len] + + def __getitem__(self, idx: int) -> dict: + """Generate a single spell backward task""" + rng = Random(self.seed + idx) + + # Select random word + word = rng.choice(self.words) + answer = word[::-1] + + return { + "question": f"Spell this word backward (example: sun -> nus): {word}", + "answer": answer, + "metadata": {"word": word, "word_len": len(word)}, + } + + +register_dataset("spell_backward", SpellBackwardDataset, SpellBackwardConfig) register_dataset("word_reversal", WordReversalDataset, WordReversalConfig) diff --git a/tests/test_word_reversal.py b/tests/test_word_reversal.py index eec4bef1..1e3f0151 100644 --- a/tests/test_word_reversal.py +++ b/tests/test_word_reversal.py @@ -54,6 +54,60 @@ def test_word_reversal_dataset_items(): assert answer_words == list(reversed(question_words)) +def test_spell_backward_config_validation(): + """Test that invalid configs raise appropriate errors""" + with pytest.raises(AssertionError): + config = SpellBackwardConfig(min_word_len=0) + config.validate() + + +def test_spell_backward_dataset_deterministic(): + """Test that dataset generates same items with same seed""" + config = SpellBackwardConfig(seed=42, size=10) + dataset1 = SpellBackwardDataset(config) + dataset2 = SpellBackwardDataset(config) + + for i in range(len(dataset1)): + assert dataset1[i] == dataset2[i] + + +def test_spell_backward_dataset_items(): + """Test basic properties of generated items""" + config = SpellBackwardConfig(min_word_len=3, size=10, seed=42) + dataset = SpellBackwardDataset(config) + + for i in range(len(dataset)): + item = dataset[i] + # Check item structure + assert isinstance(item, dict) + assert "question" in item + assert "answer" in item + assert "metadata" in item + + # Check metadata + assert "word" in item["metadata"] + assert "word_len" in item["metadata"] + + # Verify word length constraint + word = item["metadata"]["word"] + assert len(word) >= config.min_word_len + + # Verify answer is correct + assert item["answer"] == word[::-1] + + +def test_spell_backward_dataset_iteration(): + """Test that iteration respects dataset size""" + config = SpellBackwardConfig(size=5, seed=42) + dataset = SpellBackwardDataset(config) + + items = list(dataset) + assert len(items) == config.size + + # Test multiple iterations yield same items + assert items == list(dataset) + + def test_word_reversal_dataset_iteration(): """Test that iteration respects dataset size""" config = WordReversalConfig(size=5, seed=42) From 64fe0a895e124cd22cfc1bbe5dff01994da67628 Mon Sep 17 00:00:00 2001 From: "Andreas Koepf (aider)" Date: Sun, 26 Jan 2025 11:41:07 +0100 Subject: [PATCH 07/15] fix: Import SpellBackward classes in word reversal test file --- tests/test_word_reversal.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_word_reversal.py b/tests/test_word_reversal.py index 1e3f0151..020b5e8a 100644 --- a/tests/test_word_reversal.py +++ b/tests/test_word_reversal.py @@ -2,7 +2,12 @@ import pytest -from reasoning_gym.algorithmic.word_reversal import WordReversalConfig, WordReversalDataset +from reasoning_gym.algorithmic.word_reversal import ( + WordReversalConfig, + WordReversalDataset, + SpellBackwardConfig, + SpellBackwardDataset, +) def test_word_reversal_config_validation(): From 4908c06cb85861eaca3f46cb7d43462534f1ad86 Mon Sep 17 00:00:00 2001 From: "Andreas Koepf (aider)" Date: Sun, 26 Jan 2025 11:44:27 +0100 Subject: [PATCH 08/15] refactor: Move SpellBackwardDataset to separate file --- reasoning_gym/algorithmic/word_reversal.py | 41 ---------------------- tests/test_word_reversal.py | 8 ++--- 2 files changed, 2 insertions(+), 47 deletions(-) diff --git a/reasoning_gym/algorithmic/word_reversal.py b/reasoning_gym/algorithmic/word_reversal.py index 25e77ac8..b08b459d 100644 --- a/reasoning_gym/algorithmic/word_reversal.py +++ b/reasoning_gym/algorithmic/word_reversal.py @@ -9,19 +9,6 @@ from ..data import read_data_file from ..factory import ProceduralDataset, register_dataset -@dataclass -class SpellBackwardConfig: - """Configuration for spelling words backward task generation""" - - min_word_len: int = 3 # Minimum word length - seed: Optional[int] = None - size: int = 500 # Virtual dataset size - - def validate(self) -> None: - """Validate configuration parameters""" - assert self.min_word_len > 0, "min_word_len must be positive" - - @dataclass class WordReversalConfig: """Configuration for word reversal task generation""" @@ -68,32 +55,4 @@ class WordReversalDataset(ProceduralDataset): } -class SpellBackwardDataset(ProceduralDataset): - """Generates tasks to spell words backward""" - - def __init__(self, config: SpellBackwardConfig): - super().__init__(config=config, seed=config.seed, size=config.size) - - # Load and preprocess text - text = read_data_file("in_the_year_2889.txt") - # Extract words and clean them to contain only alphanumeric characters - self.words = [word for word in re.findall(r"\b\w+\b", text) - if word.isalnum() and len(word) >= config.min_word_len] - - def __getitem__(self, idx: int) -> dict: - """Generate a single spell backward task""" - rng = Random(self.seed + idx) - - # Select random word - word = rng.choice(self.words) - answer = word[::-1] - - return { - "question": f"Spell this word backward (example: sun -> nus): {word}", - "answer": answer, - "metadata": {"word": word, "word_len": len(word)}, - } - - -register_dataset("spell_backward", SpellBackwardDataset, SpellBackwardConfig) register_dataset("word_reversal", WordReversalDataset, WordReversalConfig) diff --git a/tests/test_word_reversal.py b/tests/test_word_reversal.py index 020b5e8a..310f9cc1 100644 --- a/tests/test_word_reversal.py +++ b/tests/test_word_reversal.py @@ -2,12 +2,8 @@ import pytest -from reasoning_gym.algorithmic.word_reversal import ( - WordReversalConfig, - WordReversalDataset, - SpellBackwardConfig, - SpellBackwardDataset, -) +from reasoning_gym.algorithmic.spell_backward import SpellBackwardConfig, SpellBackwardDataset +from reasoning_gym.algorithmic.word_reversal import WordReversalConfig, WordReversalDataset def test_word_reversal_config_validation(): From 71a6800d27bbd8829dedced1de14d767bc2d3bf8 Mon Sep 17 00:00:00 2001 From: "Andreas Koepf (aider)" Date: Sun, 26 Jan 2025 11:46:07 +0100 Subject: [PATCH 09/15] feat: Add spell_backward.py module for word reversal task generation --- reasoning_gym/algorithmic/spell_backward.py | 52 +++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 reasoning_gym/algorithmic/spell_backward.py diff --git a/reasoning_gym/algorithmic/spell_backward.py b/reasoning_gym/algorithmic/spell_backward.py new file mode 100644 index 00000000..83dc0dc0 --- /dev/null +++ b/reasoning_gym/algorithmic/spell_backward.py @@ -0,0 +1,52 @@ +"""Spell backward task generator""" + +import re +from dataclasses import dataclass +from random import Random +from typing import Optional + +from ..data import read_data_file +from ..factory import ProceduralDataset, register_dataset + + +@dataclass +class SpellBackwardConfig: + """Configuration for spelling words backward task generation""" + + min_word_len: int = 3 # Minimum word length + seed: Optional[int] = None + size: int = 500 # Virtual dataset size + + def validate(self) -> None: + """Validate configuration parameters""" + assert self.min_word_len > 0, "min_word_len must be positive" + + +class SpellBackwardDataset(ProceduralDataset): + """Generates tasks to spell words backward""" + + def __init__(self, config: SpellBackwardConfig): + super().__init__(config=config, seed=config.seed, size=config.size) + + # Load and preprocess text + text = read_data_file("in_the_year_2889.txt") + # Extract words and clean them to contain only alphanumeric characters + self.words = [word for word in re.findall(r"\b\w+\b", text) + if word.isalnum() and len(word) >= config.min_word_len] + + def __getitem__(self, idx: int) -> dict: + """Generate a single spell backward task""" + rng = Random(self.seed + idx) + + # Select random word + word = rng.choice(self.words) + answer = word[::-1] + + return { + "question": f"Spell this word backward (example: sun -> nus): {word}", + "answer": answer, + "metadata": {"word": word, "word_len": len(word)}, + } + + +register_dataset("spell_backward", SpellBackwardDataset, SpellBackwardConfig) From 6811c2ba04c2bcc9d3e3d926369016888ded9998 Mon Sep 17 00:00:00 2001 From: "Andreas Koepf (aider)" Date: Sun, 26 Jan 2025 11:48:18 +0100 Subject: [PATCH 10/15] feat: Add SpellBackward imports and exports to algorithmic package --- reasoning_gym/algorithmic/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/reasoning_gym/algorithmic/__init__.py b/reasoning_gym/algorithmic/__init__.py index 78136d66..b9810fdc 100644 --- a/reasoning_gym/algorithmic/__init__.py +++ b/reasoning_gym/algorithmic/__init__.py @@ -12,9 +12,12 @@ from .letter_counting import LetterCountingConfig, LetterCountingDataset from .letter_jumble import LetterJumbleConfig, LetterJumbleDataset from .number_filtering import NumberFilteringConfig, NumberFilteringDataset from .number_sorting import NumberSortingConfig, NumberSortingDataset +from .spell_backward import SpellBackwardConfig, SpellBackwardDataset from .word_reversal import WordReversalConfig, WordReversalDataset __all__ = [ + "SpellBackwardConfig", + "SpellBackwardDataset", "BaseConversionConfig", "BaseConversionDataset", "CaesarCipherConfig", From 5e57848fa2f1ae48084895dbda5419787ee82ad1 Mon Sep 17 00:00:00 2001 From: "Andreas Koepf (aider)" Date: Sun, 26 Jan 2025 11:52:15 +0100 Subject: [PATCH 11/15] refactor: Rename WordReversalDataset to WordSequenceReversalDataset --- README.md | 2 +- reasoning_gym/algorithmic/__init__.py | 6 ++-- reasoning_gym/algorithmic/word_reversal.py | 10 +++---- tests/test_word_reversal.py | 32 +++++++++++----------- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 06b71bfb..60952e1f 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ Available dataset names (which can be used with `create_dataset()`): - `NumberSortingDataset`: Sort lists of numbers in ascending or descending order - `LetterJumbleDataset`: Unscramble words that have had their letters randomly jumbled - `SpellBackwardDataset`: Spell individual words backward (e.g. "sun" -> "nus") -- `WordReversalDataset`: Reverse word order in text spans +- `WordSequenceReversalDataset`: Reverse word order in text spans #### Cognition Tasks diff --git a/reasoning_gym/algorithmic/__init__.py b/reasoning_gym/algorithmic/__init__.py index b9810fdc..7026cc7d 100644 --- a/reasoning_gym/algorithmic/__init__.py +++ b/reasoning_gym/algorithmic/__init__.py @@ -13,7 +13,7 @@ from .letter_jumble import LetterJumbleConfig, LetterJumbleDataset from .number_filtering import NumberFilteringConfig, NumberFilteringDataset from .number_sorting import NumberSortingConfig, NumberSortingDataset from .spell_backward import SpellBackwardConfig, SpellBackwardDataset -from .word_reversal import WordReversalConfig, WordReversalDataset +from .word_reversal import WordSequenceReversalConfig, WordSequenceReversalDataset __all__ = [ "SpellBackwardConfig", @@ -30,6 +30,6 @@ __all__ = [ "NumberFilteringDataset", "NumberSortingConfig", "NumberSortingDataset", - "WordReversalConfig", - "WordReversalDataset", + "WordSequenceReversalConfig", + "WordSequenceReversalDataset", ] diff --git a/reasoning_gym/algorithmic/word_reversal.py b/reasoning_gym/algorithmic/word_reversal.py index b08b459d..ce9f273d 100644 --- a/reasoning_gym/algorithmic/word_reversal.py +++ b/reasoning_gym/algorithmic/word_reversal.py @@ -10,8 +10,8 @@ from ..factory import ProceduralDataset, register_dataset @dataclass -class WordReversalConfig: - """Configuration for word reversal task generation""" +class WordSequenceReversalConfig: + """Configuration for word sequence reversal task generation""" min_words: int = 3 # Minimum words in list max_words: int = 8 # Maximum words in list @@ -24,8 +24,8 @@ class WordReversalConfig: assert self.max_words >= self.min_words, "max_words must be >= min_words" -class WordReversalDataset(ProceduralDataset): - """Generates word reversal tasks from text spans""" +class WordSequenceReversalDataset(ProceduralDataset): + """Generates word sequence reversal tasks from text spans""" def __init__(self, config: WordReversalConfig): super().__init__(config=config, seed=config.seed, size=config.size) @@ -55,4 +55,4 @@ class WordReversalDataset(ProceduralDataset): } -register_dataset("word_reversal", WordReversalDataset, WordReversalConfig) +register_dataset("word_sequence_reversal", WordSequenceReversalDataset, WordSequenceReversalConfig) diff --git a/tests/test_word_reversal.py b/tests/test_word_reversal.py index 310f9cc1..bcf042ce 100644 --- a/tests/test_word_reversal.py +++ b/tests/test_word_reversal.py @@ -3,13 +3,13 @@ import pytest from reasoning_gym.algorithmic.spell_backward import SpellBackwardConfig, SpellBackwardDataset -from reasoning_gym.algorithmic.word_reversal import WordReversalConfig, WordReversalDataset +from reasoning_gym.algorithmic.word_reversal import WordSequenceReversalConfig, WordSequenceReversalDataset -def test_word_reversal_config_validation(): +def test_word_sequence_reversal_config_validation(): """Test that invalid configs raise appropriate errors""" with pytest.raises(AssertionError): - config = WordReversalConfig(min_words=0) + config = WordSequenceReversalConfig(min_words=0) config.validate() with pytest.raises(AssertionError): @@ -17,20 +17,20 @@ def test_word_reversal_config_validation(): config.validate() -def test_word_reversal_dataset_deterministic(): +def test_word_sequence_reversal_dataset_deterministic(): """Test that dataset generates same items with same seed""" - config = WordReversalConfig(seed=42, size=10) - dataset1 = WordReversalDataset(config) - dataset2 = WordReversalDataset(config) + config = WordSequenceReversalConfig(seed=42, size=10) + dataset1 = WordSequenceReversalDataset(config) + dataset2 = WordSequenceReversalDataset(config) for i in range(len(dataset1)): assert dataset1[i] == dataset2[i] -def test_word_reversal_dataset_items(): +def test_word_sequence_reversal_dataset_items(): """Test basic properties of generated items""" - config = WordReversalConfig(min_words=3, max_words=6, size=10, seed=42) - dataset = WordReversalDataset(config) + config = WordSequenceReversalConfig(min_words=3, max_words=6, size=10, seed=42) + dataset = WordSequenceReversalDataset(config) for i in range(len(dataset)): item = dataset[i] @@ -109,10 +109,10 @@ def test_spell_backward_dataset_iteration(): assert items == list(dataset) -def test_word_reversal_dataset_iteration(): +def test_word_sequence_reversal_dataset_iteration(): """Test that iteration respects dataset size""" - config = WordReversalConfig(size=5, seed=42) - dataset = WordReversalDataset(config) + config = WordSequenceReversalConfig(size=5, seed=42) + dataset = WordSequenceReversalDataset(config) items = list(dataset) assert len(items) == config.size @@ -121,10 +121,10 @@ def test_word_reversal_dataset_iteration(): assert items == list(dataset) -def test_word_reversal_text_preprocessing(): +def test_word_sequence_reversal_text_preprocessing(): """Test that text preprocessing handles edge cases""" - config = WordReversalConfig(size=1, seed=42) - dataset = WordReversalDataset(config) + config = WordSequenceReversalConfig(size=1, seed=42) + dataset = WordSequenceReversalDataset(config) # Verify words were extracted from text assert len(dataset.words) > 0 From ae7948360968637a0f93d24439f332f43de1e92e Mon Sep 17 00:00:00 2001 From: "Andreas Koepf (aider)" Date: Sun, 26 Jan 2025 11:52:25 +0100 Subject: [PATCH 12/15] fix: Correct WordReversalConfig references to WordSequenceReversalConfig --- reasoning_gym/algorithmic/word_reversal.py | 2 +- tests/test_word_reversal.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/reasoning_gym/algorithmic/word_reversal.py b/reasoning_gym/algorithmic/word_reversal.py index ce9f273d..5ad94ea7 100644 --- a/reasoning_gym/algorithmic/word_reversal.py +++ b/reasoning_gym/algorithmic/word_reversal.py @@ -27,7 +27,7 @@ class WordSequenceReversalConfig: class WordSequenceReversalDataset(ProceduralDataset): """Generates word sequence reversal tasks from text spans""" - def __init__(self, config: WordReversalConfig): + def __init__(self, config: WordSequenceReversalConfig): super().__init__(config=config, seed=config.seed, size=config.size) # Load and preprocess text diff --git a/tests/test_word_reversal.py b/tests/test_word_reversal.py index bcf042ce..15680223 100644 --- a/tests/test_word_reversal.py +++ b/tests/test_word_reversal.py @@ -13,7 +13,7 @@ def test_word_sequence_reversal_config_validation(): config.validate() with pytest.raises(AssertionError): - config = WordReversalConfig(min_words=10, max_words=5) + config = WordSequenceReversalConfig(min_words=10, max_words=5) config.validate() From cf864d523a2d5d1f6bbbc3e2067624364302c6a0 Mon Sep 17 00:00:00 2001 From: "Andreas Koepf (aider)" Date: Sun, 26 Jan 2025 11:53:48 +0100 Subject: [PATCH 13/15] refactor: Update import for word sequence reversal module --- reasoning_gym/algorithmic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reasoning_gym/algorithmic/__init__.py b/reasoning_gym/algorithmic/__init__.py index 7026cc7d..e5386365 100644 --- a/reasoning_gym/algorithmic/__init__.py +++ b/reasoning_gym/algorithmic/__init__.py @@ -13,7 +13,7 @@ from .letter_jumble import LetterJumbleConfig, LetterJumbleDataset from .number_filtering import NumberFilteringConfig, NumberFilteringDataset from .number_sorting import NumberSortingConfig, NumberSortingDataset from .spell_backward import SpellBackwardConfig, SpellBackwardDataset -from .word_reversal import WordSequenceReversalConfig, WordSequenceReversalDataset +from .word_sequence_reversal import WordSequenceReversalConfig, WordSequenceReversalDataset __all__ = [ "SpellBackwardConfig", From 7977895275524905a7c6e4123bb73110905749e0 Mon Sep 17 00:00:00 2001 From: Andreas Koepf Date: Sun, 26 Jan 2025 11:54:54 +0100 Subject: [PATCH 14/15] rename word_reversal.py -> word_sequence_reversal.py --- README.md | 2 +- reasoning_gym/algorithmic/spell_backward.py | 11 ++-- ..._reversal.py => word_sequence_reversal.py} | 0 tests/test_spell_backward.py | 59 +++++++++++++++++++ ...rsal.py => test_word_sequence_reversal.py} | 59 +------------------ 5 files changed, 67 insertions(+), 64 deletions(-) rename reasoning_gym/algorithmic/{word_reversal.py => word_sequence_reversal.py} (100%) create mode 100644 tests/test_spell_backward.py rename tests/{test_word_reversal.py => test_word_sequence_reversal.py} (57%) diff --git a/README.md b/README.md index 60952e1f..db866593 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ Available dataset names (which can be used with `create_dataset()`): - `LetterCountingDataset`: Count letter occurrences in text spans - `NumberFilteringDataset`: Filter numbers based on comparison with threshold - `NumberSortingDataset`: Sort lists of numbers in ascending or descending order -- `LetterJumbleDataset`: Unscramble words that have had their letters randomly jumbled +- `LetterJumbleDataset`: Unscramble words that have had their letters randomly jumbled - `SpellBackwardDataset`: Spell individual words backward (e.g. "sun" -> "nus") - `WordSequenceReversalDataset`: Reverse word order in text spans diff --git a/reasoning_gym/algorithmic/spell_backward.py b/reasoning_gym/algorithmic/spell_backward.py index 83dc0dc0..59b163ee 100644 --- a/reasoning_gym/algorithmic/spell_backward.py +++ b/reasoning_gym/algorithmic/spell_backward.py @@ -9,12 +9,12 @@ from ..data import read_data_file from ..factory import ProceduralDataset, register_dataset -@dataclass +@dataclass class SpellBackwardConfig: """Configuration for spelling words backward task generation""" - + min_word_len: int = 3 # Minimum word length - seed: Optional[int] = None + seed: Optional[int] = None size: int = 500 # Virtual dataset size def validate(self) -> None: @@ -31,8 +31,9 @@ class SpellBackwardDataset(ProceduralDataset): # Load and preprocess text text = read_data_file("in_the_year_2889.txt") # Extract words and clean them to contain only alphanumeric characters - self.words = [word for word in re.findall(r"\b\w+\b", text) - if word.isalnum() and len(word) >= config.min_word_len] + self.words = [ + word for word in re.findall(r"\b\w+\b", text) if word.isalnum() and len(word) >= config.min_word_len + ] def __getitem__(self, idx: int) -> dict: """Generate a single spell backward task""" diff --git a/reasoning_gym/algorithmic/word_reversal.py b/reasoning_gym/algorithmic/word_sequence_reversal.py similarity index 100% rename from reasoning_gym/algorithmic/word_reversal.py rename to reasoning_gym/algorithmic/word_sequence_reversal.py diff --git a/tests/test_spell_backward.py b/tests/test_spell_backward.py new file mode 100644 index 00000000..2db86c62 --- /dev/null +++ b/tests/test_spell_backward.py @@ -0,0 +1,59 @@ +"""Tests for spell backward task generation""" + +import pytest + +from reasoning_gym.algorithmic.spell_backward import SpellBackwardConfig, SpellBackwardDataset + + +def test_spell_backward_config_validation(): + """Test that invalid configs raise appropriate errors""" + with pytest.raises(AssertionError): + config = SpellBackwardConfig(min_word_len=0) + config.validate() + + +def test_spell_backward_dataset_deterministic(): + """Test that dataset generates same items with same seed""" + config = SpellBackwardConfig(seed=42, size=10) + dataset1 = SpellBackwardDataset(config) + dataset2 = SpellBackwardDataset(config) + + for i in range(len(dataset1)): + assert dataset1[i] == dataset2[i] + + +def test_spell_backward_dataset_items(): + """Test basic properties of generated items""" + config = SpellBackwardConfig(min_word_len=3, size=10, seed=42) + dataset = SpellBackwardDataset(config) + + for i in range(len(dataset)): + item = dataset[i] + # Check item structure + assert isinstance(item, dict) + assert "question" in item + assert "answer" in item + assert "metadata" in item + + # Check metadata + assert "word" in item["metadata"] + assert "word_len" in item["metadata"] + + # Verify word length constraint + word = item["metadata"]["word"] + assert len(word) >= config.min_word_len + + # Verify answer is correct + assert item["answer"] == word[::-1] + + +def test_spell_backward_dataset_iteration(): + """Test that iteration respects dataset size""" + config = SpellBackwardConfig(size=5, seed=42) + dataset = SpellBackwardDataset(config) + + items = list(dataset) + assert len(items) == config.size + + # Test multiple iterations yield same items + assert items == list(dataset) diff --git a/tests/test_word_reversal.py b/tests/test_word_sequence_reversal.py similarity index 57% rename from tests/test_word_reversal.py rename to tests/test_word_sequence_reversal.py index 15680223..a117bfba 100644 --- a/tests/test_word_reversal.py +++ b/tests/test_word_sequence_reversal.py @@ -1,9 +1,6 @@ -"""Tests for word reversal task generation""" - import pytest -from reasoning_gym.algorithmic.spell_backward import SpellBackwardConfig, SpellBackwardDataset -from reasoning_gym.algorithmic.word_reversal import WordSequenceReversalConfig, WordSequenceReversalDataset +from reasoning_gym.algorithmic.word_sequence_reversal import WordSequenceReversalConfig, WordSequenceReversalDataset def test_word_sequence_reversal_config_validation(): @@ -55,60 +52,6 @@ def test_word_sequence_reversal_dataset_items(): assert answer_words == list(reversed(question_words)) -def test_spell_backward_config_validation(): - """Test that invalid configs raise appropriate errors""" - with pytest.raises(AssertionError): - config = SpellBackwardConfig(min_word_len=0) - config.validate() - - -def test_spell_backward_dataset_deterministic(): - """Test that dataset generates same items with same seed""" - config = SpellBackwardConfig(seed=42, size=10) - dataset1 = SpellBackwardDataset(config) - dataset2 = SpellBackwardDataset(config) - - for i in range(len(dataset1)): - assert dataset1[i] == dataset2[i] - - -def test_spell_backward_dataset_items(): - """Test basic properties of generated items""" - config = SpellBackwardConfig(min_word_len=3, size=10, seed=42) - dataset = SpellBackwardDataset(config) - - for i in range(len(dataset)): - item = dataset[i] - # Check item structure - assert isinstance(item, dict) - assert "question" in item - assert "answer" in item - assert "metadata" in item - - # Check metadata - assert "word" in item["metadata"] - assert "word_len" in item["metadata"] - - # Verify word length constraint - word = item["metadata"]["word"] - assert len(word) >= config.min_word_len - - # Verify answer is correct - assert item["answer"] == word[::-1] - - -def test_spell_backward_dataset_iteration(): - """Test that iteration respects dataset size""" - config = SpellBackwardConfig(size=5, seed=42) - dataset = SpellBackwardDataset(config) - - items = list(dataset) - assert len(items) == config.size - - # Test multiple iterations yield same items - assert items == list(dataset) - - def test_word_sequence_reversal_dataset_iteration(): """Test that iteration respects dataset size""" config = WordSequenceReversalConfig(size=5, seed=42) From 4fdf80e056b532813b020ddb8c88357a4e736f92 Mon Sep 17 00:00:00 2001 From: Andreas Koepf Date: Sun, 26 Jan 2025 12:01:48 +0100 Subject: [PATCH 15/15] update registered dataset name list in README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index db866593..dd9a5df1 100644 --- a/README.md +++ b/README.md @@ -59,9 +59,11 @@ Available dataset names (which can be used with `create_dataset()`): 'base_conversion', 'caesar_cipher', 'letter_counting', +'letter_jumble', 'number_filtering', 'number_sorting', -'word_reversal', +'spell_backward', +'word_sequence_reversal', 'basic_arithmetic', 'chain_sum', 'fraction_simplification', @@ -77,7 +79,7 @@ Available dataset names (which can be used with `create_dataset()`): 'sudoku', 'family_relationships', 'propositional_logic', -'syllogism' +'syllogism', ``` ### Task Overview