open-thought/reasoning-gym
1
0
Fork 0
mirror of https://github.com/open-thought/reasoning-gym.git synced 2026-04-19 12:58:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Ensure only words are considered

Browse source
This commit is contained in:
abdulhakeem 2025-01-25 23:08:41 -06:00
parent 8debeef913
commit 9d025b43fa
2 changed files with 3 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

3
reasoning_gym/algorithmic/sentence_reordering.py
View file

@ -29,10 +29,11 @@ class SentenceReorderingDataset(ProceduralDataset):
# Load and preprocess text
text = read_data_file("in_the_year_2889.txt")
# Extract sentences make sure they are greater than or equal to the number of words in a sentence
# Ensure that only the length of alpganumeric characters in the sentence is considered
self.sentences = [
sentence
for sentence in re.findall(r"[^.!?]+", text)
if len(sentence.split()) >= self.config.num_of_words_in_sentence
if len(re.findall(r"\b\w+\b", sentence)) >= self.config.num_of_words_in_sentence
]
def _generate_sentence_dataset(self, sentence: str, seed: int, idx: int, shuffle=True):