Add curriculum to ab dataset (#345)

* Add curriculum to ab dataset

* Add difficulty to metadata
This commit is contained in:
Adefioye 2025-03-13 15:03:02 -05:00 committed by GitHub
parent 4f45c8d655
commit 454250a4ea
3 changed files with 66 additions and 4 deletions

View file

@ -2,6 +2,7 @@ from dataclasses import dataclass
from random import Random
from typing import Any, Optional
from ..coaching import AttributeType, BaseCurriculum, ScalarAttributeDefinition
from ..factory import ProceduralDataset, register_dataset
@ -114,7 +115,11 @@ Return the final state of the program.
return {
"question": prompt,
"answer": " ".join(steps[-1]),
"metadata": {},
"metadata": {
"difficulty": {
"length": self.config.length,
}
},
}
def score_answer(self, answer: Optional[str], entry: dict[str, Any]) -> float:
@ -135,5 +140,25 @@ Return the final state of the program.
return 0.0
class ABCurriculum(BaseCurriculum):
"""Curriculum for A::B dataset"""
def __init__(self):
super().__init__(ABCurriculum.__name__, ABConfig)
# Define attributes
self._define_attributes(
ScalarAttributeDefinition(
name="length",
field_name="length",
levels=[1, 10, 50, 100],
default_level=0,
description="Length of the A::B program",
attr_type=AttributeType.STATIC,
min_value=1,
)
)
# Register the dataset
register_dataset("ab", ABDataset, ABConfig)
register_dataset("ab", ABDataset, ABConfig, ABCurriculum)