open-thought/reasoning-gym
1
0
Fork 0
mirror of https://github.com/open-thought/reasoning-gym.git synced 2026-04-19 12:58:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve prompt for better LLM adherence

Browse source
This commit is contained in:
Oliver 2025-02-21 23:00:10 +00:00
parent fc2c43b7d3
commit 3297fc1bc0
1 changed files with 55 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

88
notebooks/codeio/PreprocessCode.ipynb
View file

@ -31,9 +31,16 @@
"raw_files = list(Path(\"raw_files/\").iterdir())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Note that the below prompt is built for DeepSeekV3. It may not work with other LLMs."
]
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
@ -91,13 +98,13 @@
"\n",
"{0}\n",
"\n",
"Output the components (reference code, query, description, input generator) in order. Separate each component with a triple newline (\\n\\n\\n). Avoid code blocks and do not output any Markdown formatting. Respond only with the four components, no prefix or additional text.\n",
"Output the components (reference code, query, description, input generator) in order. Separate each component with a line of dashes (---). Avoid code blocks and do not output any Markdown formatting. Respond only with the four components, no prefix or additional text.\n",
"\"\"\""
]
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@ -139,58 +146,73 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"def main(word1, word2):\n",
" m = len(word1)\n",
" n = len(word2)\n",
" dp = [[0 for _ in range(n + 1)] for _ in range(m + 1)]\n",
"raw_files/bitmask.py\n",
"def main(task_performed, total_tasks):\n",
" dp = [[-1 for _ in range(total_tasks + 1)] for _ in range(2 ** len(task_performed))]\n",
" task = defaultdict(list)\n",
" final_mask = (1 << len(task_performed)) - 1\n",
"\n",
" for i in range(m + 1):\n",
" for j in range(n + 1):\n",
" if i == 0:\n",
" dp[i][j] = j\n",
" elif j == 0:\n",
" dp[i][j] = i\n",
" elif word1[i - 1] == word2[j - 1]:\n",
" dp[i][j] = dp[i - 1][j - 1]\n",
" else:\n",
" insert = dp[i][j - 1]\n",
" delete = dp[i - 1][j]\n",
" replace = dp[i - 1][j - 1]\n",
" dp[i][j] = 1 + min(insert, delete, replace)\n",
" \n",
" return {\"min_edit_distance\": dp[m][n]}\n",
" def count_ways_until(mask, task_no):\n",
" if mask == final_mask:\n",
" return 1\n",
" if task_no > total_tasks:\n",
" return 0\n",
" if dp[mask][task_no] != -1:\n",
" return dp[mask][task_no]\n",
"\n",
" total_ways_util = count_ways_until(mask, task_no + 1)\n",
" for p in task[task_no]:\n",
" if mask & (1 << p):\n",
" continue\n",
" total_ways_util += count_ways_until(mask | (1 << p), task_no + 1)\n",
" \n",
" dp[mask][task_no] = total_ways_util\n",
" return dp[mask][task_no]\n",
"\n",
"You are given two strings, `word1` and `word2`. Your task is to compute the minimum number of operations required to transform `word1` into `word2`. The permitted operations are insertion, deletion, and substitution of a single character. Return the minimum number of operations needed to achieve the transformation.\n",
" for i in range(len(task_performed)):\n",
" for j in task_performed[i]:\n",
" task[j].append(i)\n",
"\n",
" total_ways = count_ways_until(0, 1)\n",
" return {\"total_ways\": total_ways}\n",
"\n",
"---\n",
"\n",
"You are given a list `task_performed` and an integer `total_tasks`. `task_performed` represents the tasks that can be performed by each person, where each sublist corresponds to the tasks a person can do. `total_tasks` is the total number of tasks (N). Your task is to calculate the total number of ways to distribute the tasks among the persons. Each person can do only one task, and a task can be done by only one person. Return the total number of ways.\n",
"\n",
"---\n",
"\n",
"Input:\n",
" word1 (str): The first string to be transformed.\n",
" word2 (str): The second string to be matched.\n",
" task_performed (list of list of int): List of tasks that each person can perform. Each sublist contains the tasks a person can do.\n",
" total_tasks (int): The total number of tasks.\n",
"\n",
"Output:\n",
" return (dict): A dictionary with one key:\n",
" - min_edit_distance (int): The minimum number of operations required to transform `word1` into `word2`.\n",
" - total_ways (int): The total number of ways to distribute the tasks.\n",
"\n",
"---\n",
"\n",
"def input_generator():\n",
" import random\n",
" import string\n",
" word1 = ''.join(random.choice(string.ascii_lowercase) for _ in range(random.randint(5, 15)))\n",
" word2 = ''.join(random.choice(string.ascii_lowercase) for _ in range(random.randint(5, 15)))\n",
" return {\"word1\": word1, \"word2\": word2}\n"
" M = random.randint(2, 5)\n",
" N = random.randint(2, 5)\n",
" task_performed = [random.sample(range(1, N + 1), random.randint(1, N)) for _ in range(M)]\n",
" return {\"task_performed\": task_performed, \"total_tasks\": N}\n"
]
}
],
"source": [
"raw_file = random.choice(raw_files)\n",
"\n",
"print(raw_file)\n",
"\n",
"raw_code = raw_file.read_text()\n",
"\n",
"prompt = prompt_template.format(raw_code)\n",
@ -205,11 +227,11 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"code, query, parameters, generator = response.choices[0].message.content.split(\"\\n\\n\\n\")"
"code, query, parameters, generator = response.choices[0].message.content.split(\"\\n---\\n\")"
]
},
{