diff --git a/notebooks/codeio.ipynb b/notebooks/codeio.ipynb index 623fff2a..4d24110e 100644 --- a/notebooks/codeio.ipynb +++ b/notebooks/codeio.ipynb @@ -573,7 +573,7 @@ "WARMUP_GENERATIONS = 5\n", "TOTAL_GENERATIONS = 1_000\n", "TIMEOUT_CODE_INIT = 10\n", - "TIMEOUT_PER_SAMPLE = 2\n", + "TIMEOUT_PER_SAMPLE = 0.01\n", "\n", "errors = defaultdict(int)\n", "total_entries = sum(1 for _ in open(\"data/codeio-pyedu-with-input-generator.jsonl\", \"r\"))\n", @@ -584,7 +584,7 @@ " iterator = tqdm(enumerate(f_in), total=total_entries)\n", "\n", " for i, line in iterator:\n", - " iterator.set_description(f\"Processing {i}/{total_entries} | \" + \" | \".join(f\"{k}: {v}\" for k, v in errors.items()) + f\" | total: {sum(errors.values())}\")\n", + " iterator.set_description(f\"Failures:\" + \" | \".join(f\"{k}: {v}\" for k, v in errors.items()) + f\" | total: {sum(errors.values())}\")\n", " entry = json.loads(line)\n", "\n", " if not \"input_generator\" in entry:\n", @@ -607,7 +607,7 @@ " try:\n", " execution = sandbox.run_code(\n", " code=SAMPLING_TEMPLATE.format(num_generations=WARMUP_GENERATIONS),\n", - " timeout=TIMEOUT_PER_SAMPLE * WARMUP_GENERATIONS\n", + " timeout=TIMEOUT_CODE_INIT,\n", " )\n", " assert not execution.error, \"Error in input generator (warmup)\"\n", " assert execution.text, \"Empty input generator output (warmup)\"\n", @@ -620,7 +620,7 @@ " try:\n", " execution = sandbox.run_code(\n", " code=SAMPLING_TEMPLATE.format(num_generations=TOTAL_GENERATIONS),\n", - " timeout=TIMEOUT_PER_SAMPLE * TOTAL_GENERATIONS\n", + " timeout=int(TIMEOUT_PER_SAMPLE * TOTAL_GENERATIONS),\n", " )\n", " assert not execution.error, \"Error in input generator (full)\"\n", " assert execution.text, \"Empty input generator output (full)\"\n",