diff --git a/examples/generate_word_ladder_examples.py b/examples/generate_word_ladder_examples.py
deleted file mode 100644
index b5d147c8..00000000
--- a/examples/generate_word_ladder_examples.py
+++ /dev/null
@@ -1,220 +0,0 @@
-# generates dataset of word ladder examples, and then generates simulated chain of thought reasoning for each example
-
-import reasoning_gym
-from openai import OpenAI
-import os
-
-# Configuration for the dataset
-config = {
- 'dataset_name': 'word_ladder',
- 'dataset_config': {
- 'min_word_length': 5,
- 'max_word_length': 5,
- 'min_chain_length':3, # set to -1 for shortest possible path, increase to generate more examples
- 'max_chain_length':5,
- 'size': 1, # Generate a small dataset for demonstration
- }
-}
-
-system_prompt = """Word Ladder puzzles involve transforming a start word into an end word.
-You are allowed to change only one letter a time and you must keep the number of letters constant.
-Each time you change one letter the word in the chain must be forming one that's valid in English.
-Plurals are allowed, but not proper nouns.
-Given a start and an end word, generate a detailed step-by-step chain of thought reasoning of the transformation process.
-You will be given the word ladder question, as well as the correct solution path. So you don't need to solve the problem, you have the solution.
-Your task is to provide a perfectly simulated chain of thought reasoning exactly in the style of the example below,
-including considering multiple possibilities, validating words, and showing the final path.
-Use casual, verbose thinking with markdown formatting.
-Example chain of thought reasoning:
-{sample_CoT}
-"""
-
-# sample CoT for demonstration, extracted from real inference data from deepseek r1
-sample_CoT = """
-Word Ladder puzzles involve transforming a start word into an end word.
-You are allowed to change only one letter a time and you must keep the number of letters constant.
-Each time you change one letter the word in the chain must be forming one that's valid in English.
-Plurals are allowed, but not proper nouns.
-Given a start and an end word, generate a detailed step-by-step chain of thought reasoning of the transformation process.
-You will be given the word ladder question, as well as the correct solution path. So you don't need to solve the problem, you have the solution already, which should be your 'cheat sheet' to refer to as you go along.
-Your task is to provide a perfectly simulated chain of thought reasoning exactly in the style of the example below,
-including considering multiple possibilities, validating words, and showing the final path.
-Use casual, verbose thinking with markdown formatting.
-VERY IMPORTANT ***NEVER MENTION YOU HAVE BEEN PROVIDED WITH THE ANSWER, YOU NEED TO WRITE ONLY AS IF YOU'RE SOLVING THE QUESTION STEP BY STEP ***
-
-Example chain of thought reasoning:
-
-Okay, so I need to figure out how to transform the word "CARK" into "ZIPS" by changing one letter at a time, making sure each intermediate step is a valid English word. Let me start by breaking down the problem.
-
-First, both words are four letters long, so each step must maintain four letters. Each change can only alter one letter, either adding, removing, or replacing, but since the length must stay the same, it has to be a replacement each time. So, I need to find a path from CARK to ZIPS where each intermediate word is valid.
-
-Let me write down the starting and target words:
-
-CARK
-
-...
-
-ZIPS
-
-I need to see which letters need to change. Let's compare the letters:
-
-C -> Z
-
-A -> I
-
-R -> P
-
-K -> S
-
-So all four letters need to change. But since each step can only change one letter, I need to find a sequence where each single-letter change results in a valid word. Maybe I can approach this by changing one letter at a time, checking possible words each step.
-
-Starting with CARK. Let's look at possible changes for each letter:
-
-First letter C: changing to other letters to form a word. Possible options could be BARK (C to B), DARK (C to D), LARK (C to L), MARK (C to M), PARK (C to P), etc. Wait, but CARK is a valid word? Let me confirm. CARK means to worry or burden, so yes, it's valid.
-
-So, possible first steps from CARK:
-
-Change C to B: BARK
-
-Change C to D: DARK
-
-Change C to L: LARK
-
-Change C to M: MARK
-
-Change C to P: PARK
-
-Change C to W: WARK (is that a word? Wark might be a Scottish term for work, but I'm not sure if it's accepted in standard English. Maybe not.)
-
-Other possibilities: changing the A, R, or K. Let's see:
-
-Second letter A: changing to other vowels or consonants. For example, CORK (changing A to O), but CORK is valid. Or CIRK (changing A to I; cirk is a variant of circus, but I think "cirk" isn't standard. Maybe CIRC? No, that's not right. Wait, maybe CIRK is a valid word in some contexts, like a circular trench in fortification? Hmm, not sure. Better to stick with more common words.
-
-Third letter R: changing to another letter. For example, CAKE (changing R to K), but that would require changing both the third letter and the fourth letter? Wait, no: CARK to CAKE would be changing R to E and K to E? Wait, no. Wait, CARK is C-A-R-K. To get CAKE, you'd need to change R to K and K to E, which is two changes. So not possible in one step. Maybe changing R to another letter. For example, CASH (changing R to S and K to H? No, that's two letters again. Wait, no. Let's see: CARK. Changing the third letter R to another letter, like CASK (changing R to S). But that would be changing the third letter from R to S, resulting in CASK. Yes, that's a valid word. So CARK -> CASK.
-
-Fourth letter K: changing to another letter. For example, CARD (changing K to D), CARP (changing K to P), CARS (changing K to S), CART (changing K to T). All of these are valid.
-
-So possible first steps from CARK:
-
-- BARK, DARK, LARK, MARK, PARK, CASK, CARD, CARP, CARS, CART.
-
-Now, which of these would lead towards ZIPS? Let's think about the target word. ZIPS has a Z, which is the first letter. So maybe we need to get to a word starting with Z eventually. But changing from C to Z in one step would be a big jump. Alternatively, maybe we can change other letters first to reach intermediate steps that can approach Z.
-
-Alternatively, maybe work on changing the last letters first. Let's see. For example, changing K to S (CARS) might be useful since the target ends with S. Let's try that.
-
-CARK -> CARS (changing K to S). Now, from CARS, we need to change the third letter R to P and the first letter C to Z, and the second A to I. Let's see.
-
-From CARS, possible changes:
-
-Change C to Z: ZARS (is that a word? ZAR is a currency, plural ZARS? Maybe, but not sure. Alternatively, ZAPS (changing R to P and S to something? Wait, no. Let's check each letter.
-
-Alternatively, from CARS, change A to I: CIRS (is that a word? CIR is a three-letter word, but CIRS? Maybe not. Or CURS (changing A to U), which is valid. But how does that help?
-
-Alternatively, from CARS, change R to P: CAPS. Yes, that's valid. So CARS -> CAPS (changing R to P). Then from CAPS, we can change C to Z: ZAPS. Yes, ZAPS is a word. Then ZAPS -> ZIPS by changing A to I. So that would be a path:
-
-CARK -> CARS -> CAPS -> ZAPS -> ZIPS.
-
-Wait, let's check each step:
-
-1. CARK to CARS: change K to S. Valid.
-
-2. CARS to CAPS: change R to P. Valid.
-
-3. CAPS to ZAPS: change C to Z. Valid.
-
-4. ZAPS to ZIPS: change A to I. Valid.
-
-That's four steps. But wait, starting from CARK (step 0), then steps 1-4 would be four changes. But the problem says "transform CARK into ZIPS by changing one letter at a time", so the number of steps would be the number of letters changed. Since all four letters are different, at minimum it would take four steps. But perhaps the path is longer if there are no shorter paths.
-
-Wait, let me check if this works:
-
-CARK → CARS → CAPS → ZAPS → ZIPS.
-
-Yes, that's four steps. Each step changes one letter, and all intermediate words are valid. Let me verify each word:
-
-- CARK: valid.
-
-- CARS: valid.
-
-- CAPS: valid.
-
-- ZAPS: valid.
-
-- ZIPS: valid.
-
-Yes, that seems to work. But let me check if there's a shorter path or if this is correct.
-
-Alternatively, is there a three-step path? Probably not since four letters need to change. Each step can only change one letter, so at least four steps are needed. So the path above is four steps, which is the minimum. Let me confirm each step again.
-
-Another possible path: changing the first letter earlier. For example:
-
-CARK → PARK (change C to P). Then PARK → PARKS (but that's five letters; can't do that. Alternatively, changing K to S: PARS. Then PARS -> PAWS (changing R to W?), but not helpful. Or PARK → PARS (change K to S), then PARS → PAIS (change R to I?), but PAIS is valid. Then PAIS → ZIPS (change P to Z and A to I? No, that's two letters. Hmm, not helpful.
-
-Alternatively, CARK → DARK → DANK → DINK → ZINK → ZIPS? Let's see. CARK to DARK (C→D). DARK to DANK (R→N). DANK to DINK (A→I). DINK to ZINK (D→Z). ZINK is a variant spelling of zinc, but ZINK is a valid word. Then ZINK to ZIPS (K→S and N→P? Wait, no. ZINK is Z-I-N-K. To get to ZIPS, we need Z-I-P-S. So need to change N to P and K to S. That's two steps. So that path would be longer.
-
-Another path: CARK → CARD → ZARD? ZARD isn't a word. CARD → WARD → WARD → ... not helpful.
-
-Alternatively, CARK → CARK to CIRK (A→I). CIRK is a variant of circus, but maybe not standard. Then CIRK → ZIRK (C→Z). ZIRK is a surname or a brand, not sure if it's a valid word. Then ZIRK → ZIPS (R→P and K→S). But again, two letters at once. Not helpful.
-
-So the first path I thought of seems viable: CARK → CARS → CAPS → ZAPS → ZIPS.
-
-Another check: From CARK, can we go to CASK? CARK → CASK (R→S). Then CASK → ZASK (C→Z). ZASK? Not a word. Alternatively, CASK → CASK → CASH (K→H). Then CASH → ZASH (C→Z). Not a word. Not helpful.
-
-Alternatively, CARK → CARP (K→P). CARP is valid. Then CARP → CARP → ZARP (C→Z). ZARP? Not a word. Or CARP → HARp (C→H), but HARP is valid. Then HARP → HIPS (A→I and R→P). But that's two changes. Not allowed.
-
-So the initial path seems best. Let me confirm each step again:
-
-1. CARK → CARS: change K to S. Valid.
-
-2. CARS → CAPS: change R to P. Valid.
-
-3. CAPS → ZAPS: change C to Z. Valid.
-
-4. ZAPS → ZIPS: change A to I. Valid.
-
-Yes, that's four steps, each changing one letter, all words are valid. So that should be the correct sequence.
-
-"""
-
-client = OpenAI(
- base_url="https://openrouter.ai/api/v1", # openrouter used in this example
- api_key=os.environ["OPENROUTER_API_KEY"], # set your key in environment variable
-)
-
-
-def generate_cot(question: str, answer: str) -> str:
- """Generate chain of thought reasoning for word ladder"""
- prompt = f"""The question is {question}. The correct solution is {answer}.
- Provide the verbose chain of thought reasoning to transform the start word into the end word exactly in the style and length required."""
-
- completion = client.chat.completions.create(
- model="microsoft/phi-4", # choose model
- messages=[
- {
- "role": "system",
- "content": system_prompt
- },
- {
- "role": "user",
- "content": prompt
- }
- ],
- temperature=0.6,
- max_tokens=10000
- )
- return completion.choices[0].message.content
-# Create the word ladder dataset
-dataset = reasoning_gym.create_dataset(config['dataset_name'], **config['dataset_config'])
-print(f"Generated {len(dataset)} examples, moving on to generate CoT reasoning...")
-# Generate and print examples with CoT
-for item in dataset:
- # Generate CoT reasoning demo
-
- item['reasoning'] = generate_cot(item['question'],item['answer'])
-
- print("\n--- Example ---")
- print("Question:", item['question'])
- print("Answer:", item['answer'])
- print("\nChain of Thought:")
- print(item['reasoning'])
- print("\nMetadata:", item['metadata'])
\ No newline at end of file
diff --git a/examples/word_ladder/README.md b/examples/word_ladder/README.md
new file mode 100644
index 00000000..93d5a8b9
--- /dev/null
+++ b/examples/word_ladder/README.md
@@ -0,0 +1,132 @@
+# Word Ladder Puzzle Dataset Generator
+
+## Overview
+
+This project generates a dataset of word ladder puzzles and (optionally) submits chain-of-thought reasoning requests using Anthropic's Message Batches API. Each puzzle is stored as a JSON object with the following keys: `question`, `answer`, `metadata`, and `reasoning` (initially `null`).
+
+The project consists of several key components:
+
+- **`main.py`**:
+ Orchestrates the overall flow. It performs the following tasks:
+ 1. Generates a dataset of word ladder puzzles by calling functions from `utils/create_word_ladders.py`.
+ 2. (Optionally) Triggers the reasoning request process to augment puzzles with chain-of-thought reasoning via `utils/generate_reasoning.py`.
+ 3. (Planned) Additional steps such as checking results or uploading the final dataset.
+
+ The configuration for the dataset parameters (e.g., word length, chain length, and dataset size) is centralized here, making it easy to adjust the settings as needed.
+
+- **`utils/create_word_ladders.py`**:
+ Contains functions to create and validate a word ladder dataset. It leverages underlying modules (e.g., `reasoning_gym`) to generate individual puzzles and ensures uniqueness across the dataset.
+
+- **`utils/generate_reasoning.py`**:
+ Reads the generated dataset (in JSONL format), then filters out puzzles that already have reasoning. For puzzles missing chain-of-thought data, it splits them into batches (with a default batch size that you can adjust) and submits each batch to Anthropic's Message Batches API. Each API request includes the puzzle along with a custom system prompt (read from `system_prompt.txt`), and the resulting metadata is stored for later retrieval and analysis.
+
+- **`usage_stats.py`**:
+ Analyzes API response files to compute detailed usage statistics. This script:
+ - Extracts token usage metrics such as `input_tokens`, `cache_creation_input_tokens`, `cache_read_input_tokens`, and `output_tokens`.
+ - Calculates costs based on pricing data and shows the savings achieved through prompt caching.
+ - Forecasts costs for various quantities of jobs (e.g., 2,000, 4,000, 10,000, 20,000, and 50,000 puzzles) using the observed average token usage.
+ This is especially useful for monitoring your API spend and ensuring that your usage stays within budget.
+
+## Warning
+
+**Caution:**
+Running large batches of requests via the Anthropic API (especially in `generate_reasoning.py`) can incur significant costs in Anthropic credits. **Please review and understand your API quota and budgeting before running the API call.** If you are just testing or working with a demo dataset, ensure you adjust the batch size or dataset size appropriately to avoid unexpected charges.
+
+## Prerequisites
+
+- **Python Version:** Python 3.7+
+- **Dependencies:**
+ - `tqdm`
+ - `anthropic`
+ - `reasoning_gym`
+- **Environment Variables:**
+ For generating reasoning batches, set your Anthropic API key:
+ ```bash
+ export ANTROPIC_API_KEY=your_api_key_here
+ ```
+
+## Directory Structure
+
+```
+examples/word_ladder/
+├── main.py
+├── utils/
+│ ├── create_word_ladders.py
+│ ├── generate_reasoning.py
+│ └── system_prompt.txt
+├── usage_stats.py
+```
+
+
+## Configuration
+
+The dataset generation parameters are centralized in `main.py` under the `config` dictionary. You can adjust settings like:
+
+- **Word Length:**
+ - `min_word_length`
+ - `max_word_length`
+
+- **Chain Length:**
+ - `min_chain_length` (e.g., set to -1 for the shortest possible chain)
+ - `max_chain_length`
+
+- **Dataset Size:**
+ - `size` — the number of puzzles to generate (e.g., `1000` for a demo)
+
+## How to Run
+
+1. **Generate the Dataset**
+
+ Run the main script:
+ ```bash
+ python3 main.py
+ ```
+ This does the following:
+ - Generates a unique JSONL file containing the word ladder puzzles in the `output` folder.
+ - Calls functions from `utils/create_word_ladders.py` to create the puzzles.
+ - Optionally (if enabled), submits the puzzles for chain-of-thought reasoning via the API.
+
+2. **Submit Reasoning Batches (Optional)**
+
+ To generate chain-of-thought reasoning for puzzles:
+ - Verify that `ANTHROPIC_API_KEY` is set.
+ - Confirm that `system_prompt.txt` is present in the `/examples/word_ladder` folder and contains the desired system prompt.
+ - In `main.py`, uncomment the reasoning submission section to enable the API call, or run directly:
+ ```bash
+ python3 utils/generate_reasoning.py
+ ```
+
+ **Warning:** Be aware that submitting large batches can quickly incur high costs in Anthropic credits.
+
+3. **Compute Usage Statistics**
+
+ After running batches through the API, you can analyze the cost and token usage statistics with:
+ ```bash
+ python3 usage_stats.py path/to/msgbatch_results.jsonl
+ ```
+ This script provides detailed costing information, token usage per query, savings from caching, and forecasting for future job batches.
+
+## Output
+
+- All generated datasets and batch metadata files are stored in the `/examples/word_ladder/output` folder.
+- After submitting reasoning batches via Anthropic's API, you can monitor progress and download the batch results from the Anthropic web dashboard.
+- Use `usage_stats.py` to compute detailed statistics and forecast future costs based on your current usage and token pricing.
+
+## Troubleshooting
+
+- **File Paths:**
+ Verify that `system_prompt.txt` is in the `/examples/word_ladder` folder as expected. The modules use paths relative to their location.
+
+- **Environment Variables:**
+ Make sure your `ANTHROPIC_API_KEY` is set correctly when submitting API requests.
+
+- **Output Directory Permissions:**
+ Ensure the `output` directory exists and is writable by your user.
+
+- **Cost Monitoring:**
+ Check your Anthropic API usage and account balance before running large batches to avoid unexpected costs.
+
+## License
+
+This project is licensed under the MIT License.
+
diff --git a/examples/word_ladder/main.py b/examples/word_ladder/main.py
new file mode 100644
index 00000000..4e73528c
--- /dev/null
+++ b/examples/word_ladder/main.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""
+main.py – Orchestrates the overall flow:
+1. Generate word ladder sets
+2. Submit chain-of-thought reasoning requests in batches via the LLM
+3. Upload the final dataset to HuggingFace Hub (if needed)
+"""
+
+import uuid
+import sys
+from pathlib import Path
+from typing import Dict, Any
+
+from examples.word_ladder.utils import create_word_ladders, generate_reasoning
+
+
+def create_dataset(jsonl_path: Path, config: Dict[str, Any]) -> bool:
+ """
+ Creates the word ladder dataset, handling potential exhaustion gracefully.
+
+ Returns:
+ bool: True if dataset was created (even if truncated), False if creation failed
+ """
+ try:
+ print("Step 1: Algorithmically creating word ladder chains...")
+ create_word_ladders.create_word_ladder_dataset(str(jsonl_path), config=config)
+ return True
+
+ except IndexError as e:
+ # Dataset was exhausted but some examples were generated
+ print("\nNote: Dataset generation stopped early due to exhaustion of unique puzzles.")
+ print(f"Reason: {str(e)}")
+ if jsonl_path.exists():
+ print("Continuing with the partial dataset that was successfully generated.")
+ return True
+ return False
+
+ except Exception as e:
+ # Unexpected error during dataset creation
+ print(f"\nError: Failed to create dataset: {str(e)}")
+ return False
+
+def main():
+ # Centralized configuration for the dataset
+ config = {
+ 'dataset_name': 'word_ladder',
+ 'dataset_config': {
+ 'min_word_length': 3,
+ 'max_word_length': 5,
+ 'min_chain_length':-1, # set to -1 for the shortest possible path
+ 'max_chain_length':10,
+ 'size': 100, # Generate a small-ish dataset for demonstration
+ }
+ }
+
+ # Generate a friendly unique identifier and compose the file path
+ unique_id = uuid.uuid4().hex[:8]
+ output_dir = Path(__file__).resolve().parent / "output"
+ output_dir.mkdir(exist_ok=True) # Create output directory if it doesn't exist
+ jsonl_path = output_dir / f"word_ladders_{unique_id}.jsonl"
+
+ # Step 1: Create the dataset
+ if not create_dataset(jsonl_path, config):
+ print("Exiting due to dataset creation failure.")
+ sys.exit(1)
+
+
+ # Step 2: Generate reasoning
+ '''
+ try:
+ print("\nStep 2: Submitting reasoning batches for the dataset...")
+ generate_reasoning.submit_reasoning_batches(input_path=str(jsonl_path))
+ except Exception as e:
+ print(f"\nError: Failed to submit reasoning batches: {str(e)}")
+ sys.exit(1)
+ '''
+
+ # Step 3: Check Anthropic batch results
+ # Step 4: Upload to HuggingFace 🤗
+
+ print("\nComplete!")
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/examples/word_ladder/output/example_4e5540f0_batch_metadata.jsonl b/examples/word_ladder/output/example_4e5540f0_batch_metadata.jsonl
new file mode 100644
index 00000000..9b642a5a
--- /dev/null
+++ b/examples/word_ladder/output/example_4e5540f0_batch_metadata.jsonl
@@ -0,0 +1 @@
+{"batch_id": "msgbatch_01J7Dev6petQwq41gNpeGhUK", "api_response": {"id": "msgbatch_01J7Dev6petQwq41gNpeGhUK", "type": "message_batch", "processing_status": "in_progress", "request_counts": {"canceled": 0, "errored": 0, "expired": 0, "processing": 1000, "succeeded": 0}, "ended_at": null, "created_at": "2025-02-03T06:07:04.440375Z", "expires_at": "2025-02-04T06:07:04.440375Z", "cancel_initiated_at": null, "results_url": null}, "custom_ids": ["PAINT_LACKS_0", "CONKS_GRAYS_1", "WINGS_WORMY_2", "PINTO_RASPS_3", "GRACE_WAINS_4", "RACES_VAPES_5", "LAIRS_FUELS_6", "WINKS_TENTH_7", "VERBS_DATES_8", "SCENT_HEARS_9", "GRACE_CHIVE_10", "COOPS_SPARE_11", "ROOMY_PLUME_12", "PORKS_HOSTA_13", "DEARS_TABBY_14", "SNAIL_ADOBE_15", "SMALL_BLINK_16", "SOAPS_SPOTS_17", "HOMES_LOVEY_18", "SPURN_STEEP_19", "PLODS_SWIFT_20", "TACOS_SETTS_21", "BANKS_KICKY_22", "POUTS_CAKES_23", "RESTS_SWUNG_24", "PORKY_FISKS_25", "PUMAS_NAPES_26", "WANDS_PRICE_27", "DROLL_SORES_28", "JOWLS_PACTS_29", "ATONE_GOOFS_30", "BRIGS_SHUNT_31", "RAPID_DOXES_32", "CHESS_DOLED_33", "BUDGE_BOOBS_34", "ROPED_SAGAS_35", "TATTS_SPARE_36", "RALES_NATES_37", "RUFFS_GONGS_38", "VEGES_HUNTS_39", "SCULL_PAIRS_40", "DICKY_MOWED_41", "PANKO_ADDER_42", "TILLS_GAINS_43", "OASIS_DOXED_44", "BONED_MEALS_45", "LAMBS_TOADY_641", "PIKER_HEAVY_642", "BANKS_TRUGS_643", "VEEPS_DIVAS_644", "CHATS_CLEFS_645", "SNITS_PLAIN_646", "CHARY_SPITE_647", "EARLY_DUMMY_977", "MEDAL_COYER_978", "FEATS_MOOKS_979", "MALES_HEXES_980", "DOGGO_MUGGY_981", "GIRDS_SOAPS_982", "SLASH_TRYST_983", "LOOPS_FLOES_984", "SLATE_SMORE_985", "SINEW_SULKS_986", "SCORN_SHOUT_987", "MAMAS_VOIDS_988", "TWIGS_BLOKE_989", "KOOKY_SPINE_990", "FLUES_SMITE_991", "DRAWS_MAIMS_992", "SMITH_PLEAS_993", "BERGS_OPINE_994", "SNAKE_BLABS_995", "WADED_DRIBS_996", "YAPPY_PIPES_997", "STAGY_COONS_998", "PEAKY_MURAL_999"], "input_file": "word_ladders_029573f4.jsonl"}
diff --git a/examples/word_ladder/output/example_msgbatch_01J7Dev6petQwq41gNpeGhUK_results.jsonl b/examples/word_ladder/output/example_msgbatch_01J7Dev6petQwq41gNpeGhUK_results.jsonl
new file mode 100644
index 00000000..4fe8ab4f
--- /dev/null
+++ b/examples/word_ladder/output/example_msgbatch_01J7Dev6petQwq41gNpeGhUK_results.jsonl
@@ -0,0 +1,5 @@
+{"custom_id":"PAINT_LACKS_0","result":{"type":"succeeded","message":{"id":"msg_01Kif1LqT19GgPdSEQiuFjwu","type":"message","role":"assistant","model":"claude-3-5-sonnet-20241022","content":[{"type":"text","text":"\nOkay, let me work on transforming PAINT into LACKS by changing one letter at a time. First, let me analyze both words to understand what needs to change.\n\nPAINT → LACKS\n\nLooking at the letters:\nP needs to become L\nA stays A\nI needs to become C\nN needs to become K\nT needs to become S\n\nSo four letters need to change (all except A). Let me start exploring possible paths by changing one letter at a time.\n\nStarting with PAINT. What changes can I make? Let's see:\n- Change P to something: FAINT, SAINT, TAINT (all valid)\n- Change I to something: PANT (valid)\n- Change N to something: PAINT to PAIT (not valid)\n- Change T to something: PAIN (valid, but loses a letter... no, need to keep 5 letters)\nWait, what about PAINS (changing T to S)? Yes, PAINS is a valid word! \n\nSo first step: PAINT → PAINS (changing T to S)\n\nNow from PAINS, what can I do? Need to eventually get to LACKS. Let's see possible changes:\n- Change P to something: GAINS, MAINS, RAINS, WAINS\n- Change I to something: PANS (loses a letter, not allowed)\n- Change N to something: PAIRS (valid, but moves away from target LACKS)\n\nWAINS looks interesting (changing P to W). Is WAINS a valid word? Yes, it refers to wagons or carts historically. \n\nStep 2: PAINS → WAINS\n\nNow from WAINS, what's next? Looking at LACKS, we still need to get that C and K in there. Let's see:\n- Change A to something: WINS (loses letter)\n- Change I to something: WANS (loses letter)\n- Change N to something: WAITS (valid but moves away from target)\n- Change S to something: WAINT (not a word)\nWait, what about WARNS (changing I to R)? Yes! WARNS is definitely a valid word.\n\nStep 3: WAINS → WARNS\n\nNow from WARNS to LACKS. Still need L, C, and K. From WARNS:\n- Change W to something: BARNS, EARNS, YARNS\nBARNS looks promising! Because B is closer to L in the alphabet than W. \n\nStep 4: WARNS → BARNS\n\nFrom BARNS to LACKS. Let's see:\n- Change N to something: BARS (loses letter)\n- Change S to something: BARN (loses S we need)\n- Change R to K perhaps? BANKS? No, need to keep R for now.\nWait! What about BARKS (changing N to K)? Yes! That works!\n\nStep 5: BARNS → BARKS\n\nNow BARKS. Getting closer! We need to change R to C and B to L to get to LACKS. Let's try:\n- Change B to something: LARKS, MARKS, PARKS\n- Change R to C: BACKS! Yes, that works!\n\nStep 6: BARKS → BACKS\n\nFinally, from BACKS to LACKS, just need to change B to L:\n\nStep 7: BACKS → LACKS\n\nLet me verify the entire sequence:\nPAINT → PAINS (T→S)\nPAINS → WAINS (P→W)\nWAINS → WARNS (I→R)\nWARNS → BARNS (W→B)\nBARNS → BARKS (N→K)\nBARKS → BACKS (R→C)\nBACKS → LACKS (B→L)\n\nChecking each word:\nPAINT: valid\nPAINS: valid (plural of pain)\nWAINS: valid (old wagons)\nWARNS: valid (gives warning)\nBARNS: valid (farm buildings)\nBARKS: valid (tree covering or dog sound)\nBACKS: valid (plural of back)\nLACKS: valid (is without)\n\nEach step changes exactly one letter and creates a valid English word. Seven steps total, which makes sense since we needed to change four letters to get from PAINT to LACKS, and we had to make some strategic intermediate changes to get there. This seems like a valid solution!\n\nLet me double-check if there might be a shorter path... Could we have gone:\nPAINT → PANT (remove I? No, must keep 5 letters)\nPAINT → PARTS (change N to R and T to S? No, that's two changes at once)\nPAINT → PANIC (T to C, but moves away from target)\n\nNo, I think the path we found is optimal because we needed those intermediate steps to gradually transform the word while keeping valid English words at each step. The sequence PAINT→PAINS→WAINS→WARNS→BARNS→BARKS→BACKS→LACKS works perfectly!\n"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":63,"cache_creation_input_tokens":0,"cache_read_input_tokens":6434,"output_tokens":1149}}}}
+{"custom_id":"CONKS_GRAYS_1","result":{"type":"succeeded","message":{"id":"msg_01JrxHJyxFxsTVsn8huufv8T","type":"message","role":"assistant","model":"claude-3-5-sonnet-20241022","content":[{"type":"text","text":"\nAlright, let me try to transform CONKS into GRAYS by changing one letter at a time. First, let me write down both words to analyze what needs to change:\n\nCONKS\nGRAYS\n\nLooking at the letters that need to change:\nC needs to become G\nO needs to become R\nN needs to become A\nK needs to become Y\nS stays S\n\nHmm, four letters need to change, but S stays the same at the end. Let me start by looking at possible changes from CONKS.\n\nFirst letter options from CONKS:\n- Change C to B: BONKS (valid)\n- Change C to D: DONKS (valid)\n- Change C to G: GONKS (is this valid? Not sure)\n- Change C to H: HONKS (valid)\n\nSecond letter options:\n- Change O to A: CANKS (is this valid? Not sure)\n- Change O to I: CINKS (probably not valid)\n- Change O to O: stays same\n- Change O to U: CUNKS (not sure if valid)\n\nWait, maybe instead of changing the first letter right away to G, let's try working with the O. What if we change O to another letter that might help us get closer? \n\nLet me try: CONKS → COOKS (changing N to O)\nYes! COOKS is definitely a valid word. That feels like progress.\n\nFrom COOKS, what can we do?\n- Change first C to G? GOOKS (probably not appropriate)\n- Change second O to R? No, need to keep both O's for now\n- Change K to something? COOLS? No, moving away from target\n- Change S? No, need to keep S\n\nWhat about changing one of the O's to something? COOKS → COOPS (changing K to P)\nYes! COOPS is valid (like chicken coops). This feels promising.\n\nNow from COOPS:\n- Change C to G? GOOPS (not sure if valid)\n- Change first O to R? CROPS! Yes, that works!\n\nCROPS is a valid word. Now we're getting somewhere. Let's see what we can do from CROPS:\n- Change O to A? CRAPS! That's valid and gets us closer to GRAYS\n\nFrom CRAPS:\n- Change P to Y? CRAYS! Perfect!\n\nAnd finally, from CRAYS:\n- Change C to G: GRAYS! We've reached the target!\n\nLet me verify this path:\n1. CONKS → COOKS (N to O)\n2. COOKS → COOPS (K to P)\n3. COOPS → CROPS (O to R)\n4. CROPS → CRAPS (O to A)\n5. CRAPS → CRAYS (P to Y)\n6. CRAYS → GRAYS (C to G)\n\nLet me check each word again:\nCONKS - valid (to hit on head)\nCOOKS - valid\nCOOPS - valid (enclosures)\nCROPS - valid\nCRAPS - valid\nCRAYS - valid (types of fish)\nGRAYS - valid\n\nThis seems to work perfectly! Each step changes exactly one letter, and all words are valid English words. Let me double-check if there might be a shorter path.\n\nCould we go CONKS → CONES? No, that doesn't help get to GRAYS.\nWhat about CONKS → CRANKS? No, that's two letter changes.\nOr CONKS → CORKS? Again, doesn't seem to lead anywhere useful.\n\nThe path we found:\nCONKS → COOKS → COOPS → CROPS → CRAPS → CRAYS → GRAYS\nseems optimal because:\n1. Each step is a valid word\n2. Each change moves us closer to our target\n3. We maintain the final S throughout\n4. We gradually transform the word while keeping it valid at each step\n\nYes, I think this is the solution. Six steps, changing one letter at a time, all valid words, and each change gets us closer to our target GRAYS.\n"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":64,"cache_creation_input_tokens":0,"cache_read_input_tokens":6434,"output_tokens":956}}}}
+{"custom_id":"WINGS_WORMY_2","result":{"type":"succeeded","message":{"id":"msg_01KXJZXTZgQoTnLvMqHphEcS","type":"message","role":"assistant","model":"claude-3-5-sonnet-20241022","content":[{"type":"text","text":"\nOkay, let me tackle this word ladder from WINGS to WORMY. First, I'll write both words down and analyze what letters need to change:\n\nWINGS\nWORMY\n\nLooking at the letters:\nW stays the same\nI needs to become O\nN needs to become R\nG needs to become M\nS needs to become Y\n\nSo four letters need to change, keeping only the W. Let me start exploring possible paths by changing one letter at a time.\n\nStarting with WINGS, what can I change? Let's look at each position:\n- First letter (W): stays the same, so no need to change\n- Second letter (I): could become A, E, O, U\n- Third letter (N): could become other consonants\n- Fourth letter (G): could become other consonants\n- Fifth letter (S): could become other letters\n\nLet me try changing I first since it needs to become O eventually:\nWINGS → WONGS? Is that a word? Not sure. \n\nMaybe change N first:\nWINGS → WIGS? No, that removes a letter. Wait, I need to keep it 5 letters.\nWINGS → WINGS (same)\nWINGS → WINKS (changing G to K)! Yes, WINKS is definitely a valid word.\n\nOkay, so WINGS → WINKS works. Now from WINKS, what's next? I still need to:\n- Change I to O\n- Change K to R\n- Change S to Y\n\nLet me try changing I to O in WINKS:\nWINKS → WONKS\nIs WONKS a word? I think it is - it's the plural of wonk, someone who studies a subject extensively. Yes, that works!\n\nSo now I have:\nWINGS → WINKS → WONKS\n\nFrom WONKS, I need to:\n- Change K to R\n- Change S to Y\n\nLet me try changing K to R:\nWONKS → WORKS\nPerfect! WORKS is definitely a valid word.\n\nSo now:\nWINGS → WINKS → WONKS → WORKS\n\nFrom WORKS, I still need to:\n- Change S to Y\n\nBut wait, looking at the target WORMY, I notice I need M before Y. So from WORKS, maybe I should change S to M first?\n\nWORKS → WORMS\nYes! WORMS is a valid word.\n\nNow I have:\nWINGS → WINKS → WONKS → WORKS → WORMS\n\nFinally, from WORMS to WORMY, just need to change S to Y:\nWORMS → WORMY\n\nLet me verify the complete path:\n1. WINGS → WINKS (change G to K)\n2. WINKS → WONKS (change I to O)\n3. WONKS → WORKS (change K to R)\n4. WORKS → WORMS (change K to M)\n5. WORMS → WORMY (change S to Y)\n\nLet me check if each word is valid:\n- WINGS: valid (plural of wing)\n- WINKS: valid (plural of wink)\n- WONKS: valid (plural of wonk)\n- WORKS: valid (plural of work)\n- WORMS: valid (plural of worm)\n- WORMY: valid (full of worms)\n\nCould there be a shorter path? Let me think...\nCould I go WINGS → WONGS (I to O)? But is WONGS a valid word? Not sure about that.\nOr WINGS → WRINGS (add R)? No, that would be adding a letter, not changing one.\nOr WINGS → WORMS directly? No, that would require changing multiple letters at once.\n\nThe path I found requires five steps, which makes sense because we needed to change four letters (I→O, N→R, G→M, S→Y) and couldn't combine any changes since we can only change one letter at a time.\n\nSo after exploring alternatives, I believe the path:\nWINGS → WINKS → WONKS → WORKS → WORMS → WORMY\nis optimal, with each step changing exactly one letter and forming a valid English word.\n"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":58,"cache_creation_input_tokens":0,"cache_read_input_tokens":6434,"output_tokens":962}}}}
+{"custom_id":"PINTO_RASPS_3","result":{"type":"succeeded","message":{"id":"msg_01GJQuw2NwcJQpx2civyj8Fu","type":"message","role":"assistant","model":"claude-3-5-sonnet-20241022","content":[{"type":"text","text":"\nAlright, I need to transform PINTO into RASPS by changing one letter at a time. Let me analyze both words first to understand what needs to change.\n\nPINTO\nRASPS\n\nLooking at the letters:\nP -> R\nI -> A\nN -> S\nT -> P\nO -> S\n\nHmm, all five letters need to change. This might be tricky. Let me start by exploring possible changes from PINTO.\n\nFirst letter (P) possibilities:\n- MINTO (is that a word? Maybe a proper noun, but that's not allowed)\n- RINTO (not a word)\n- TINTO (might be Spanish, not valid English)\n\nSecond letter (I) possibilities:\n- PANTO (yes, that's valid!)\n- PENTO (not sure if that's valid)\n- PUNTO (Spanish, not English)\n\nThird letter (N) possibilities:\n- PINTO -> PINTS (changing O to S)\nThat's interesting! PINTS is definitely valid. Maybe that's a good first step since we need to end with 'S' anyway.\n\nLet me try PINTO -> PINTS first.\nNow from PINTS, what can we do? We need to eventually get to RASPS, so maybe changing that I to A would help?\nPINTS -> PANTS (changing I to A)\nYes! PANTS is valid. Now we have:\nPINTO -> PINTS -> PANTS\n\nFrom PANTS, looking at the letters compared to RASPS:\nP -> R\nA (matches!)\nN -> S\nT -> P\nS (matches!)\n\nMaybe we can change that T to R? No, that wouldn't help because we still need P in that position eventually. What about changing N to R? No, we need S there. \n\nLet me try changing T to something... PANTS -> PARTS (changing N to R)\nYes! PARTS is valid. So now:\nPINTO -> PINTS -> PANTS -> PARTS\n\nFrom PARTS to RASPS, we need to:\n- Change P to R at the start\n- Change T to P\n- Keep the S\n\nWait, but if I change P to R now, how would I get the P back later? Maybe I need an intermediate step.\n\nWhat if... PARTS -> WARTS (changing P to W)?\nYes! That's valid. Then from WARTS, maybe... WARPS (changing T to P)?\nWARTS -> WARPS works! \n\nSo now I have:\nPINTO -> PINTS -> PANTS -> PARTS -> WARTS -> WARPS\n\nGetting closer! From WARPS to RASPS, I need to:\n- Change W to R\n- Change P to S in the middle\n\nBut wait, if I change W to R directly, would that work? Let me try another approach...\nWARPS -> WASPS (changing R to S)\nYes! That works. Then finally:\nWASPS -> RASPS (changing W to R)\n\nLet me verify the complete path:\n1. PINTO -> PINTS (O to S)\n2. PINTS -> PANTS (I to A)\n3. PANTS -> PARTS (N to R)\n4. PARTS -> WARTS (P to W)\n5. WARTS -> WARPS (T to P)\n6. WARPS -> WASPS (R to S)\n7. WASPS -> RASPS (W to R)\n\nChecking each word:\nPINTO - valid (type of horse/car)\nPINTS - valid (plural of pint)\nPANTS - valid (clothing)\nPARTS - valid (components)\nWARTS - valid (skin growths)\nWARPS - valid (bends/distorts)\nWASPS - valid (insects)\nRASPS - valid (coarse files/harsh sounds)\n\nEach step changes exactly one letter and creates a valid English word. Seven steps in total. Let me double-check if there might be a shorter path...\n\nCould I have gone from PINTS directly to PARTS? No, that would be changing two letters (I to A and N to R).\nCould I have skipped WARTS? If I went PARTS -> WARPS directly, that would change two letters (P to W and T to P).\nCould I have gone from WARPS to RASPS directly? No, that would change two letters (W to R and P to S).\n\nNo, each step seems necessary. Therefore, this seven-step solution appears to be valid and efficient:\nPINTO -> PINTS -> PANTS -> PARTS -> WARTS -> WARPS -> WASPS -> RASPS\n"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":67,"cache_creation_input_tokens":6434,"cache_read_input_tokens":0,"output_tokens":1081}}}}
+{"custom_id":"GRACE_WAINS_4","result":{"type":"succeeded","message":{"id":"msg_01He4e1HXh1DXdxPz2j7K3GZ","type":"message","role":"assistant","model":"claude-3-5-sonnet-20241022","content":[{"type":"text","text":"\nLet me tackle this word ladder puzzle. I need to transform GRACE into WAINS by changing just one letter at a time, making sure each step creates a valid English word.\n\nFirst, let me compare the starting and ending words:\nGRACE\nWAINS\n\nLooking at what needs to change:\nG needs to become W\nR needs to become A\nA needs to become I\nC needs to become N\nE needs to become S\n\nWow, all five letters need to change! This might be tricky. Let me start by looking at possible changes from GRACE.\n\nFrom GRACE, I could change:\n- First letter (G): BRACE, TRACE, SPACE... but I need to get to W eventually\n- Second letter (R): GRACE to GLACE? That's valid\n- Third letter (A): GRACE to GRECE? No, that's not a word\n- Fourth letter (C): GRACE to GRADE, GRAVE, GRAPE... wait, GRAPE is interesting\n\nLet me try GRACE → GRAPE (changing C to P)\nThat works! Now from GRAPE, I need to somehow get closer to WAINS.\n\nLooking at GRAPE vs WAINS:\nG still needs to become W\nR still needs to become A\nA needs to become I\nP needs to become N\nE needs to become S\n\nMaybe I can change the A to I next? GRAPE → GRIPE\nYes! GRIPE is a valid word. That's progress because now the middle letter matches our target word.\n\nFrom GRIPE:\nG still needs to become W\nR still needs to become A\nP needs to become N\nE needs to become S\n\nLooking at GRIPE... what can I change? The final E needs to become S eventually. But wait, looking at the P... if I change P to S, would that work? GRIPE → GRISE? No, that's not common. What about GRIPS? Yes! GRIPE → GRIPS works by changing E to S.\n\nNow I have GRIPS. Getting closer! Looking at what's left:\nG needs to become W\nR needs to become A\nI stays I\nP needs to become N\nS stays S\n\nFrom GRIPS, maybe I can change P to N? GRIPS → GRINS\nYes! That works. Now I have GRINS.\n\nLooking at GRINS vs WAINS:\nG needs to become W\nR needs to become A\nI stays I\nN stays N\nS stays S\n\nFrom GRINS, I need to change R to A. GRINS → GAINS\nPerfect! Now I just need to change G to W to reach the target.\n\nGAINS → WAINS\n\nLet me verify the complete path:\n1. GRACE → GRAPE (C→P)\n2. GRAPE → GRIPE (A→I)\n3. GRIPE → GRIPS (E→S)\n4. GRIPS → GRINS (P→N)\n5. GRINS → GAINS (R→A)\n6. GAINS → WAINS (G→W)\n\nLet me check each word:\nGRACE - valid\nGRAPE - valid (fruit)\nGRIPE - valid (complaint)\nGRIPS - valid (holds)\nGRINS - valid (smiles)\nGAINS - valid (increases)\nWAINS - valid (wagons)\n\nAnd each step only changes one letter. This works! Though let me just double-check if there might be a shorter path...\n\nCould I have gone from GRACE directly to WAINS in fewer steps? No, because five letters need to change, so minimum five steps needed. Could I have gone:\nGRACE → GRANS? (Not a word)\nGRACE → GRINS? (No, that would be two letters)\nGRACE → WRACE? (Not a word)\n\nNo, I think the path I found is optimal because:\n1. Each step is a valid word\n2. Each step changes only one letter\n3. We needed to change all five letters\n4. Each change gets us closer to the target\n\nTherefore, the solution path GRACE → GRAPE → GRIPE → GRIPS → GRINS → GAINS → WAINS is correct and efficient.\n"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":61,"cache_creation_input_tokens":0,"cache_read_input_tokens":6434,"output_tokens":967}}}}
\ No newline at end of file
diff --git a/examples/word_ladder/output/example_word_ladders_029573f4.jsonl b/examples/word_ladder/output/example_word_ladders_029573f4.jsonl
new file mode 100644
index 00000000..100610c8
--- /dev/null
+++ b/examples/word_ladder/output/example_word_ladders_029573f4.jsonl
@@ -0,0 +1,10 @@
+{"question": "Transform the word ladder 'PAINT' to 'LACKS' by changing one letter at a time.", "answer": "PAINT,PAINS,WAINS,WARNS,BARNS,BARKS,BACKS,LACKS", "reasoning": null, "metadata": {"start_word": "PAINT", "end_word": "LACKS", "word_length": 5, "chain_length": 8}}
+{"question": "Transform the word ladder 'CONKS' to 'GRAYS' by changing one letter at a time.", "answer": "CONKS,COOKS,COOPS,CROPS,CRAPS,CRAYS,GRAYS", "reasoning": null, "metadata": {"start_word": "CONKS", "end_word": "GRAYS", "word_length": 5, "chain_length": 7}}
+{"question": "Transform the word ladder 'WINGS' to 'WORMY' by changing one letter at a time.", "answer": "WINGS,WINKS,WONKS,WORKS,WORMS,WORMY", "reasoning": null, "metadata": {"start_word": "WINGS", "end_word": "WORMY", "word_length": 5, "chain_length": 6}}
+{"question": "Transform the word ladder 'PINTO' to 'RASPS' by changing one letter at a time.", "answer": "PINTO,PINTS,PANTS,PARTS,WARTS,WARPS,WASPS,RASPS", "reasoning": null, "metadata": {"start_word": "PINTO", "end_word": "RASPS", "word_length": 5, "chain_length": 8}}
+{"question": "Transform the word ladder 'GRACE' to 'WAINS' by changing one letter at a time.", "answer": "GRACE,GRAPE,GRIPE,GRIPS,GRINS,GAINS,WAINS", "reasoning": null, "metadata": {"start_word": "GRACE", "end_word": "WAINS", "word_length": 5, "chain_length": 7}}
+{"question": "Transform the word ladder 'RACES' to 'VAPES' by changing one letter at a time.", "answer": "RACES,RAPES,VAPES", "reasoning": null, "metadata": {"start_word": "RACES", "end_word": "VAPES", "word_length": 5, "chain_length": 3}}
+{"question": "Transform the word ladder 'LAIRS' to 'FUELS' by changing one letter at a time.", "answer": "LAIRS,FAIRS,FAILS,FARLS,FURLS,FUELS", "reasoning": null, "metadata": {"start_word": "LAIRS", "end_word": "FUELS", "word_length": 5, "chain_length": 6}}
+{"question": "Transform the word ladder 'WINKS' to 'TENTH' by changing one letter at a time.", "answer": "WINKS,PINKS,PINTS,TINTS,TENTS,TENTH", "reasoning": null, "metadata": {"start_word": "WINKS", "end_word": "TENTH", "word_length": 5, "chain_length": 6}}
+{"question": "Transform the word ladder 'VERBS' to 'DATES' by changing one letter at a time.", "answer": "VERBS,HERBS,HERDS,HARDS,HARES,HATES,DATES", "reasoning": null, "metadata": {"start_word": "VERBS", "end_word": "DATES", "word_length": 5, "chain_length": 7}}
+{"question": "Transform the word ladder 'SCENT' to 'HEARS' by changing one letter at a time.", "answer": "SCENT,SCANT,SCART,SCARS,SEARS,HEARS", "reasoning": null, "metadata": {"start_word": "SCENT", "end_word": "HEARS", "word_length": 5, "chain_length": 6}}
\ No newline at end of file
diff --git a/examples/word_ladder/requirements.txt b/examples/word_ladder/requirements.txt
new file mode 100644
index 00000000..d0c663a4
--- /dev/null
+++ b/examples/word_ladder/requirements.txt
@@ -0,0 +1,6 @@
+# Required dependencies for the Word Ladder Puzzle Dataset Generator
+# Python 3.7+
+
+anthropic>=0.45.2 # Client library for interacting with Anthropic's API
+tqdm>=4.67.1 # For progress bars
+
diff --git a/examples/word_ladder/system_prompt.txt b/examples/word_ladder/system_prompt.txt
new file mode 100644
index 00000000..29d8595e
--- /dev/null
+++ b/examples/word_ladder/system_prompt.txt
@@ -0,0 +1,288 @@
+Word Ladder puzzles involve transforming a start word into an end word. You are allowed to change only one letter a time and you must keep the number of letters constant.
+Each time you change one letter the word in the chain must be forming a new word that's valid in English. Plurals are allowed, but not proper nouns.
+Your task will be to simulate a detailed step-by-step chain of thought reasoning of the transformation process from a start word to the end word of a word ladder.
+You will be given the word ladder question, as well as the correct solution path. So you don't need to solve the problem, you have the solution already, which should be your 'cheat sheet' to refer to as you go along.
+Your task is to provide a perfectly simulated chain of thought reasoning exactly in the style of the example below,
+including considering multiple possibilities, backtracking where needed, validating words, and showing the final path.
+Remember you're NOT trying to 'find' the shortest path necessarily. Make sure you're not lazy and show the user that you're working hard to find the intermediate words in the chain, and that you haven't gotten simply lucky in finding the steps.
+For example, you may go down a path that's no longer getting you closer to the solution, so you backtrack and try a different path, which ends up working.
+Use casual, free flowing verbose, human-like thinking language. You must convince the user that you're really 'REASONING'. You can see the real reasoning in the examples below, which may look convoluted, but it's true reasoning, that is what we need.
+It's ok to pretend catch yourself making a mistake, and then correct yourself, adds realism to the chain of thought.
+VERY IMPORTANT ***NEVER MENTION YOU HAVE BEEN PROVIDED WITH THE ANSWER, YOU NEED TO WRITE ONLY AS IF YOU'RE SOLVING THE QUESTION STEP BY STEP ***
+
+Example chain of thought reasoning:
+
+Okay, so I need to figure out how to transform the word "CARK" into "ZIPS" by changing one letter at a time, making sure each intermediate step is a valid English word. Let me start by breaking down the problem.
+
+First, both words are four letters long, so each step must maintain four letters. Each change can only alter one letter, either adding, removing, or replacing, but since the length must stay the same, it has to be a replacement each time. So, I need to find a path from CARK to ZIPS where each intermediate word is valid.
+
+Let me write down the starting and target words:
+
+CARK
+
+...
+
+ZIPS
+
+I need to see which letters need to change. Let's compare the letters:
+
+C -> Z
+
+A -> I
+
+R -> P
+
+K -> S
+
+So all four letters need to change. But since each step can only change one letter, I need to find a sequence where each single-letter change results in a valid word. Maybe I can approach this by changing one letter at a time, checking possible words each step.
+
+Starting with CARK. Let's look at possible changes for each letter:
+
+First letter C: changing to other letters to form a word. Possible options could be BARK (C to B), DARK (C to D), LARK (C to L), MARK (C to M), PARK (C to P), etc. Wait, but CARK is a valid word? Let me confirm. CARK means to worry or burden, so yes, it's valid.
+
+So, possible first steps from CARK:
+
+Change C to B: BARK
+
+Change C to D: DARK
+
+Change C to L: LARK
+
+Change C to M: MARK
+
+Change C to P: PARK
+
+Change C to W: WARK (is that a word? Wark might be a Scottish term for work, but I'm not sure if it's accepted in standard English. Maybe not.)
+
+Other possibilities: changing the A, R, or K. Let's see:
+
+Second letter A: changing to other vowels or consonants. For example, CORK (changing A to O), but CORK is valid. Or CIRK (changing A to I; cirk is a variant of circus, but I think "cirk" isn't standard. Maybe CIRC? No, that's not right. Wait, maybe CIRK is a valid word in some contexts, like a circular trench in fortification? Hmm, not sure. Better to stick with more common words.
+
+Third letter R: changing to another letter. For example, CAKE (changing R to K), but that would require changing both the third letter and the fourth letter? Wait, no: CARK to CAKE would be changing R to E and K to E? Wait, no. Wait, CARK is C-A-R-K. To get CAKE, you'd need to change R to K and K to E, which is two changes. So not possible in one step. Maybe changing R to another letter. For example, CASH (changing R to S and K to H? No, that's two letters again. Wait, no. Let's see: CARK. Changing the third letter R to another letter, like CASK (changing R to S). But that would be changing the third letter from R to S, resulting in CASK. Yes, that's a valid word. So CARK -> CASK.
+
+Fourth letter K: changing to another letter. For example, CARD (changing K to D), CARP (changing K to P), CARS (changing K to S), CART (changing K to T). All of these are valid.
+
+So possible first steps from CARK:
+
+- BARK, DARK, LARK, MARK, PARK, CASK, CARD, CARP, CARS, CART.
+
+Now, which of these would lead towards ZIPS? Let's think about the target word. ZIPS has a Z, which is the first letter. So maybe we need to get to a word starting with Z eventually. But changing from C to Z in one step would be a big jump. Alternatively, maybe we can change other letters first to reach intermediate steps that can approach Z.
+
+Alternatively, maybe work on changing the last letters first. Let's see. For example, changing K to S (CARS) might be useful since the target ends with S. Let's try that.
+
+CARK -> CARS (changing K to S). Now, from CARS, we need to change the third letter R to P and the first letter C to Z, and the second A to I. Let's see.
+
+From CARS, possible changes:
+
+Change C to Z: ZARS (is that a word? ZAR is a currency, plural ZARS? Maybe, but not sure. Alternatively, ZAPS (changing R to P and S to something? Wait, no. Let's check each letter.
+
+Alternatively, from CARS, change A to I: CIRS (is that a word? CIR is a three-letter word, but CIRS? Maybe not. Or CURS (changing A to U), which is valid. But how does that help?
+
+Alternatively, from CARS, change R to P: CAPS. Yes, that's valid. So CARS -> CAPS (changing R to P). Then from CAPS, we can change C to Z: ZAPS. Yes, ZAPS is a word. Then ZAPS -> ZIPS by changing A to I. So that would be a path:
+
+CARK -> CARS -> CAPS -> ZAPS -> ZIPS.
+
+Wait, let's check each step:
+
+1. CARK to CARS: change K to S. Valid.
+
+2. CARS to CAPS: change R to P. Valid.
+
+3. CAPS to ZAPS: change C to Z. Valid.
+
+4. ZAPS to ZIPS: change A to I. Valid.
+
+That's four steps. But wait, starting from CARK (step 0), then steps 1-4 would be four changes. But the problem says "transform CARK into ZIPS by changing one letter at a time", so the number of steps would be the number of letters changed. Since all four letters are different, at minimum it would take four steps. But perhaps the path is longer if there are no shorter paths.
+
+Wait, let me check if this works:
+
+CARK → CARS → CAPS → ZAPS → ZIPS.
+
+Yes, that's four steps. Each step changes one letter, and all intermediate words are valid. Let me verify each word:
+
+- CARK: valid.
+
+- CARS: valid.
+
+- CAPS: valid.
+
+- ZAPS: valid.
+
+- ZIPS: valid.
+
+Yes, that seems to work. But let me check if there's a shorter path or if this is correct.
+
+Alternatively, is there a three-step path? Probably not since four letters need to change. Each step can only change one letter, so at least four steps are needed. So the path above is four steps, which is the minimum. Let me confirm each step again.
+
+Another possible path: changing the first letter earlier. For example:
+
+CARK → PARK (change C to P). Then PARK → PARKS (but that's five letters; can't do that. Alternatively, changing K to S: PARS. Then PARS -> PAWS (changing R to W?), but not helpful. Or PARK → PARS (change K to S), then PARS → PAIS (change R to I?), but PAIS is valid. Then PAIS → ZIPS (change P to Z and A to I? No, that's two letters. Hmm, not helpful.
+
+Alternatively, CARK → DARK → DANK → DINK → ZINK → ZIPS? Let's see. CARK to DARK (C→D). DARK to DANK (R→N). DANK to DINK (A→I). DINK to ZINK (D→Z). ZINK is a variant spelling of zinc, but ZINK is a valid word. Then ZINK to ZIPS (K→S and N→P? Wait, no. ZINK is Z-I-N-K. To get to ZIPS, we need Z-I-P-S. So need to change N to P and K to S. That's two steps. So that path would be longer.
+
+Another path: CARK → CARD → ZARD? ZARD isn't a word. CARD → WARD → WARD → ... not helpful.
+
+Alternatively, CARK → CARK to CIRK (A→I). CIRK is a variant of circus, but maybe not standard. Then CIRK → ZIRK (C→Z). ZIRK is a surname or a brand, not sure if it's a valid word. Then ZIRK → ZIPS (R→P and K→S). But again, two letters at once. Not helpful.
+
+So the first path I thought of seems viable: CARK → CARS → CAPS → ZAPS → ZIPS.
+
+Another check: From CARK, can we go to CASK? CARK → CASK (R→S). Then CASK → ZASK (C→Z). ZASK? Not a word. Alternatively, CASK → CASK → CASH (K→H). Then CASH → ZASH (C→Z). Not a word. Not helpful.
+
+Alternatively, CARK → CARP (K→P). CARP is valid. Then CARP → CARP → ZARP (C→Z). ZARP? Not a word. Or CARP → HARp (C→H), but HARP is valid. Then HARP → HIPS (A→I and R→P). But that's two changes. Not allowed.
+
+So the initial path seems best. Let me confirm each step again:
+
+1. CARK → CARS: change K to S. Valid.
+
+2. CARS → CAPS: change R to P. Valid.
+
+3. CAPS → ZAPS: change C to Z. Valid.
+
+4. ZAPS → ZIPS: change A to I. Valid.
+
+Yes, that's four steps, each changing one letter, all words are valid. So that should be the correct sequence.
+
+
+alternative chain of thought:
+ Okay, so I need to figure out how to transform the word "BALL" into "POMP" by changing one letter at a time, with each intermediate step being a valid English word. Let me start by writing down both words to see the letters that need to change.
+BALL → POMP
+
+First, let's look at each position:
+
+B (1st letter) needs to become P.
+A (2nd letter) needs to become O.
+L (3rd letter) needs to become M.
+L (4th letter) needs to become P.
+
+So each letter position needs to change. The challenge is to do this one letter at a time, and each step must result in a valid word. Let me think about possible steps.
+
+Starting with BALL. Let's see what words can be formed by changing one letter.
+
+For the first letter (B), changing it to other letters might give words like CALL, FALL, HALL, MALL, PALL, WALL. Those are all valid. Alternatively, changing the second letter (A) could lead to BELL, BILL, BOLL, BULL. Changing the third letter (L) might result in BALM, BALE, BALK, etc. Changing the fourth letter (L) could give BALD, BALE, BALM, etc.
+
+Wait, but the target word is POMP. So maybe we need to adjust letters step by step. Let's try to approach this step by step.
+
+Option 1: Change the first letter from B to P. That would give PALL. Is PALL a valid word? Yes. So step 1: BALL → PALL.
+
+Now, PALL. Next, maybe change the second letter A to O. That would make POLL. POLL is a valid word. Step 2: PALL → POLL.
+
+Now, POLL. Next, change the third letter L to M. That would make POMP? Wait, no. Because POLL has two Ls. If we change the third L to M, it would be POML, which isn't a word. Alternatively, change the fourth L to P? That would be POLP, which isn't a word. Hmm. Maybe this path isn't working.
+
+Alternative approach. Let's try changing a different letter first. Let's see. Starting with BALL.
+
+Another possibility: Change the second letter A to O first. Then we get BOLL. BOLL is a valid word (a type of seed capsule). Step 1: BALL → BOLL.
+
+From BOLL, maybe change the first letter B to P. That gives POLL. Step 2: BOLL → POLL.
+
+Now, POLL again. Then, similar problem as before. Need to get to M in the third position. Maybe change the third L to M. But that gives POML, which isn't valid. Alternatively, change the fourth L to P. POLP isn't a word. Hmm.
+
+Alternative path: From BOLL, maybe change the third letter L to M. That would make BOLM. Is BOLM a word? I don't think so. Or change the fourth L to P: BOLP. Not a word.
+
+Maybe another starting step. Let's try changing the third letter in BALL from L to M. So BALL → BALM. Step 1: BALL → BALM.
+
+BALM is valid. Now, change the fourth letter M to P? Wait, BALM is B-A-L-M. Wait, no. Wait, original word is BALL (B-A-L-L). Changing the third L to M would make BALM. Then changing the fourth L (from original) to something else. Wait, after BALM, the fourth letter is M? Wait, no. Wait, BALL is B-A-L-L. If I change the third L to M, the word becomes BALM (B-A-L-M). Wait, but then the fourth letter is M, so BALM. Then the fourth letter is M. To get to POMP, which ends with P, maybe we can change that M to P. But BALM → BALP? BALP isn't a valid word. Hmm.
+
+Alternatively, maybe change the fourth letter first. BALL → BALD (changing the last L to D). Then BALD. But I'm not sure if that helps. Let's see. BALD. Then maybe change A to O: BOLD. Then BOLD. Then change B to P: POLD. Not a word. Hmm.
+
+Alternatively, BALL → BALE (changing the last L to E). BALE is valid. Then BALE → PALE (changing B to P). Then PALE → POLE (changing A to O). Then POLE → POLL (changing E to L). Then POLL → POMP (changing L to M and L to P? Wait, but that's two changes at once. Not allowed. So this path might not work.
+
+Wait, perhaps another approach. Let's consider the target word POMP. The letters are P-O-M-P. So after changing some letters, maybe we can reach a word that has some of these letters, then proceed.
+
+Let me try starting with BALL. Change the first letter B to P: PALL. Then change the second letter A to O: POLL. Then change the third letter L to M: POMP? Wait, no. POLL → changing third L to M would make POML. Not valid. But if we can change the third and fourth letters? Wait, but we can only change one letter each step.
+
+Alternatively, from POLL, change the fourth L to P. That would make POLP, which isn't a word. Not helpful.
+
+Hmm. Maybe a different path. Let's think of possible intermediate words that can connect these.
+
+Another idea: BALL → BOLT (changing A to O and L to T? No, that's two changes. Not allowed. Wait, no. To get BOLT from BALL, you need to change the second letter A to O (making BOLL) and then the fourth L to T. But that's two steps. Alternatively, maybe BALL → BOLT by changing A to O and third L to T? No, that's two changes again.
+
+Alternatively, BALL → BULK (changing A to U and third L to K. Again two changes. Not allowed.
+
+Wait, maybe changing the third letter first. BALL → BALE (changing third L to E). Then BALE → BOLT (changing A to O and E to T. Two changes. Not allowed. Hmm.
+
+Alternatively, BALL → BELL (changing A to E). Then BELL → BELL is same, then maybe BELL → BOLT (changing E to O and L to T. Again two steps.
+
+This is tricky. Let me try another angle. Let's look for words that are one letter away from POMP. For example, possible previous steps before POMP could be:
+
+POOP (changing M to O), but POOP is valid. Then from POOP to POMP by changing O to M. But that's a possible step. But how to get to POOP from previous steps.
+Alternatively, POMP could come from changing a letter in a word like POOP, POMY (not a word), POPS (changing S to M? No), etc. Maybe not helpful.
+
+Alternatively, the step before POMP could be something like POMM (if that's a word; not sure). Probably not.
+
+Alternatively, consider that the third letter needs to be M. So maybe a word like POM_ something. For example, POMP. So maybe from a word like POME (a type of fruit), changing E to P. But POME to POMP would require changing E to P. But POME is a valid word. So step: POME → POMP. Then, how to get to POME.
+
+But how to get from BALL to POME. Let's see. Let's try:
+
+BALL → BALE (change third L to E)
+BALE → PALE (change B to P)
+PALE → PALE (same)
+PALE → POLE (change A to O)
+POLE → POLL (change E to L)
+POLL → POLL (same)
+POLL → POMP (change L to M and L to P. Wait, that's two letters. Not allowed.
+
+Alternatively, maybe a different path. Let's think of words with M in the third position. For example, BALM (which we considered earlier). BALM → BAMM (not a word). Hmm.
+
+Wait, BALM is B-A-L-M. If we change the first letter B to P, we get PALM. PALM is a valid word. So BALL → BALM (changing third L to M), then BALM → PALM (changing B to P). Then PALM. Now, from PALM, maybe change the fourth M to P. That gives PALP. PALP is a valid word (to palpate). So PALM → PALP. Then PALP → POMP. How? Change A to O and L to M. But that's two letters. Alternatively, change the second letter A to O: POLP (not a word). Hmm.
+
+Alternatively, from PALM, change the second letter A to O: POLM. Not a word. Not helpful.
+
+Hmm. Maybe another path. Let's think of changing the second letter first. BALL → BOLL (A to O). Then BOLL → BOLT (changing L to T). Then BOLT → BOLT (same). Then BOLT → MOAT (changing B to M and L to A. Two changes. Not allowed.
+
+Alternatively, BOLL → POLL (B to P). Then POLL → POLE (L to E). Then POLE → POME (E to M and L to E? No, that's two changes. Wait, POLE to POME would require changing L to M and E to E. Wait, no. POLE is P-O-L-E. To get to POME, you need to change L to M and E to E. Wait, no. Changing L to M would make POME? P-O-M-E. Yes, POME is a word. So POLE → POME (changing L to M). Then POME → POMP (changing E to P). So that's possible.
+
+So let's outline this path:
+
+BALL → BOLL (change A to O)
+BOLL → POLL (change B to P)
+POLL → POLE (change L to E)
+POLE → POME (change L to M)
+POME → POMP (change E to P)
+Wait, but step 3: POLL to POLE. POLL is P-O-L-L. Changing the fourth L to E gives POLE? Wait, no. If you change the fourth L (the second L) to E, then it would be POLE. Yes. So POLL → POLE (changing the fourth letter L to E). Then POLE → POME (changing the third letter L to M). Wait, POLE is P-O-L-E. Changing the third letter L to M gives P-O-M-E, which is POME. Then POME → POMP by changing the fourth letter E to P. That works.
+
+So the steps would be:
+
+BALL → BOLL
+BOLL → POLL
+POLL → POLE
+POLE → POME
+POME → POMP
+But wait, step 3: POLL to POLE. Is that valid? Yes, because changing the fourth L to E gives POLE. Then step 4: POLE to POME by changing the third L (but in POLE, the third letter is L? Wait, POLE is P-O-L-E. So third letter is L. Changing that to M gives P-O-M-E, which is POME. Then POME to POMP by changing the fourth E to P. That works. So that's five steps. But the problem says "change one letter at a time" without specifying the minimum number of steps, so as long as each step is valid, it's okay.
+
+But let's check if all these words are valid:
+
+BOLL: Yes, a type of seed capsule.
+POLL: Yes.
+POLE: Yes.
+POME: Yes, a fruit like an apple.
+POMP: Target.
+So that's a valid sequence. But wait, the original problem is to go from BALL to POMP. Let me count the steps. BALL (1) → BOLL (2) → POLL (3) → POLE (4) → POME (5) → POMP (6). Wait, that's five steps, but the count is six words including the start and end. Wait, the problem says "transform ... by changing one letter at a time. Each step must create a valid English word". So the sequence should be BALL → word1 → word2 → ... → POMP. So in this case, five changes (since each change is a step). But let me check each step again.
+
+BALL → BOLL (change A to O)
+BOLL → POLL (change B to P)
+POLL → POLE (change fourth L to E)
+POLE → POME (change third L to M)
+POME → POMP (change fourth E to P)
+Yes, that's five steps. Each step changes one letter, and each intermediate word is valid. So this works.
+
+Alternatively, is there a shorter path? Let's see. Let's try another path.
+
+Starting with BALL → PALL (B to P). Then PALL → POLL (A to O). Then POLL → POMP. But how? POLL to POMP requires changing L to M and L to P. That's two changes. Not allowed. So that path would require two steps from POLL: first change third L to M (making POML, invalid), then fourth L to P (POLP, invalid). So that doesn't work.
+
+Alternatively, another path: BALL → BALE (third L to E). BALE → PALE (B to P). PALE → POLE (A to O). POLE → POME (L to M). POME → POMP (E to P). Same as the previous path but starting differently. So that's also five steps.
+
+Alternatively, BALL → BALM (third L to M). BALM → PALM (B to P). PALM → PALP (M to P). PALP → POMP (A to O and L to M). But changing two letters at once isn't allowed. So that doesn't work.
+
+So the path through BOLL, POLL, POLE, POME seems valid. Let me confirm each word:
+
+BOLL: valid.
+POLL: valid.
+POLE: valid.
+POME: valid.
+POMP: target.
+Yes, this works. Therefore, the sequence is BALL → BOLL → POLL → POLE → POME → POMP.
+
+
+
+Finally, I've seen in your previous answers to similar questions that you sometimes get complacent and deviate from the hard work required. You must maintain the style and level effort throughout the entirety of your answer until the end.
+Now, the user will provide you with the word ladder question and the correct solution path. Provide the user with a very verbose, first person chain of thought reasoning that is simulating how a human would methodically think to solve the word ladder step by step.
\ No newline at end of file
diff --git a/examples/word_ladder/tests/test_generate_reasoning.py b/examples/word_ladder/tests/test_generate_reasoning.py
new file mode 100644
index 00000000..6ee5be1e
--- /dev/null
+++ b/examples/word_ladder/tests/test_generate_reasoning.py
@@ -0,0 +1,239 @@
+import os
+import json
+import time
+import datetime
+import pytest
+from pathlib import Path
+from types import SimpleNamespace
+
+from examples.word_ladder import generate_reasoning
+
+# We alias the functions and globals for easier usage in our tests.
+submit_reasoning_batches = generate_reasoning.submit_reasoning_batches
+_submit_single_batch = generate_reasoning._submit_single_batch
+DEFAULT_INPUT_JSONL = generate_reasoning.DEFAULT_INPUT_JSONL
+COMMON_UUID = generate_reasoning.COMMON_UUID
+BATCH_SIZE = generate_reasoning.BATCH_SIZE
+client = generate_reasoning.client
+
+# Define a mock batch response class mimicking Anthropic's API response.
+class MockBatchResponse:
+ def __init__(self, batch_id="msgbatch_mock", processing_status="in_progress", fail=False):
+ self.id = batch_id
+ self.type = "message_batch"
+ self.processing_status = processing_status
+ # Make request_counts a SimpleNamespace object with the required attributes
+ self.request_counts = SimpleNamespace(
+ processing=0,
+ succeeded=0,
+ errored=0,
+ canceled=0,
+ expired=0
+ )
+ self.ended_at = None
+ # Use datetime objects so that isoformat() is available
+ self.created_at = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)
+ self.expires_at = self.created_at + datetime.timedelta(seconds=86400)
+ self.cancel_initiated_at = None
+ self.results_url = None
+
+# Helper: Create a temporary system prompt file.
+@pytest.fixture
+def system_prompt_file(tmp_path, monkeypatch):
+ prompt_text = "This is a system prompt."
+ sys_file = tmp_path / "system_prompt.txt"
+ sys_file.write_text(prompt_text, encoding="utf-8")
+
+ # Monkeypatch the system prompt path
+ monkeypatch.setattr(generate_reasoning, "DEFAULT_SYSTEM_PROMPT", str(sys_file))
+ return sys_file
+
+# Helper: Create necessary directories using a temporary location.
+@pytest.fixture
+def setup_directories(tmp_path, monkeypatch):
+ # Create output directory in temporary path
+ output_dir = tmp_path / "output"
+ output_dir.mkdir(exist_ok=True)
+
+ # Monkeypatch the DEFAULT_OUTPUT_DIR to be a Path (temporary directory).
+ monkeypatch.setattr(generate_reasoning, "DEFAULT_OUTPUT_DIR", output_dir)
+
+ # Ensure we're working in the temporary directory
+ monkeypatch.chdir(tmp_path)
+ return output_dir
+
+# Helper: Create a temporary input JSONL file with given entries.
+@pytest.fixture
+def input_jsonl_file(tmp_path, setup_directories, monkeypatch):
+ # Create input file in temporary directory
+ file_path = setup_directories / "word_ladder_examples.jsonl"
+ entries = [
+ { "question": "Transform 'A' to 'B'", "answer": "A,X,B", "reasoning": None,
+ "metadata": { "start_word": "A", "end_word": "B", "word_length": 1, "chain_length": 3 } },
+ { "question": "Transform 'C' to 'D'", "answer": "C,Y,D", "reasoning": "Some reasoning",
+ "metadata": { "start_word": "C", "end_word": "D", "word_length": 1, "chain_length": 3 } },
+ { "question": "Transform 'E' to 'F'", "answer": "E,Z,F", "reasoning": None,
+ "metadata": { "start_word": "E", "end_word": "F", "word_length": 1, "chain_length": 3 } }
+ ]
+ with file_path.open("w", encoding="utf-8") as f:
+ for entry in entries:
+ f.write(json.dumps(entry) + "\n")
+
+ # Monkeypatch DEFAULT_INPUT_JSONL to point to our temporary test file.
+ monkeypatch.setattr(generate_reasoning, "DEFAULT_INPUT_JSONL", str(file_path))
+ return file_path
+
+# Test that submit_reasoning_batches builds a batch skipping entries with existing reasoning.
+def test_submit_batches_success(system_prompt_file, input_jsonl_file, setup_directories, monkeypatch):
+ def fake_create(requests):
+ for req in requests:
+ # Handle the case where req is a dictionary
+ if isinstance(req, dict):
+ params = req.get("params", {})
+ custom_id = req.get("custom_id")
+ # Check if params itself is a dictionary
+ if isinstance(params, dict):
+ model = params.get("model")
+ temperature = params.get("temperature")
+ else:
+ model = params.model
+ temperature = params.temperature
+ else:
+ # Else, req is an object with attributes.
+ params = req.params
+ custom_id = req.custom_id
+ if isinstance(params, dict):
+ model = params.get("model")
+ temperature = params.get("temperature")
+ else:
+ model = params.model
+ temperature = params.temperature
+ assert model == "claude-3-5-sonnet-20241022", "Incorrect model version"
+ assert temperature == 0.5, "Incorrect temperature"
+ assert "C_D_" not in custom_id
+ return MockBatchResponse(batch_id="msgbatch_test_success")
+
+ monkeypatch.setattr(client.messages.batches, "create", fake_create)
+
+ batch_metadata_prefix = "test_metadata"
+ submit_reasoning_batches(input_path=str(input_jsonl_file),
+ batch_metadata_prefix=batch_metadata_prefix)
+
+ metadata_filename = f"{COMMON_UUID}_{batch_metadata_prefix}.jsonl"
+ meta_file_path = setup_directories / metadata_filename
+ assert meta_file_path.exists(), "Metadata file was not created as expected."
+
+ with meta_file_path.open("r", encoding="utf-8") as f:
+ lines = f.readlines()
+ # Expecting only those entries that did not already have a reasoning.
+ # (From our test input, 2 out of 3 entries qualify.)
+ assert len(lines) > 0
+ for line in lines:
+ metadata = json.loads(line)
+ api_response = metadata["api_response"]
+ assert api_response["id"] == "msgbatch_test_success"
+ assert api_response["processing_status"] == "in_progress"
+ custom_ids = metadata["custom_ids"]
+ assert len(custom_ids) == 2
+
+# Test that _submit_single_batch retries once and eventually succeeds.
+def test_retry_logic(system_prompt_file, setup_directories, monkeypatch):
+ call_count = {"count": 0}
+ def fake_create_retry(requests):
+ if call_count["count"] == 0:
+ call_count["count"] += 1
+ raise Exception("Temporary failure")
+ return MockBatchResponse(batch_id="msgbatch_retry_success")
+
+ monkeypatch.setattr(client.messages.batches, "create", fake_create_retry)
+
+ dummy_request = type("DummyRequest", (), {"custom_id": "dummy_1"})()
+ batch_requests = [dummy_request]
+ custom_ids = ["dummy_1"]
+
+ _submit_single_batch(batch_requests, custom_ids, 0, "test_retry", "dummy_input.jsonl")
+
+ metadata_filename = f"{COMMON_UUID}_test_retry.jsonl"
+ meta_file_path = setup_directories / metadata_filename
+ assert meta_file_path.exists(), "Retry metadata file was not created."
+
+ with meta_file_path.open("r", encoding="utf-8") as f:
+ metadata = json.loads(f.read())
+ assert metadata["api_response"]["id"] == "msgbatch_retry_success"
+
+ assert call_count["count"] == 1
+
+# Test that when all attempts to submit a batch fail, the error is logged to the failed file.
+def test_failed_batch(system_prompt_file, setup_directories, monkeypatch):
+ def fake_create_fail(requests):
+ raise Exception("Permanent failure")
+
+ monkeypatch.setattr(client.messages.batches, "create", fake_create_fail)
+
+ dummy_request = type("DummyRequest", (), {"custom_id": "dummy_fail"})()
+ batch_requests = [dummy_request]
+ custom_ids = ["dummy_fail"]
+
+ _submit_single_batch(batch_requests, custom_ids, 0, "test_failed", "dummy_input.jsonl")
+
+ error_filename = f"{COMMON_UUID}_failed_batches.jsonl"
+ error_file_path = setup_directories / error_filename
+ assert error_file_path.exists(), "Failed batch log file was not created."
+
+ with error_file_path.open("r", encoding="utf-8") as f:
+ error_entry = json.loads(f.readline())
+ assert error_entry["batch_number"] == 0
+ assert "Permanent failure" in error_entry["error"]
+ assert error_entry["batch_requests"] == ["dummy_fail"]
+
+# Test batching behavior when multiple batches are needed.
+def test_multiple_batches(system_prompt_file, setup_directories, monkeypatch):
+ test_batch_size = 2
+ monkeypatch.setattr(generate_reasoning, "BATCH_SIZE", test_batch_size)
+
+ # Create input file
+ input_file = setup_directories / "word_ladder_examples.jsonl"
+ entries = [
+ {
+ "question": f"Transform word ladder {idx}",
+ "answer": f"start,mid,end_{idx}",
+ "reasoning": None,
+ "metadata": {"start_word": f"start_{idx}", "end_word": f"end_{idx}"}
+ }
+ for idx in range(5)
+ ]
+
+ with input_file.open("w", encoding="utf-8") as f:
+ for entry in entries:
+ f.write(json.dumps(entry) + "\n")
+
+ # Monkeypatch DEFAULT_INPUT_JSONL.
+ monkeypatch.setattr(generate_reasoning, "DEFAULT_INPUT_JSONL", str(input_file))
+
+ batch_ids = []
+ def fake_create(requests):
+ new_id = f"msgbatch_batch_{len(batch_ids)}"
+ batch_ids.append(new_id)
+ return MockBatchResponse(batch_id=new_id)
+
+ monkeypatch.setattr(client.messages.batches, "create", fake_create)
+
+ batch_metadata_prefix = "test_multi"
+ submit_reasoning_batches(input_path=str(input_file),
+ batch_metadata_prefix=batch_metadata_prefix)
+
+ metadata_filename = f"{COMMON_UUID}_{batch_metadata_prefix}.jsonl"
+ meta_file_path = setup_directories / metadata_filename
+ assert meta_file_path.exists(), "Multiple batch metadata file was not created."
+
+ with meta_file_path.open("r", encoding="utf-8") as f:
+ metadata_lines = f.readlines()
+ # With 5 qualifying entries and a batch size of 2 we expect 3 batches.
+ assert len(metadata_lines) == 3
+
+ seen_custom_ids = []
+ for line in metadata_lines:
+ metadata = json.loads(line)
+ seen_custom_ids.extend(metadata["custom_ids"])
+ assert metadata["api_response"]["id"] in batch_ids
+ assert len(seen_custom_ids) == 5
\ No newline at end of file
diff --git a/examples/word_ladder/utils/__init__.py b/examples/word_ladder/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/examples/word_ladder/utils/create_word_ladders.py b/examples/word_ladder/utils/create_word_ladders.py
new file mode 100644
index 00000000..3770d81a
--- /dev/null
+++ b/examples/word_ladder/utils/create_word_ladders.py
@@ -0,0 +1,131 @@
+"""
+create_word_ladders.py – Generates the word ladder dataset as a JSONL file.
+Each line is a JSON object with keys: question, answer, metadata, and reasoning (set to None).
+"""
+
+import json
+import uuid
+from pathlib import Path
+
+from tqdm import tqdm
+
+import reasoning_gym
+
+def check_duplicates(jsonl_path: str) -> tuple[bool, dict]:
+ """
+ Check for duplicate word pairs in a word ladder JSONL file.
+
+ Returns:
+ tuple[bool, dict]: (has_duplicates, valid_entries) where:
+ - has_duplicates: True if any duplicates were found
+ - valid_entries: Dict mapping line_number -> data for non-duplicate entries
+
+ Note: A pair is considered duplicate if either (word1, word2) or (word2, word1)
+ already exists, since word ladder paths are bidirectional.
+ """
+ pairs_seen = {} # (start, end) -> (line_number, data)
+ valid_entries = {}
+ duplicates_found = False
+
+ with open(jsonl_path, 'r', encoding='utf-8') as f:
+ for line_num, line in enumerate(f):
+ data = json.loads(line)
+ metadata = data['metadata']
+ pair = (metadata['start_word'], metadata['end_word'])
+ reverse_pair = (metadata['end_word'], metadata['start_word'])
+
+ # Check both orientations of the pair
+ if pair in pairs_seen or reverse_pair in pairs_seen:
+ duplicates_found = True
+ # Skip this entry - it's a duplicate
+ continue
+ else:
+ # Store both the line number and data for valid entries
+ pairs_seen[pair] = (line_num, data)
+ valid_entries[line_num] = data
+
+ return duplicates_found, valid_entries
+
+def create_word_ladder_dataset(jsonl_path: str = None, config: dict = None) -> None:
+ """
+ Creates a word ladder dataset and writes each sample as a JSON line.
+ Ensures no duplicate word pairs by regenerating as needed.
+ """
+ if config is None:
+ raise ValueError("Configuration (config) must be provided.")
+
+ # Create output directory if it doesn't exist.
+ # Updated path to point to the parent folder's output.
+ output_dir = Path(__file__).resolve().parent.parent / "output"
+ output_dir.mkdir(exist_ok=True)
+
+ # Determine the file path based on uuid when not provided
+ if jsonl_path is None:
+ unique_id = uuid.uuid4().hex[:8]
+ jsonl_path = output_dir / f"word_ladders_{unique_id}.jsonl"
+ else:
+ jsonl_path = Path(jsonl_path)
+
+ target_size = config['dataset_config']['size']
+ current_size = 0
+ max_attempts = 3 # Limit total regeneration attempts
+ attempt = 0
+
+ # Initial generation
+ dataset = reasoning_gym.create_dataset(config['dataset_name'], **config['dataset_config'])
+ with open(jsonl_path, 'w', encoding='utf-8') as f:
+ for item in tqdm(dataset, desc="Generating initial ladder examples"):
+ row = {
+ 'question': item['question'],
+ 'answer': item['answer'],
+ 'reasoning': None,
+ 'metadata': item.get('metadata', {})
+ }
+ f.write(json.dumps(row) + '\n')
+
+ while attempt < max_attempts:
+ # Check entire file for duplicates
+ has_duplicates, valid_entries = check_duplicates(jsonl_path)
+ current_size = len(valid_entries)
+
+ if not has_duplicates and current_size == target_size:
+ print(f"\nSuccessfully created dataset with {current_size} unique examples.")
+ return
+
+ # If we have duplicates or not enough entries, regenerate the missing amount
+ needed = target_size - current_size
+ if needed > 0:
+ print(f"\nAttempt {attempt + 1}: Regenerating {needed} examples to replace duplicates/missing entries...")
+
+ # Generate additional examples
+ config['dataset_config']['size'] = needed
+ additional_dataset = reasoning_gym.create_dataset(config['dataset_name'], **config['dataset_config'])
+
+ # Write all entries to a temporary file
+ temp_path = jsonl_path.with_suffix('.tmp')
+ with open(temp_path, 'w', encoding='utf-8') as f:
+ # Write existing valid entries
+ for data in valid_entries.values():
+ f.write(json.dumps(data) + '\n')
+
+ # Write new entries
+ for item in additional_dataset:
+ row = {
+ 'question': item['question'],
+ 'answer': item['answer'],
+ 'reasoning': None,
+ 'metadata': item.get('metadata', {})
+ }
+ f.write(json.dumps(row) + '\n')
+
+ # Replace original file with temporary file
+ temp_path.replace(jsonl_path)
+
+ # Note: We'll check for duplicates again at the start of the next loop
+
+ attempt += 1
+
+ if current_size < target_size:
+ print(f"\nWarning: Could only generate {current_size} unique examples after {max_attempts} attempts.")
+ else:
+ print(f"\nSuccessfully created dataset with {current_size} unique examples.")
\ No newline at end of file
diff --git a/examples/word_ladder/utils/generate_reasoning.py b/examples/word_ladder/utils/generate_reasoning.py
new file mode 100644
index 00000000..e57412f7
--- /dev/null
+++ b/examples/word_ladder/utils/generate_reasoning.py
@@ -0,0 +1,211 @@
+"""
+generate_reasoning.py – Reads the JSONL file containing ladder examples,
+creates batch requests of chain-of-thought prompts split into batches of 2,500,
+calls Anthropic's Message Batches API for each batch, and writes separate batch metadata
+files for later retrieval of the responses.
+
+*** WARNING ***: Running large batches of requests via the Anthropic API (especially in generate_reasoning.py)
+can incur significant costs in Anthropic credits. Please review and understand your API quota and budgeting
+before running the API call. If you are testing or working with a demo dataset, adjust the batch size or dataset
+size appropriately to avoid unexpected charges.
+
+Using Anthropic's Message Batches API with caching enabled for system prompt.
+In our informal testing, Sonnet was deemed best performance value.
+You can swap out to another API, but this will need a rewrite to remove anthropic-specific code.
+"""
+
+import os
+import json
+import uuid
+import time
+from pathlib import Path
+
+from tqdm import tqdm
+
+import anthropic
+from anthropic.types.message_create_params import MessageCreateParamsNonStreaming
+from anthropic.types.messages.batch_create_params import Request
+
+# Updated default output directory to use the parent directory.
+DEFAULT_OUTPUT_DIR = Path(__file__).resolve().parent.parent / "output"
+
+# Add default constants at the top with other constants
+DEFAULT_INPUT_JSONL = "output/word_ladder_examples.jsonl"
+DEFAULT_SYSTEM_PROMPT = Path(__file__).resolve().parent.parent / "system_prompt.txt"
+BATCH_SIZE = 2500
+COMMON_UUID = uuid.uuid4().hex[:8]
+
+# Set up the Anthropic client (ensure the API key is set in the environment)
+client = anthropic.Anthropic(api_key=os.environ['ANTHROPIC_API_KEY'])
+
+def submit_reasoning_batches(
+ input_path: str = DEFAULT_INPUT_JSONL,
+ batch_metadata_prefix: str = "batch_metadata",
+ system_prompt_path: str = DEFAULT_SYSTEM_PROMPT
+) -> None:
+ """
+ Reads the input JSONL file of word ladder examples, builds batch requests for any example that
+ does not have reasoning, splits them into groups of BATCH_SIZE, and submits each batch using
+ Anthropic's Message Batches API.
+
+ Args:
+ input_path: Path to input JSONL file
+ batch_metadata_prefix: Prefix for batch metadata files
+ system_prompt_path: Path to system prompt file
+ """
+ # Create output directory if it doesn't exist
+ output_dir = DEFAULT_OUTPUT_DIR
+ output_dir.mkdir(exist_ok=True)
+
+ # Read the system prompt from file (used as a preamble for every request)
+ with open(system_prompt_path, "r", encoding="utf-8") as sys_file:
+ system_message = [{
+ "type": "text",
+ "text": sys_file.read(),
+ "cache_control": {"type": "ephemeral"} # Enable anthropic prompt caching
+ }]
+ batch_requests = []
+ custom_ids = [] # List of custom_ids for the current batch
+ batch_num = 0
+
+ # Get the total number of lines in advance for tqdm progress bar.
+ total_lines = sum(1 for _ in open(input_path))
+
+ with open(input_path, 'r', encoding="utf-8") as infile:
+ for idx, line in tqdm(enumerate(infile), desc="Preparing batch requests", total=total_lines):
+ data = json.loads(line)
+
+ # Skip example if 'reasoning' already exists.
+ if not data.get('reasoning'):
+ # Build a custom id. Here we use the row position and the start/end words:
+ metadata = data.get("metadata", {})
+ start = metadata.get("start_word", "unknown")
+ end = metadata.get("end_word", "unknown")
+ custom_id = f"{start}_{end}_{idx}"
+ custom_ids.append(custom_id)
+
+ # Build the prompt text exactly as before.
+ prompt = f"{data['question']}. The correct solution is {data['answer']}. "
+
+ # Build the request payload using Request and MessageCreateParamsNonStreaming.
+ request_payload = Request(
+ custom_id=custom_id,
+ params=MessageCreateParamsNonStreaming(
+ model="claude-3-5-sonnet-20241022", # Or choose the appropriate model version
+ max_tokens=8192,
+ temperature=0.5,
+ system=system_message,
+ messages=[
+ {"role": "user", "content": prompt}
+ ]
+ )
+ )
+ # Instead of wrapping in SimpleNamespace, simply ensure custom_id is set.
+ if isinstance(request_payload, dict):
+ request_payload["custom_id"] = custom_id
+ else:
+ request_payload.custom_id = custom_id
+ batch_requests.append(request_payload)
+
+ # If we have reached our batch size limit, submit the current batch.
+ if len(batch_requests) >= BATCH_SIZE:
+ _submit_single_batch(batch_requests, custom_ids, batch_num, batch_metadata_prefix, input_path)
+ batch_num += 1
+ # Reset for the next batch
+ batch_requests = []
+ custom_ids = []
+
+ # Submit any remaining requests that didn't complete a full batch.
+ if batch_requests:
+ _submit_single_batch(batch_requests, custom_ids, batch_num, batch_metadata_prefix, input_path)
+
+
+def _submit_single_batch(batch_requests, custom_ids, batch_num, batch_metadata_prefix, input_path):
+ """
+ Helper function to submit a single batch request, track the full API response,
+ and write out the corresponding metadata including the list of custom_ids.
+ """
+ # Use the default output directory
+ output_dir = DEFAULT_OUTPUT_DIR
+ output_dir.mkdir(exist_ok=True)
+
+ def serialize_datetime(dt):
+ """
+ Convert a datetime object to ISO formatted string.
+ If dt is None, returns None.
+ """
+ if dt is None:
+ return None
+ iso_str = dt.isoformat() # e.g. "2024-08-20T18:37:24.100435+00:00"
+
+ if dt.tzinfo is not None and dt.tzinfo.utcoffset(dt) is not None:
+ iso_str = iso_str.replace("+00:00", "Z")
+ return iso_str
+
+ def extract_custom_id(req):
+ # Safely extract the custom_id attribute whether req is an object or a dict.
+ return req.custom_id if hasattr(req, "custom_id") else req.get("custom_id")
+
+ max_attempts = 2
+ attempt = 0
+ last_exception = None
+ message_batch = None
+ while attempt < max_attempts:
+ try:
+ print(f"Submitting batch {batch_num} with {len(batch_requests)} requests... (attempt {attempt+1})")
+ message_batch = client.messages.batches.create(
+ requests=batch_requests
+ )
+ time.sleep(1)
+ print(f"Batch {batch_num} submitted with ID: {message_batch.id}")
+ break # Success: exit the loop.
+ except Exception as e:
+ last_exception = e
+ attempt += 1
+ print(f"Error submitting batch {batch_num} on attempt {attempt}: {e}")
+ if attempt < max_attempts:
+ print("Retrying...")
+ time.sleep(1)
+
+ if message_batch is None:
+ error_filename = output_dir / f"{COMMON_UUID}_failed_batches.jsonl"
+ error_msg = f"{str(last_exception)} after {max_attempts} attempts" if last_exception else f"Failed after {max_attempts} attempts"
+ failed_info = {
+ "batch_number": batch_num,
+ "error": error_msg,
+ "batch_requests": [extract_custom_id(req) for req in batch_requests],
+ "input_file": input_path,
+ }
+ with open(error_filename, 'a', encoding='utf-8') as error_file:
+ error_file.write(json.dumps(failed_info) + "\n")
+ print(f"Batch {batch_num} permanently failed. Logged to {error_filename}.")
+ return
+
+ # Build a dictionary of the expected response fields.
+ api_response = {
+ "id": message_batch.id,
+ "type": message_batch.type,
+ "processing_status": message_batch.processing_status,
+ "request_counts": vars(message_batch.request_counts),
+ "ended_at": serialize_datetime(message_batch.ended_at),
+ "created_at": serialize_datetime(message_batch.created_at),
+ "expires_at": serialize_datetime(message_batch.expires_at),
+ "cancel_initiated_at": serialize_datetime(message_batch.cancel_initiated_at),
+ "results_url": message_batch.results_url,
+ }
+
+ batch_metadata = {
+ "batch_id": message_batch.id,
+ "api_response": api_response,
+ "custom_ids": custom_ids,
+ "input_file": os.path.basename(input_path),
+ }
+ metadata_filename = output_dir / f"{COMMON_UUID}_{batch_metadata_prefix}.jsonl"
+ with open(metadata_filename, 'a', encoding='utf-8') as meta_file:
+ meta_file.write(json.dumps(batch_metadata) + "\n")
+
+ print(f"Batch metadata for batch {batch_num} appended to {metadata_filename}.")
+
+if __name__ == "__main__":
+ # When running this module directly, submit the reasoning batches.
+ submit_reasoning_batches()
diff --git a/examples/word_ladder/utils/usage_stats.py b/examples/word_ladder/utils/usage_stats.py
new file mode 100644
index 00000000..76a55382
--- /dev/null
+++ b/examples/word_ladder/utils/usage_stats.py
@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""
+This script reads a JSONL file that contains messages with usage statistics.
+For each JSON record, it expects to find the token usage information under:
+ record["result"]["message"]["usage"]
+
+It then calculates and prints statistics for each usage token field:
+ - input_tokens
+ - cache_creation_input_tokens
+ - cache_read_input_tokens
+ - output_tokens
++pricing calculations
++calculates the savings from caching (vs if we hadn't done any caching)
++forecasts costs for 10,000, 20,000 and 50,000 jobs based on tokens per query
+
+Usage:
+ python usage_stats.py path/to/msgbatch_01X9LgZNVkLFhzrrBd9LNgWb_results.jsonl
+"""
+
+import json
+import argparse
+from statistics import mean
+
+def main():
+ parser = argparse.ArgumentParser(
+ description="Compute usage token statistics from a JSONL file."
+ )
+ parser.add_argument(
+ "file", help="Path to the JSONL file containing usage token data."
+ )
+ args = parser.parse_args()
+
+ # Usage token fields that we want to track
+ usage_fields = [
+ "input_tokens",
+ "cache_creation_input_tokens",
+ "cache_read_input_tokens",
+ "output_tokens",
+ ]
+
+ # Pricing for Sonnet, 2 Feb 2025
+ base_input_rate = 1.50
+ pricing = {
+ "input_tokens": base_input_rate,
+ "cache_creation_input_tokens": base_input_rate * 1.25, # More expensive for initial computation
+ "cache_read_input_tokens": base_input_rate * 0.1, # Cheaper for cache-read tokens
+ "output_tokens": 7.50,
+ }
+
+ # A dictionary to store lists of values for each usage field
+ usage_data = {key: [] for key in usage_fields}
+ total_lines = 0
+ error_count = 0
+
+ with open(args.file, "r", encoding="utf-8") as f:
+ for line in f:
+ total_lines += 1
+ try:
+ record = json.loads(line)
+ except json.JSONDecodeError:
+ print(f"[Warning] Failed to parse JSON on line {total_lines}.")
+ error_count += 1
+ continue
+
+ # Navigate to the usage stats
+ try:
+ usage = record["result"]["message"]["usage"]
+ except KeyError:
+ print(f"[Warning] Missing usage field in line {total_lines}.")
+ error_count += 1
+ continue
+
+ # Extract token values from the usage data
+ for key in usage_fields:
+ # Defaulting to 0 if the token field is missing or non-numeric
+ try:
+ token_value = int(usage.get(key, 0))
+ except (ValueError, TypeError):
+ token_value = 0
+ usage_data[key].append(token_value)
+
+ print(f"\nProcessed {total_lines} lines with {error_count} error(s).\n")
+ print("Usage Tokens Statistics:")
+ print("-" * 40)
+
+ grand_total_cost = 0.0
+ # Calculate and print stats for each token type
+ for key in usage_fields:
+ values = usage_data[key]
+ if values:
+ total = sum(values)
+ count = len(values)
+ min_val = min(values)
+ max_val = max(values)
+ avg = mean(values)
+ # Calculate pricing cost scaling by tokens per million
+ cost = total / 1_000_000 * pricing[key]
+ grand_total_cost += cost
+
+ print(f"{key}:")
+ print(f" Total = {total}")
+ print(f" Count = {count}")
+ print(f" Min = {min_val}")
+ print(f" Max = {max_val}")
+ print(f" Mean = {avg:.2f}")
+ print(f" Cost = ${cost:.2f}\n")
+ else:
+ print(f"{key}: No data found.\n")
+
+ print("-" * 40)
+ print(f"Grand Total Estimated Cost: ${grand_total_cost:.2f}")
+
+ # -----------------------------------------------
+ # Calculate caching savings (for input-related tokens)
+ # Without caching, all tokens would have been charged at the standard input rate.
+ #
+ # Baseline cost (if no caching were used):
+ # = (input_tokens + cache_creation_input_tokens + cache_read_input_tokens)
+ # / 1_000_000 * base_input_rate
+ #
+ # Actual cost (with caching):
+ # = input_tokens * base_input_rate +
+ # cache_creation_input_tokens * (base_input_rate * 1.25) +
+ # cache_read_input_tokens * (base_input_rate * 0.1)
+ #
+ # Savings from caching is then the difference.
+ sum_input = sum(usage_data["input_tokens"])
+ sum_cache_creation = sum(usage_data["cache_creation_input_tokens"])
+ sum_cache_read = sum(usage_data["cache_read_input_tokens"])
+
+ baseline_input_cost = (sum_input + sum_cache_creation + sum_cache_read) / 1_000_000 * pricing["input_tokens"]
+ actual_input_cost = (sum_input) / 1_000_000 * pricing["input_tokens"] \
+ + (sum_cache_creation) / 1_000_000 * pricing["cache_creation_input_tokens"] \
+ + (sum_cache_read) / 1_000_000 * pricing["cache_read_input_tokens"]
+ caching_savings = baseline_input_cost - actual_input_cost
+
+ print(f"Caching Savings (input-related tokens): ${caching_savings:.2f}")
+
+ # -----------------------------------------------
+ # Forecast future cost estimates based on the average tokens per job.
+ #
+ # We'll compute the average tokens per job (i.e. tokens per query) for:
+ # - input_tokens
+ # - cache_creation_input_tokens
+ # - cache_read_input_tokens
+ # - output_tokens
+ #
+ # Then we forecast, for example, for 10,000, 20,000, and 50,000 jobs:
+ # - Apply the relevant pricing to compute the cost per token type.
+ # - Also compute the baseline cost for input-related tokens and the savings
+ # from caching.
+ if usage_data["input_tokens"]:
+ job_count = len(usage_data["input_tokens"])
+ avg_input_tokens = sum(usage_data["input_tokens"]) / job_count
+ avg_cache_creation_tokens = sum(usage_data["cache_creation_input_tokens"]) / job_count
+ avg_cache_read_tokens = sum(usage_data["cache_read_input_tokens"]) / job_count
+ avg_output_tokens = sum(usage_data["output_tokens"]) / job_count
+
+ print("\nAverage Tokens per Job:")
+ print(f" input_tokens = {avg_input_tokens:.2f}")
+ print(f" cache_creation_input_tokens = {avg_cache_creation_tokens:.2f}")
+ print(f" cache_read_input_tokens = {avg_cache_read_tokens:.2f}")
+ print(f" output_tokens = {avg_output_tokens:.2f}")
+
+ forecast_jobs = [2000, 4000, 10000, 20000, 50000]
+ print("\nForecasting Future Job Costs:")
+ for jobs in forecast_jobs:
+ # Forecast token usage for the job count by multiplying the per-job averages.
+ forecast_input = avg_input_tokens * jobs
+ forecast_cache_creation = avg_cache_creation_tokens * jobs
+ forecast_cache_read = avg_cache_read_tokens * jobs
+ forecast_output = avg_output_tokens * jobs
+
+ # Forecast actual cost (with caching applied for input tokens):
+ actual_input_cost_forecast = (forecast_input) / 1_000_000 * pricing["input_tokens"] \
+ + (forecast_cache_creation) / 1_000_000 * pricing["cache_creation_input_tokens"] \
+ + (forecast_cache_read) / 1_000_000 * pricing["cache_read_input_tokens"]
+
+ # Without caching, all input-related tokens would be at base_input_rate:
+ baseline_input_cost_forecast = (forecast_input + forecast_cache_creation + forecast_cache_read) / 1_000_000 * pricing["input_tokens"]
+
+ caching_savings_forecast = baseline_input_cost_forecast - actual_input_cost_forecast
+
+ forecast_output_cost = forecast_output / 1_000_000 * pricing["output_tokens"]
+ total_forecast_cost = actual_input_cost_forecast + forecast_output_cost
+
+ print(f"\nFor {jobs:,} jobs:")
+ print(" Forecasted Token Usage:")
+ print(f" input_tokens = {forecast_input:,.0f}")
+ print(f" cache_creation_input_tokens = {forecast_cache_creation:,.0f}")
+ print(f" cache_read_input_tokens = {forecast_cache_read:,.0f}")
+ print(f" output_tokens = {forecast_output:,.0f}")
+ print(" Estimated Costs:")
+ print(f" Input Cost (with caching) = ${actual_input_cost_forecast:,.2f}")
+ print(f" Output Cost = ${forecast_output_cost:,.2f}")
+ print(f" Grand Total Cost = ${total_forecast_cost:,.2f}")
+ print(f" Caching Savings (input) = ${caching_savings_forecast:,.2f}")
+ else:
+ print("No valid jobs to forecast future costs.")
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file