import asyncio import csv import inspect import math import re from dataclasses import dataclass from typing import Callable, Dict, Optional, Tuple from asteval import Interpreter from games import ( card_matching_game_2, card_matching_game_3, card_matching_game_4, easy_game_1, easy_game_2, easy_game_3, easy_game_4, odd_card_game, ) from openai import AsyncOpenAI GUIDELINES = """ Please provide your analysis using the exact format below, including all tags: [Your initial approach to solving this probability problem] [List important observations about the game mechanics] [Show your step-by-step mathematical derivation using probability theory] [Include explanations of any combinations, permutations, or conditional probabilities used] [IMPORTANT: Write ONLY the final, simplified mathematical formula for the probability of winning below.] [CRITICAL: Do NOT include any text, explanations, comments, multiple formulas, or intermediate calculation steps within this tag.] [CRITICAL: If a precise mathematical formula cannot be determined, leave this section EMPTY.] [Use C(n,r), P(n,r), factorial(n) and standard math operators: + - * / ^ ( ) ] Note: Use these notations ONLY in your formula: - Factorial: factorial(n) - Combinations: C(n,r) - Permutations: P(n,r) - Standard operators: *, /, +, -, ^, (, ) The formula must be in a format that can be directly evaluated. Use parentheses liberally to ensure correct order of operations. For example, write (A * B) / (C * D) instead of A * B / C * D if you intend the division to apply to the result of (C * D). Be explicit! What is the mathematical formula to calculate the exact probability of winning this game? """ @dataclass class GameAnalysis: """Class to hold the analysis results of a game.""" ai_analysis: str formula: Optional[str] calculated_probability: Optional[float] simulated_probability: float n_simulations: int probability_difference: Optional[float] class GamePredictor: def __init__( self, openai_api_key: str, openai_api_base: str, model: str = "llama-4-maverick-17b-128e-instruct-fp8", ): """Initialize the GamePredictor with OpenAI API credentials.""" self.client = AsyncOpenAI( api_key=openai_api_key, base_url=openai_api_base, ) self.model = model # Create a persistent asteval interpreter self.aeval = Interpreter() # Add math functions to the interpreter's symbol table self.aeval.symtable["factorial"] = self.factorial self.aeval.symtable["C"] = self.combination self.aeval.symtable["P"] = self.permutation # Add standard math functions if needed (optional, asteval includes many) # self.aeval.symtable['sqrt'] = math.sqrt # self.aeval.symtable['pow'] = math.pow @staticmethod def factorial(n: int) -> int: """Calculate factorial.""" return math.factorial(n) @staticmethod def combination(n: int, r: int) -> int: """Calculate combination nCr.""" return math.factorial(n) // (math.factorial(r) * math.factorial(n - r)) @staticmethod def permutation(n: int, r: int) -> int: """Calculate permutation nPr.""" return math.factorial(n) // math.factorial(n - r) def _extract_formula(self, response_text: str) -> Optional[str]: """Extract formula from the AI response.""" # Find all formula blocks formula_matches = re.findall( r"\n(.*?)\n", response_text, re.DOTALL ) if not formula_matches: return None # Use the content of the last formula block found last_formula_content = formula_matches[-1].strip() if not last_formula_content: return None # Split the content into lines and filter out empty lines lines = [ line.strip() for line in last_formula_content.split("\\n") if line.strip() ] if not lines: return None # Return the last non-empty line as the potential formula # This assumes the AI puts the final, clean formula last in the block return lines[-1] def _evaluate_formula(self, formula: str) -> float: """Evaluate the mathematical formula using asteval.""" # No need for regex replacements if the AI uses C(), P(), factorial() try: # Evaluate the formula using the pre-configured interpreter result = self.aeval(formula) if isinstance(result, (int, float)): return float(result) else: # Explicitly handle non-numeric results raise ValueError( f"Formula '{formula}' evaluated to non-numeric type: {type(result).__name__} ({result})" ) except KeyError as e: # Handle cases where symbols are not found (e.g., undefined variables in formula) raise ValueError( f"Error evaluating formula '{formula}': Undefined symbol {e}" ) except Exception as e: # Catch other potential evaluation errors from asteval raise ValueError(f"Error evaluating formula '{formula}' using asteval: {e}") def _create_prompt(self, game_func: Callable) -> str: """Create the prompt for the AI model.""" # Get the function's source code and docstring source_code = inspect.getsource(game_func) description = game_func.__doc__ or "No description available." return f""" Analyze this game implemented in the following Python code: ```python {source_code} ``` {description} """ def simulate_game(self, game_func: Callable, n_simulations: int = 100000) -> float: """ Simulate a game multiple times and return the win probability. Args: game_func: The game function to simulate n_simulations: Number of simulations to run Returns: float: The probability of winning the game based on simulation """ wins = sum(1 for _ in range(n_simulations) if game_func()) return wins / n_simulations def compare_probabilities( self, calculated: float, simulated: float ) -> Tuple[float, str]: """ Compare calculated and simulated probabilities. Args: calculated: The probability calculated from the formula simulated: The probability obtained from simulation Returns: Tuple[float, str]: The absolute difference and a qualitative assessment """ diff = abs(calculated - simulated) if diff < 0.01: assessment = "Excellent match between theory and simulation" elif diff < 0.05: assessment = "Good match between theory and simulation" elif diff < 0.1: assessment = "Fair match between theory and simulation" else: assessment = "Poor match between theory and simulation" return diff, assessment async def predict_game( self, game_func: Callable, n_simulations: int = 100000 ) -> GameAnalysis: """ Predict the probability of winning a game using both AI analysis and simulation. Args: game_func: Function that implements the game n_simulations: Number of simulations to run for verification Returns: GameAnalysis object containing all analysis results """ # Create and send message to AI message = { "role": "user", "content": [ { "type": "text", "text": self._create_prompt(game_func) + "\n\n" + GUIDELINES, }, ], } try: chat_response = await self.client.chat.completions.create( model=self.model, messages=[message], temperature=0, # Set temperature to 0 for deterministic output ) response_text = chat_response.choices[0].message.content except Exception as e: # Handle potential API errors gracefully response_text = f"API call failed: {e}" # Consider logging the error here print(f"Warning: API call failed for a game: {e}") # Extract and evaluate formula formula = self._extract_formula(response_text) calculated_prob = None if formula: try: calculated_prob = self._evaluate_formula(formula) except ValueError as e: # Formula evaluation failed, set probability to None and maybe log/store the error calculated_prob = None # Optionally add error information to the analysis object if needed print(f"Warning: Could not evaluate formula '{formula}': {e}") # Update response_text or add a field to GameAnalysis if needed response_text += f"Formula evaluation failed: {e}" # Run simulation in a separate thread to avoid blocking the event loop simulated_prob = await asyncio.to_thread( self.simulate_game, game_func, n_simulations ) # Calculate difference if both probabilities are available prob_diff = ( abs(calculated_prob - simulated_prob) if calculated_prob is not None else None ) return GameAnalysis( ai_analysis=response_text, formula=formula, calculated_probability=calculated_prob, simulated_probability=simulated_prob, n_simulations=n_simulations, probability_difference=prob_diff, ) async def predict_games( self, games: Dict[str, Callable], n_simulations: int = 100000 ) -> Dict[str, GameAnalysis]: """ Predict probabilities for multiple games concurrently. Args: games: Dictionary mapping game names to game functions n_simulations: Number of simulations per game Returns: Dictionary mapping game names to their GameAnalysis results """ # Create tasks for each game prediction tasks = { game_name: asyncio.create_task(self.predict_game(game_func, n_simulations)) for game_name, game_func in games.items() } # Wait for all tasks to complete await asyncio.gather(*tasks.values()) # Collect results results = {name: task.result() for name, task in tasks.items()} return results async def generate_qa_csv( self, games: Dict[str, Callable], n_simulations: int, csv_filepath: str ): """ Generates a CSV file with questions (prompts) and answers (simulated probabilities). Args: games: Dictionary mapping game names to game functions. n_simulations: Number of simulations per game. csv_filepath: Path to save the CSV file. """ qa_data = [] # Create a list of tasks for simulation to run them concurrently if desired, # or simply iterate and await if sequential processing per game is fine. # For simplicity here, we'll process game simulations sequentially for prompt generation, # but the simulation itself runs in a thread. for game_name, game_func in games.items(): print(f"Processing game for CSV: {game_name}") prompt = self._create_prompt(game_func) # simulate_game is synchronous, run it in a thread to avoid blocking simulated_prob = await asyncio.to_thread( self.simulate_game, game_func, n_simulations ) answer = f"{simulated_prob:.6f}" # Format probability as string qa_data.append({"question": prompt, "answer": answer}) try: with open(csv_filepath, "w", newline="", encoding="utf-8") as csvfile: fieldnames = ["question", "answer"] writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() for row_data in qa_data: writer.writerow(row_data) print(f"Successfully generated Q&A CSV at {csv_filepath}") except IOError as e: print(f"Error writing CSV file {csv_filepath}: {e}") except Exception as e: print(f"An unexpected error occurred during CSV generation: {e}") # Example usage: async def main(): # API credentials - Set these as environment variables or pass as parameters openai_api_key = "your_openai_api_key_here" openai_api_base = "https://api.lambda.ai/v1" # Create predictor instance predictor = GamePredictor(openai_api_key, openai_api_base) # Define games to analyze games = { "easy_game_1": easy_game_1, "easy_game_2": easy_game_2, "easy_game_3": easy_game_3, "easy_game_4": easy_game_4, "card_matching_2": card_matching_game_2, "card_matching_3": card_matching_game_3, "card_matching_4": card_matching_game_4, "odd_card": odd_card_game, } await predictor.generate_qa_csv( games, 100000, "environments/community/solitaire_winning_probability/qa_data.csv", ) # Get predictions for all games results = await predictor.predict_games(games) # Print results for game_name, analysis in results.items(): print(f"\nResults for {game_name}:") # print("AI Analysis:") # print(analysis.ai_analysis) # print(f"\nFormula: {analysis.formula}") # # Handle potential None for calculated probability # if analysis.calculated_probability is not None: # print(f"Calculated probability: {analysis.calculated_probability:.4f}") # else: # print("Calculated probability: N/A (Formula missing or invalid)") print(f"Simulated probability: {analysis.simulated_probability:.4f}") # Compare only if calculated probability is available if analysis.calculated_probability is not None: diff, assessment = predictor.compare_probabilities( analysis.calculated_probability, analysis.simulated_probability ) print(f"Probability difference: {diff:.4f}") print(f"Assessment: {assessment}") else: print("Probability difference: N/A") print("Assessment: N/A (Cannot compare without calculated probability)") if __name__ == "__main__": asyncio.run(main())