diff --git a/README.md b/README.md index 0062e9d..bd62266 100644 --- a/README.md +++ b/README.md @@ -290,6 +290,33 @@ The game JSON includes special fields for AI analysis: - `agent_relationships` - Diplomatic standings at each phase - `final_agent_states` - End-game goals and relationships +### Data Processing and RL Analysis Pipeline + +For detailed analysis of LLM interactions and order success rates, a two-step pipeline is used: + +1. **Convert CSV to RL JSON**: + The `csv_to_rl_json.py` script processes `llm_responses.csv` files, typically found in game-specific subdirectories ending with "FULL_GAME" (e.g., `results/20250524_..._FULL_GAME/`). It converts this raw interaction data into a JSON format suitable for Reinforcement Learning (RL) analysis. + + To process all relevant CSVs in batch: + ```bash + python csv_to_rl_json.py --scan_dir results/ + ``` + This command scans the `results/` directory for "FULL_GAME" subfolders, converts their `llm_responses.csv` files, and outputs all generated `*_rl.json` files into the `results/json/` directory. + +2. **Analyze RL JSON Files**: + The `analyze_rl_json.py` script then analyzes the JSON files generated in the previous step. It aggregates statistics on successful and failed convoy and support orders, categorized by model. + + To run the analysis: + ```bash + python analyze_rl_json.py results/json/ + ``` + This command processes all `*_rl.json` files in the `results/json/` directory and generates two reports in the project's root directory: + - `analysis_summary.txt`: A clean summary of order statistics. + - `analysis_summary_debug.txt`: A detailed report including unique 'success' field values and other debug information. + +This pipeline allows for a comprehensive understanding of LLM performance in generating valid and successful game orders. + + ### Post-Game Analysis Tools #### Strategic Moment Analysis diff --git a/analyze_rl_json.py b/analyze_rl_json.py index 912a950..51bcc63 100644 --- a/analyze_rl_json.py +++ b/analyze_rl_json.py @@ -10,35 +10,107 @@ import re # logging.basicConfig(level=logging.DEBUG) # Removed for more specific config +def strip_json_comments(json_string_with_comments): + """ + Removes // style comments from a JSON string. + Handles comments at the end of lines. + Does not handle block comments or // within string literals. + """ + if not isinstance(json_string_with_comments, str): + # If it's not a string (e.g., already parsed list/dict, or None), return as is + return json_string_with_comments + + lines = json_string_with_comments.splitlines() + cleaned_lines = [] + for line in lines: + stripped_line = line.split('//', 1)[0].rstrip() + cleaned_lines.append(stripped_line) + return "\n".join(cleaned_lines) + def extract_orders_from_llm_response(llm_response_content, model_name_for_logging="UNKNOWN_MODEL"): """ Extracts a list of order strings from various formats of llm_response_content. Handles direct lists, JSON strings, and strings with embedded "PARSABLE OUTPUT:" JSON blocks. """ orders = [] - processed_content = "" + processed_content = "" # Initialize to empty string if isinstance(llm_response_content, list): - processed_content = "\n".join(str(item) for item in llm_response_content) - logging.debug(f"Model {model_name_for_logging}: Joined list llm_response_content into single string for parsing.") + # If it's already a list, assume it's a list of orders (strings) + # Ensure all items are strings and strip them + if all(isinstance(order, str) for order in llm_response_content): + logging.debug(f"Model {model_name_for_logging}: llm_response_content is a list of strings. Using directly.") + return [order.strip() for order in llm_response_content if order.strip()] + else: + # If list contains non-strings, try to convert to string for parsing, similar to initial design + logging.debug(f"Model {model_name_for_logging}: llm_response_content is a list with non-string items. Converting to string for parsing.") + processed_content = "\n".join(str(item) for item in llm_response_content) elif isinstance(llm_response_content, str): processed_content = llm_response_content + elif isinstance(llm_response_content, dict): + logging.debug(f"Model {model_name_for_logging}: llm_response_content is a dict. Checking for 'orders' key.") + # Case 1: Dictionary contains a direct 'orders' list with strings. + if "orders" in llm_response_content and isinstance(llm_response_content["orders"], list): + potential_orders = llm_response_content["orders"] + if all(isinstance(order, str) for order in potential_orders): + logging.info(f"Model {model_name_for_logging}: Extracted orders directly from 'orders' key in dict llm_response.") + return [order.strip() for order in potential_orders if order.strip()] + else: + # 'orders' key is a list but not all items are strings. + logging.warning(f"Model {model_name_for_logging}: 'orders' key in dict llm_response is a list, but not all items are strings. Content: {str(potential_orders)[:200]}") + # Fallback: Serialize the entire dictionary to a string for further parsing attempts. + try: + processed_content = json.dumps(llm_response_content) + logging.debug(f"Model {model_name_for_logging}: Serialized dict (with non-string orders list) to string for further parsing.") + except TypeError as te: + logging.warning(f"Model {model_name_for_logging}: Could not serialize dict (with non-string orders list) to JSON string: {te}. Cannot extract orders.") + return [] # Return empty list as we can't process this dict further. + else: + # Case 2: Dictionary does not have an 'orders' list or 'orders' is not a list. + logging.debug(f"Model {model_name_for_logging}: llm_response_content is a dict but no direct 'orders' list found or 'orders' is not a list. Dict (first 300 chars): {str(llm_response_content)[:300]}.") + # Fallback: Serialize the entire dictionary to a string for further parsing attempts. + try: + processed_content = json.dumps(llm_response_content) + logging.debug(f"Model {model_name_for_logging}: Serialized dict (no direct orders list) to string for further parsing.") + except TypeError as te: + logging.warning(f"Model {model_name_for_logging}: Could not serialize dict (no direct orders list) to JSON string: {te}. Cannot extract orders.") + return [] # Return empty list as we can't process this dict further. else: - logging.warning(f"Model {model_name_for_logging}: llm_response_content is not a list or string, but {type(llm_response_content)}. Cannot extract orders.") - return [] + # llm_response_content is not a list, string, or dict (e.g., float, None). + logging.warning(f"Model {model_name_for_logging}: llm_response_content is type {type(llm_response_content)}, not list/string/dict. Cannot extract orders. Content: {str(llm_response_content)[:200]}") + return [] # Return empty list as this type is not processable for orders. - # Attempt to parse "PARSABLE OUTPUT:" block first + # At this point, 'processed_content' should be a string derived from llm_response_content + # (unless an early return occurred for direct list/dict order extraction or unhandled type). + # If 'processed_content' is empty or only whitespace, no further parsing is useful. + if not processed_content.strip(): + logging.debug(f"Model {model_name_for_logging}: llm_response_content resulted in empty or whitespace-only processed_content. No orders to extract via string parsing.") + return [] # orders is already [], just return + + # Attempt to parse "PARSABLE OUTPUT:" block first from 'processed_content' match_parsable = re.search(r"PARSABLE OUTPUT:\s*(?:\{\{)?\s*\"orders\"\s*:\s*(\[.*?\])\s*(?:\}\})?", processed_content, re.IGNORECASE | re.DOTALL) if match_parsable: orders_json_str = match_parsable.group(1) try: - parsed_orders = json.loads(orders_json_str) - if isinstance(parsed_orders, list): - orders = [str(o).strip() for o in parsed_orders if str(o).strip()] - logging.debug(f"Model {model_name_for_logging}: Extracted orders from 'PARSABLE OUTPUT:' block: {orders}") - return orders - except json.JSONDecodeError as e: - logging.warning(f"Model {model_name_for_logging}: Found 'PARSABLE OUTPUT:' but failed to parse orders JSON: {orders_json_str}. Error: {e}") + content_to_parse = orders_json_str + stripped_json_text = "[Stripping not attempted or failed before assignment]" # Initialize placeholder + stripped_json_text = strip_json_comments(content_to_parse) + orders_list = json.loads(stripped_json_text) + if isinstance(orders_list, list): + # Ensure all items are strings, as expected for orders + if all(isinstance(order, str) for order in orders_list): + orders = [str(o).strip() for o in orders_list if str(o).strip()] + logging.debug(f"Model {model_name_for_logging}: Extracted orders from 'PARSABLE OUTPUT:' block: {orders}") + return orders + else: + logging.warning(f"Model {model_name_for_logging}: Parsed JSON from 'PARSABLE OUTPUT:' but not all items are strings: {orders_list}") + else: + logging.warning(f"Model {model_name_for_logging}: Parsed JSON from 'PARSABLE OUTPUT:' but it's not a list: {type(orders_list)}") + except json.JSONDecodeError as e_direct: + # Log original and stripped content for better debugging + logging.warning(f"Model {model_name_for_logging}: Failed to parse JSON from 'PARSABLE OUTPUT:'. Error: {e_direct}. Original (first 300): '{content_to_parse[:300]}'. Stripped (first 300): '{stripped_json_text[:300]}'") + except Exception as e_unexpected: + logging.error(f"Model {model_name_for_logging}: Unexpected error parsing 'PARSABLE OUTPUT:' JSON. Error: {e_unexpected}. Original (first 300): '{content_to_parse[:300]}'. Stripped (first 300): '{stripped_json_text[:300]}'") # If not found via "PARSABLE OUTPUT:", attempt to parse the whole content as JSON try: @@ -82,251 +154,310 @@ def extract_orders_from_llm_response(llm_response_content, model_name_for_loggin return [] -def analyze_json_files(json_directory, output_file_path): - # Configure logging to file and console - logger = logging.getLogger() - logger.setLevel(logging.DEBUG) +import sys +import traceback +import pandas as pd # Ensure pandas is imported at the top - # Clear existing handlers to prevent duplicate logs if function is called multiple times - if logger.hasHandlers(): - logger.handlers.clear() - - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - - # File handler - writes DEBUG and higher to the specified output file - # Using mode 'w' to overwrite the log file for each analysis run - fh = logging.FileHandler(output_file_path, mode='w') - fh.setLevel(logging.DEBUG) - fh.setFormatter(formatter) - logger.addHandler(fh) - - # Stream handler - writes INFO and higher to console (stdout) - sh = logging.StreamHandler(sys.stdout) - sh.setLevel(logging.INFO) # Console can be less verbose - sh.setFormatter(formatter) - logger.addHandler(sh) - - logging.info(f"analyze_json_files starting. JSON Directory: '{json_directory}', Output File: '{output_file_path}'") - +def _perform_analysis(json_directory): + logging.info(f"Starting analysis of JSON directory: {json_directory}") total_rows = 0 total_characters = 0 response_type_stats = defaultdict(lambda: {'prompt_chars': [], 'response_chars': []}) - model_order_stats = defaultdict(lambda: {'successful_convoys': 0, 'successful_supports': 0, 'total_successful_orders_processed': 0}) - - outfile = None # Initialize outfile to None - try: - # Manually open the file - outfile = open(output_file_path, 'w') - print(f"DEBUG: Output file '{output_file_path}' opened successfully for writing.") - outfile.write(f"Analysis script started. Outputting to: {output_file_path}\n") - outfile.write(f"Analyzing JSON files from directory: {json_directory}\n") - outfile.flush() + model_order_stats = defaultdict(lambda: { + 'successful_convoys': 0, + 'successful_supports': 0, + 'total_successful_orders_processed': 0, + 'failed_convoys': 0, + 'failed_supports': 0, + 'total_failed_order_sets_processed': 0 + }) + all_success_values = set() + json_files_processed = 0 - if not os.path.isdir(json_directory): - err_msg_dir = f"Error: Directory not found: {json_directory}\n" - outfile.write(err_msg_dir) - print(err_msg_dir.strip()) - return # Return will trigger the 'finally' block + if not os.path.isdir(json_directory): + logging.error(f"Error: Directory not found: {json_directory}") + # Return empty/default stats if directory not found + return { + "total_rows": 0, "total_characters": 0, "response_type_stats": defaultdict(lambda: {'prompt_chars': [], 'response_chars': []}), + "model_order_stats": defaultdict(lambda: {'successful_convoys': 0, 'successful_supports': 0, 'total_successful_orders_processed': 0, 'failed_convoys': 0, 'failed_supports': 0, 'total_failed_order_sets_processed': 0}), + "all_success_values": set(), "json_files_processed": 0, "error": f"Directory not found: {json_directory}" + } - json_files_processed = 0 - for filename in os.listdir(json_directory): - if filename.endswith("_rl.json"): - file_path = os.path.join(json_directory, filename) - outfile.write(f"Processing file: {file_path}...\n") - outfile.flush() - print(f"Processing file: {file_path}...") - try: - file_size = os.path.getsize(file_path) - total_characters += file_size - json_files_processed += 1 + for filename in os.listdir(json_directory): + if filename.endswith("_rl.json"): + file_path = os.path.join(json_directory, filename) + logging.info(f"Processing file: {file_path}...") + try: + file_size = os.path.getsize(file_path) + total_characters += file_size + json_files_processed += 1 - with open(file_path, 'r') as f: - data = json.load(f) - - if not isinstance(data, list): - warning_msg = f" Warning: Expected a list of objects in {filename}, got {type(data)}. Skipping file.\n" - outfile.write(warning_msg) - print(warning_msg.strip()) - continue - - total_rows += len(data) + with open(file_path, 'r') as f: + data = json.load(f) + + if not isinstance(data, list): + logging.warning(f" Warning: Expected a list of objects in {filename}, got {type(data)}. Skipping file.") + continue + + total_rows += len(data) - for entry in data: - response_type = entry.get('response_type', "UNKNOWN_RESPONSE_TYPE") - prompt_content = entry.get('prompt') - llm_response_content = entry.get('llm_response') + for entry in data: + response_type = entry.get('response_type', "UNKNOWN_RESPONSE_TYPE") + prompt_content = entry.get('prompt') + llm_response_content = entry.get('llm_response') - # Section for parsing successful orders by model - if response_type == "order_generation": - model = entry.get('model', 'UNKNOWN_MODEL') - success = entry.get('success') # Expected to be boolean True/False or None - - # Check for boolean True or string 'Success' (case-insensitive) - is_successful_order_set = False - if success is True: - is_successful_order_set = True - elif isinstance(success, str) and success.lower() == 'success': - is_successful_order_set = True - # Add other string variations of success if needed, e.g., 'succeeded' - # elif isinstance(success, str) and success.lower() == 'succeeded': - # is_successful_order_set = True - - if is_successful_order_set and llm_response_content is not None: - orders_list = extract_orders_from_llm_response(llm_response_content, model) - - if orders_list: # Only proceed if orders were actually extracted - model_order_stats[model]['total_successful_orders_processed'] += 1 - - current_set_has_convoy = False - current_set_has_support = False - for order_str in orders_list: - parts = order_str.upper().split() - # Convoy: F ENG C A LVP - BEL - if len(parts) == 7 and parts[0] == 'F' and parts[2] == 'C' and parts[3] == 'A' and parts[5] == '-': - model_order_stats[model]['successful_convoys'] += 1 - logging.debug(f"Found convoy: {order_str} for model {model}") - current_set_has_convoy = True - # Updated Support order detection - elif len(parts) >= 4 and parts[2] == 'S' and parts[0] in ['A', 'F'] and parts[3] in ['A', 'F']: # Basic check for S and unit types - is_support = False - support_type = "unknown" - # Case 1: Implicit Support Hold (5 parts) e.g., F ENG S F NTH - if len(parts) == 5: - is_support = True - support_type = "implicit hold" - # Case 2: Explicit Support Hold (6 parts) e.g., F ENG S F NTH H - elif len(parts) == 6 and parts[5] == 'H': - is_support = True - support_type = "explicit hold" - # Case 3: Support Move (7 parts) e.g., F ENG S F NTH - BEL - elif len(parts) == 7 and parts[5] == '-': - if len(parts[6]) > 0: # Basic check for destination - is_support = True - support_type = "move" - - if is_support: - model_order_stats[model]['successful_supports'] += 1 - logging.debug(f"Found support ({support_type}): {order_str} for model {model}") - current_set_has_support = True - - if not current_set_has_convoy and not current_set_has_support: # Check if still no C/S after parsing this non-empty list - logging.debug(f"Model {model}: Successful order set (total {len(orders_list)} parsed orders) had no C/S. Parsed Orders: {orders_list}") - else: # else for 'if orders_list:' (i.e., orders_list is empty) - logging.debug(f"Model {model}: Successful order set, but no orders extracted by helper. LLM Response (first 300 chars): {str(llm_response_content)[:300]}") - - if prompt_content is not None and isinstance(prompt_content, str): - response_type_stats[response_type]['prompt_chars'].append(len(prompt_content)) + if response_type == "order_generation": + model = entry.get('model', 'UNKNOWN_MODEL') + success = entry.get('success') + is_successful_order_set = False + if success is True or (isinstance(success, str) and success.lower() == 'success'): + is_successful_order_set = True - if llm_response_content is not None: - if isinstance(llm_response_content, str): - response_type_stats[response_type]['response_chars'].append(len(llm_response_content)) - else: - try: - response_str = json.dumps(llm_response_content) - response_type_stats[response_type]['response_chars'].append(len(response_str)) - except TypeError: - warning_msg_ser = f" Warning: Could not serialize llm_response in {filename}.\n" - outfile.write(warning_msg_ser) - print(warning_msg_ser.strip()) - except json.JSONDecodeError: - warning_msg_json = f" Warning: Could not decode JSON from {filename}. Skipping file.\n" - outfile.write(warning_msg_json) - print(warning_msg_json.strip()) - except Exception as e: - warning_msg_exc = f" Warning: An error occurred processing {filename}: {e}. Skipping file.\n" - outfile.write(warning_msg_exc) - print(warning_msg_exc.strip()) + all_success_values.add(entry.get('success')) # Collect all success values - if json_files_processed == 0: - no_files_msg = f"No '*_rl.json' files found in {json_directory}.\n" - outfile.write(no_files_msg) - print(no_files_msg.strip()) - return + if is_successful_order_set and llm_response_content is not None: + model_order_stats[model]['total_successful_orders_processed'] += 1 + orders_list = extract_orders_from_llm_response(llm_response_content, model) + if orders_list: + for order_str in orders_list: + parts = order_str.upper().split() + if len(parts) == 7 and parts[0] in ['A', 'F'] and parts[2] == 'C' and parts[3] in ['A', 'F'] and parts[5] == '-': + model_order_stats[model]['successful_convoys'] += 1 + elif len(parts) >= 4 and parts[2] == 'S' and parts[0] in ['A', 'F'] and parts[3] in ['A', 'F']: + model_order_stats[model]['successful_supports'] += 1 + elif not is_successful_order_set: + model_order_stats[model]['total_failed_order_sets_processed'] += 1 + potential_failed_orders = [] + failure_reason_str = str(entry.get('success', '')) + failure_prefix = "Failure: Invalid LLM Moves (1): " + if isinstance(failure_reason_str, str) and failure_reason_str.startswith(failure_prefix): + failed_order_from_success = failure_reason_str[len(failure_prefix):].strip() + if failed_order_from_success: + potential_failed_orders.append(failed_order_from_success) + + if not potential_failed_orders and llm_response_content is not None: + extracted_llm_orders = extract_orders_from_llm_response(llm_response_content, model) + if extracted_llm_orders: + potential_failed_orders.extend(extracted_llm_orders) + + if potential_failed_orders: + for order_str in potential_failed_orders: + parts = order_str.upper().split() + if len(parts) == 7 and parts[0] in ['A', 'F'] and parts[2] == 'C' and parts[3] in ['A', 'F'] and parts[5] == '-': + model_order_stats[model]['failed_convoys'] += 1 + elif len(parts) >= 4 and parts[2] == 'S' and parts[0] in ['A', 'F'] and parts[3] in ['A', 'F']: + model_order_stats[model]['failed_supports'] += 1 + + if prompt_content is not None and isinstance(prompt_content, str): + response_type_stats[response_type]['prompt_chars'].append(len(prompt_content)) + + if llm_response_content is not None: + if isinstance(llm_response_content, str): + response_type_stats[response_type]['response_chars'].append(len(llm_response_content)) + else: + try: + response_str = json.dumps(llm_response_content) + response_type_stats[response_type]['response_chars'].append(len(response_str)) + except TypeError: + logging.warning(f" Warning: Could not serialize llm_response in {filename}.") + except json.JSONDecodeError: + logging.warning(f" Warning: Could not decode JSON from {filename}. Skipping file.") + except Exception as e: + logging.warning(f" Warning: An error occurred processing {filename}: {e}. Skipping file.") + + if json_files_processed == 0 and not os.path.isdir(json_directory): # Check if error was already set + pass # Error already logged and will be in return + elif json_files_processed == 0: + logging.warning(f"No '*_rl.json' files found in {json_directory}.") + return { + "total_rows": 0, "total_characters": 0, "response_type_stats": defaultdict(lambda: {'prompt_chars': [], 'response_chars': []}), + "model_order_stats": defaultdict(lambda: {'successful_convoys': 0, 'successful_supports': 0, 'total_successful_orders_processed': 0, 'failed_convoys': 0, 'failed_supports': 0, 'total_failed_order_sets_processed': 0}), + "all_success_values": set(), "json_files_processed": 0, "error": f"No '*_rl.json' files found in {json_directory}" + } + + return { + "total_rows": total_rows, + "total_characters": total_characters, + "response_type_stats": response_type_stats, + "model_order_stats": model_order_stats, + "all_success_values": all_success_values, + "json_files_processed": json_files_processed, + "error": None + } + +def _write_summary_output(output_file_path, analysis_data, is_debug_output): + # Setup file-specific logger for this output operation + file_logger = logging.getLogger(f"writer_{os.path.basename(output_file_path)}") + # Prevent propagation to avoid duplicate console logs if root logger has StreamHandler + file_logger.propagate = False + file_logger.setLevel(logging.DEBUG) + + # Clear existing handlers for this specific logger to avoid duplication if called multiple times for same file (though unlikely with current design) + if file_logger.hasHandlers(): + file_logger.handlers.clear() + + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + fh = logging.FileHandler(output_file_path, mode='w') + fh.setLevel(logging.DEBUG) + fh.setFormatter(formatter) + file_logger.addHandler(fh) + + file_logger.info(f"Writing summary to: {output_file_path}. Debug mode: {is_debug_output}") + outfile = None + try: + outfile = open(output_file_path, 'w') + file_logger.debug(f"Output file '{output_file_path}' opened successfully for writing.") + outfile.write(f"Analysis Summary - {'Debug' if is_debug_output else 'Standard'}\n") + outfile.write(f"Generated on: {pd.Timestamp.now()}\n") + + if analysis_data.get("error"): + outfile.write(f"Analysis Error: {analysis_data['error']}\n") + file_logger.error(f"Analysis error reported: {analysis_data['error']}") + return # Stop writing if there was a critical analysis error outfile.write("\n--- Overall Statistics ---\n") - outfile.write(f"Total JSON files processed: {json_files_processed}\n") - outfile.write(f"Total JSON objects (rows) generated: {total_rows}\n") - outfile.write(f"Total characters of JSON generated (sum of file sizes): {total_characters:,}\n") + outfile.write(f"Total JSON files processed: {analysis_data['json_files_processed']}\n") + outfile.write(f"Total JSON objects (rows) generated: {analysis_data['total_rows']}\n") + outfile.write(f"Total characters of JSON generated (sum of file sizes): {analysis_data['total_characters']:,}\n") outfile.write("\n--- Average Lengths by Response Type (in characters) ---\n") + response_type_stats = analysis_data['response_type_stats'] outfile.write(f"Found {len(response_type_stats)} unique response_type categories.\n") - print(f"Found {len(response_type_stats)} unique response_type categories.") - - avg_data = [] + avg_data = [] for rt, stats_item in response_type_stats.items(): avg_prompt_len = sum(stats_item['prompt_chars']) / len(stats_item['prompt_chars']) if stats_item['prompt_chars'] else 0 avg_response_len = sum(stats_item['response_chars']) / len(stats_item['response_chars']) if stats_item['response_chars'] else 0 count = max(len(stats_item['prompt_chars']), len(stats_item['response_chars'])) avg_data.append({ - 'Response Type': rt, - 'Count': count, + 'Response Type': rt, 'Count': count, 'Avg Prompt Length': f"{avg_prompt_len:.2f}", 'Avg LLM Response Length': f"{avg_response_len:.2f}" }) - if avg_data: - df_avg = pd.DataFrame(avg_data) - outfile.write(df_avg.to_string(index=False) + "\n") - print("DataFrame successfully written.") + outfile.write(pd.DataFrame(avg_data).to_string(index=False) + "\n") else: - no_avg_data_msg = "No data available for response type analysis.\n" - outfile.write(no_avg_data_msg) - print(no_avg_data_msg.strip()) + outfile.write("No data available for response type analysis.\n") - # --- Output Successful Convoy/Support Orders by Model --- - outfile.write("\n--- Successful Convoy/Support Orders by Model ---\n") + outfile.write("\n--- Convoy/Support Orders by Model ---\n") # Renamed for clarity + model_order_stats = analysis_data['model_order_stats'] order_stats_data = [] - # Sort models for consistent output, handle if model_order_stats is empty sorted_models = sorted(model_order_stats.keys()) - for model_key in sorted_models: counts = model_order_stats[model_key] order_stats_data.append({ 'Model': model_key, 'Successful Convoys': counts['successful_convoys'], 'Successful Supports': counts['successful_supports'], - 'Total Order Sets Processed': counts['total_successful_orders_processed'] + 'Total Successful Sets': counts['total_successful_orders_processed'], + 'Failed Convoys': counts['failed_convoys'], + 'Failed Supports': counts['failed_supports'], + 'Total Failed Sets': counts['total_failed_order_sets_processed'] }) - if order_stats_data: - df_orders = pd.DataFrame(order_stats_data) - outfile.write(df_orders.to_string(index=False) + "\n") - print("\nSuccessful Convoy/Support Orders by Model:") - print(df_orders.to_string(index=False)) + outfile.write(pd.DataFrame(order_stats_data).to_string(index=False) + "\n") else: - outfile.write("No successful convoy or support orders found for analysis, or no 'order_generation' entries were successful.\n") - print("\nNo successful convoy or support orders found for analysis, or no 'order_generation' entries were successful.") + outfile.write("No convoy or support order data found.\n") + + if is_debug_output: + all_success_values = analysis_data['all_success_values'] + outfile.write("\n--- Unique Values in 'success' field (for order_generation) ---\n") + sorted_success_values = sorted(list(all_success_values), key=lambda x: str(x) if x is not None else '') + for val in sorted_success_values: + outfile.write(f"{val} (Type: {type(val).__name__})\n") + + non_successful_values = set() + for val_iter in all_success_values: + is_val_successful = False + if val_iter is True or (isinstance(val_iter, str) and val_iter.lower() in ['success', 'true']): + is_val_successful = True + if not is_val_successful: + non_successful_values.add(val_iter) + + outfile.write("\n--- Identified Non-Successful 'success' values (for order_generation) ---\n") + if non_successful_values: + sorted_non_successful = sorted(list(non_successful_values), key=lambda x: str(x) if x is not None else '') + for val_ns in sorted_non_successful: + outfile.write(f"{val_ns} (Type: {type(val_ns).__name__})\n") + else: + outfile.write("No specific non-successful values identified beyond False/None or unhandled strings.\n") outfile.write("\nAnalysis script finished successfully.\n") - print(f"\nAnalysis complete. Summary saved to: {output_file_path}") + file_logger.info(f"Successfully wrote summary to {output_file_path}") + print(f"Analysis complete. Summary saved to: {output_file_path}") except Exception as e: - print(f"FATAL SCRIPT ERROR: An exception occurred: {e}") - traceback.print_exc() - # Attempt to write to a fallback error log + file_logger.error(f"Error writing summary to {output_file_path}: {e}", exc_info=True) + print(f"FATAL ERROR writing to {output_file_path}: {e}") + if outfile and not outfile.closed: + try: + outfile.write(f"\nFATAL ERROR during summary writing: {e}\nTraceback:\n{traceback.format_exc()}\n") + except Exception as write_err: + file_logger.error(f"Error writing FATAL ERROR message to {output_file_path}: {write_err}") + finally: + if outfile and not outfile.closed: + file_logger.debug(f"Closing output file {output_file_path} in 'finally' block.") + outfile.close() + # Important: Remove the file handler for this specific file to avoid issues on next call + if fh in file_logger.handlers: + file_logger.removeHandler(fh) + fh.close() + + +def generate_analysis_reports(json_directory): + # Configure root logger for console output (INFO and higher) + # This setup is done once here. + root_logger = logging.getLogger() # Get the root logger + root_logger.setLevel(logging.DEBUG) # Set root to DEBUG to allow handlers to control their own levels + + # Clear any existing handlers on the root logger to avoid duplication if script is re-run in same session + if root_logger.hasHandlers(): + # Be careful clearing all handlers if other libraries also use logging. + # For a standalone script, this is usually fine. + # Alternatively, manage handlers more selectively or use a dedicated logger for this app. + for handler in root_logger.handlers[:]: # Iterate over a copy + root_logger.removeHandler(handler) + handler.close() + + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + sh = logging.StreamHandler(sys.stdout) + sh.setLevel(logging.INFO) + sh.setFormatter(formatter) + root_logger.addHandler(sh) + + logging.info(f"Starting analysis report generation for directory: {json_directory}") + try: + analysis_data = _perform_analysis(json_directory) + + standard_output_path = os.path.abspath('analysis_summary.txt') + debug_output_path = os.path.abspath('analysis_summary_debug.txt') + + logging.info(f"Proceeding to write standard summary to {standard_output_path}") + _write_summary_output(standard_output_path, analysis_data, is_debug_output=False) + + logging.info(f"Proceeding to write debug summary to {debug_output_path}") + _write_summary_output(debug_output_path, analysis_data, is_debug_output=True) + + logging.info("All analysis reports generated successfully.") + + except Exception as e: + logging.critical(f"A critical error occurred during analysis report generation: {e}", exc_info=True) + print(f"CRITICAL SCRIPT ERROR: {e}") + # Attempt to write to a fallback error log for the main orchestrator try: - with open('analyze_rl_json_CRITICAL_ERROR.log', 'w') as err_log: + with open('analyze_rl_json_CRITICAL_ERROR.log', 'a') as err_log: # Append mode for critical errors err_log.write(f"Timestamp: {pd.Timestamp.now()}\n") - err_log.write(f"Failed during script execution.\nError: {e}\n") - err_log.write(f"Traceback:\n{traceback.format_exc()}\n") + err_log.write(f"Failed during generate_analysis_reports for directory: {json_directory}.\nError: {e}\n") + err_log.write(f"Traceback:\n{traceback.format_exc()}\n---\n") except Exception as e_fallback: print(f"CRITICAL FALLBACK LOGGING FAILED: {e_fallback}") - finally: - # This block will always execute, ensuring the file is closed. - if outfile and not outfile.closed: - print("DEBUG: Closing output file in 'finally' block.") - outfile.close() if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Analyze generated RL JSON files.') + parser = argparse.ArgumentParser(description='Analyze generated RL JSON files and produce standard and debug summaries.') parser.add_argument('json_dir', type=str, help='Directory containing the *_rl.json files to analyze.') - parser.add_argument('--output_file', type=str, default='analysis_summary.txt', - help='Path to save the analysis summary (default: analysis_summary.txt in the CWD).') + # --output_file argument is removed args = parser.parse_args() abs_json_dir = os.path.abspath(args.json_dir) - # Ensure output_file_path is absolute or relative to CWD as intended - output_file_path_arg = os.path.abspath(args.output_file) - analyze_json_files(abs_json_dir, output_file_path_arg) + generate_analysis_reports(abs_json_dir)