Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /CIRISAI_CIRISAgent /tools /analyze_llm_errors.py
| #!/usr/bin/env python3 | |
| """ | |
| Analyze LLM errors from CIRIS incident logs. | |
| Extracts rate limiting, token usage, retry patterns, and circuit breaker behavior. | |
| """ | |
| import re | |
| import sys | |
| from collections import defaultdict | |
| from datetime import datetime | |
| from pathlib import Path | |
| def parse_timestamp(line: str) -> datetime | None: | |
| """Extract timestamp from log line.""" | |
| match = re.match(r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+)", line) | |
| if match: | |
| return datetime.strptime(match.group(1), "%Y-%m-%d %H:%M:%S.%f") | |
| return None | |
| def analyze_log(log_path: str) -> dict: | |
| """Analyze a log file for LLM errors and patterns.""" | |
| stats = { | |
| "rate_limits": [], | |
| "circuit_breaker_events": [], | |
| "token_usage": [], | |
| "retries": [], | |
| "errors_by_type": defaultdict(int), | |
| "llm_calls": [], | |
| "first_timestamp": None, | |
| "last_timestamp": None, | |
| } | |
| with open(log_path, "r") as f: | |
| lines = f.readlines() | |
| for i, line in enumerate(lines): | |
| ts = parse_timestamp(line) | |
| if ts: | |
| if stats["first_timestamp"] is None: | |
| stats["first_timestamp"] = ts | |
| stats["last_timestamp"] = ts | |
| # Rate limit detection | |
| if "429" in line or "rate limit" in line.lower() or "rate_limit" in line.lower(): | |
| # Extract token info | |
| token_match = re.search(r"Limit (\d+), Used (\d+), Requested (\d+)", line) | |
| retry_match = re.search(r"try again in ([\d.]+)s", line, re.IGNORECASE) | |
| stats["rate_limits"].append( | |
| { | |
| "timestamp": ts, | |
| "line_num": i + 1, | |
| "limit": int(token_match.group(1)) if token_match else None, | |
| "used": int(token_match.group(2)) if token_match else None, | |
| "requested": int(token_match.group(3)) if token_match else None, | |
| "retry_after": float(retry_match.group(1)) if retry_match else None, | |
| } | |
| ) | |
| stats["errors_by_type"]["rate_limit_429"] += 1 | |
| # Circuit breaker events | |
| if "circuit breaker" in line.lower() or "Circuit breaker" in line: | |
| if "opened" in line.lower() or "OPEN" in line: | |
| stats["circuit_breaker_events"].append( | |
| { | |
| "timestamp": ts, | |
| "event": "OPENED", | |
| "line_num": i + 1, | |
| "line": line.strip()[:150], | |
| } | |
| ) | |
| elif "closed" in line.lower() or "reset" in line.lower(): | |
| stats["circuit_breaker_events"].append( | |
| { | |
| "timestamp": ts, | |
| "event": "CLOSED/RESET", | |
| "line_num": i + 1, | |
| "line": line.strip()[:150], | |
| } | |
| ) | |
| elif "skipping" in line.lower(): | |
| stats["errors_by_type"]["circuit_breaker_skip"] += 1 | |
| # Retry detection | |
| if "retry" in line.lower() and ("waiting" in line.lower() or "attempt" in line.lower()): | |
| retry_match = re.search(r"\((\d+)/(\d+)\)", line) | |
| wait_match = re.search(r"waiting ([\d.]+)s", line, re.IGNORECASE) | |
| stats["retries"].append( | |
| { | |
| "timestamp": ts, | |
| "attempt": int(retry_match.group(1)) if retry_match else None, | |
| "max_attempts": int(retry_match.group(2)) if retry_match else None, | |
| "wait_time": float(wait_match.group(1)) if wait_match else None, | |
| "line_num": i + 1, | |
| } | |
| ) | |
| # LLM call completion | |
| if "LLM call completed" in line or "call_llm_structured" in line: | |
| stats["llm_calls"].append( | |
| { | |
| "timestamp": ts, | |
| "line_num": i + 1, | |
| "success": "completed" in line.lower() or "success" in line.lower(), | |
| } | |
| ) | |
| # Error type categorization | |
| if "ERROR" in line: | |
| if "SCHEMA VALIDATION" in line: | |
| stats["errors_by_type"]["schema_validation"] += 1 | |
| elif "TIMEOUT" in line: | |
| stats["errors_by_type"]["timeout"] += 1 | |
| elif "503" in line or "SERVICE UNAVAILABLE" in line: | |
| stats["errors_by_type"]["service_unavailable_503"] += 1 | |
| elif "401" in line or "unauthorized" in line.lower(): | |
| stats["errors_by_type"]["unauthorized_401"] += 1 | |
| elif "DMA" in line and "failed" in line.lower(): | |
| stats["errors_by_type"]["dma_failure"] += 1 | |
| elif "InstructorRetryException" in line: | |
| stats["errors_by_type"]["instructor_retry"] += 1 | |
| return stats | |
| def print_report(stats: dict, log_path: str): | |
| """Print a formatted analysis report.""" | |
| print("=" * 70) | |
| print(f"LLM ERROR ANALYSIS REPORT") | |
| print(f"Log file: {log_path}") | |
| print("=" * 70) | |
| if stats["first_timestamp"] and stats["last_timestamp"]: | |
| duration = stats["last_timestamp"] - stats["first_timestamp"] | |
| print(f"\nTime Range: {stats['first_timestamp']} to {stats['last_timestamp']}") | |
| print(f"Duration: {duration}") | |
| # Rate Limits | |
| print("\n" + "-" * 50) | |
| print("RATE LIMITS (429)") | |
| print("-" * 50) | |
| if stats["rate_limits"]: | |
| print(f"Total rate limit hits: {len(stats['rate_limits'])}") | |
| # Token analysis | |
| token_events = [r for r in stats["rate_limits"] if r["limit"]] | |
| if token_events: | |
| print(f"\nToken Usage at Rate Limit Events:") | |
| for event in token_events[:10]: # Show first 10 | |
| pct = (event["used"] / event["limit"] * 100) if event["limit"] else 0 | |
| print( | |
| f" {event['timestamp']}: Used {event['used']:,}/{event['limit']:,} ({pct:.1f}%) + Requested {event['requested']:,}" | |
| ) | |
| if len(token_events) > 10: | |
| print(f" ... and {len(token_events) - 10} more") | |
| # Retry-after times | |
| retry_times = [r["retry_after"] for r in stats["rate_limits"] if r["retry_after"]] | |
| if retry_times: | |
| print(f"\nRetry-After times suggested by API:") | |
| print(f" Min: {min(retry_times):.2f}s") | |
| print(f" Max: {max(retry_times):.2f}s") | |
| print(f" Avg: {sum(retry_times)/len(retry_times):.2f}s") | |
| else: | |
| print("No rate limit events found") | |
| # Circuit Breaker | |
| print("\n" + "-" * 50) | |
| print("CIRCUIT BREAKER EVENTS") | |
| print("-" * 50) | |
| if stats["circuit_breaker_events"]: | |
| for event in stats["circuit_breaker_events"]: | |
| print(f" {event['timestamp']}: {event['event']}") | |
| print(f" Line {event['line_num']}: {event['line'][:100]}...") | |
| # Time from first rate limit to circuit breaker open | |
| opens = [e for e in stats["circuit_breaker_events"] if e["event"] == "OPENED"] | |
| if opens and stats["rate_limits"]: | |
| first_rl = stats["rate_limits"][0]["timestamp"] | |
| first_open = opens[0]["timestamp"] | |
| if first_rl and first_open: | |
| delta = first_open - first_rl | |
| print(f"\n Time from first rate limit to CB open: {delta.total_seconds():.2f}s") | |
| else: | |
| print("No circuit breaker events found") | |
| cb_skips = stats["errors_by_type"].get("circuit_breaker_skip", 0) | |
| if cb_skips: | |
| print(f"\n Circuit breaker skips (calls rejected): {cb_skips}") | |
| # Retries | |
| print("\n" + "-" * 50) | |
| print("RETRY BEHAVIOR") | |
| print("-" * 50) | |
| if stats["retries"]: | |
| print(f"Total retry attempts: {len(stats['retries'])}") | |
| wait_times = [r["wait_time"] for r in stats["retries"] if r["wait_time"]] | |
| if wait_times: | |
| print(f"Wait times used: {', '.join(f'{t}s' for t in sorted(set(wait_times)))}") | |
| else: | |
| print("No retry events found") | |
| # Error Summary | |
| print("\n" + "-" * 50) | |
| print("ERROR TYPE SUMMARY") | |
| print("-" * 50) | |
| if stats["errors_by_type"]: | |
| for error_type, count in sorted(stats["errors_by_type"].items(), key=lambda x: -x[1]): | |
| print(f" {error_type}: {count}") | |
| else: | |
| print("No categorized errors found") | |
| # Recommendations | |
| print("\n" + "-" * 50) | |
| print("ANALYSIS & RECOMMENDATIONS") | |
| print("-" * 50) | |
| rl_count = len(stats["rate_limits"]) | |
| cb_opens = len([e for e in stats["circuit_breaker_events"] if e["event"] == "OPENED"]) | |
| if rl_count > 0 and cb_opens > 0: | |
| print(" [WARN] Rate limits triggered circuit breaker opening") | |
| print(" - Fix: Rate limits should NOT count as circuit breaker failures") | |
| print(" - Rate limits are transient; circuit breaker is for persistent failures") | |
| if stats["rate_limits"]: | |
| avg_retry = sum(r["retry_after"] or 0 for r in stats["rate_limits"]) / max( | |
| 1, len([r for r in stats["rate_limits"] if r["retry_after"]]) | |
| ) | |
| if avg_retry > 0: | |
| print(f" API suggests retry after ~{avg_retry:.1f}s on average") | |
| print(f" - Consider setting retry wait to {max(2, avg_retry + 0.5):.1f}s") | |
| token_events = [r for r in stats["rate_limits"] if r["used"] and r["limit"]] | |
| if token_events: | |
| avg_usage_pct = sum(e["used"] / e["limit"] * 100 for e in token_events) / len(token_events) | |
| print(f" Average token usage at rate limit: {avg_usage_pct:.1f}%") | |
| if avg_usage_pct > 90: | |
| print(" - Tokens nearly exhausted before rate limit") | |
| print(" - Consider reducing concurrent LLM calls") | |
| def main(): | |
| # Find the most recent log file | |
| log_dir = Path("logs") | |
| # Check for incidents_latest.log symlink or direct file | |
| latest_log = log_dir / "incidents_latest.log" | |
| if not latest_log.exists(): | |
| # Find in sqlite subdirectory | |
| sqlite_logs = list((log_dir / "sqlite").glob("incidents_*.log")) | |
| if sqlite_logs: | |
| latest_log = max(sqlite_logs, key=lambda p: p.stat().st_mtime) | |
| else: | |
| # Try root logs directory | |
| root_logs = list(log_dir.glob("incidents_*.log")) | |
| if root_logs: | |
| latest_log = max(root_logs, key=lambda p: p.stat().st_mtime) | |
| if len(sys.argv) > 1: | |
| latest_log = Path(sys.argv[1]) | |
| if not latest_log.exists(): | |
| print(f"Error: Log file not found: {latest_log}") | |
| print("Usage: python analyze_llm_errors.py [log_file_path]") | |
| sys.exit(1) | |
| print(f"Analyzing: {latest_log}") | |
| stats = analyze_log(str(latest_log)) | |
| print_report(stats, str(latest_log)) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 10.8 kB
- Xet hash:
- 2b69a3c2d2f1d13dd7a47ad6cbd6b25c36407ecef2eea60ed1aed3beefb68d51
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.