""" Completes FinOps Failover Whole-Repo Self-Healing (`schwifty`, `python_stdnum`), clean `chunked_top1` baselines (`price_parser`, `schwifty`), and Multi-Model Head-to-Head Benchmark with `max_tokens=12288` so `:free` 550B models never hit 4096-token markdown truncation. Preserves all completed `reflection/beam-1.0-501b-a23b` checkpoints (`py_moneyed` 40/40 & 37/40, `price_parser` 40/40). """ from __future__ import annotations import hashlib import json import os import re import sys import time from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parent.parent if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from backend.real_oracles import build_real_test_vectors, exact_beam_token_count, get_raw_repo_disk_stats, load_real_repo_files from execute_live_beam_campaigns import ( MCP_COMPAT_PREAMBLE, REPO_SPECS, call_openrouter, evaluate_generated_code_subprocess, ) def safe_extract_python(raw_text: str) -> str: m = re.search(r"```python\s*(.*?)```", raw_text, flags=re.DOTALL | re.IGNORECASE) if m: return m.group(1).strip() if "```python" in raw_text: return raw_text.split("```python", 1)[1].rsplit("```", 1)[0].strip() m2 = re.search(r"```\s*(.*?)```", raw_text, flags=re.DOTALL) if m2: return m2.group(1).strip() return raw_text.strip() def run_failover_modernization(api_key: str, repo_key: str, mode: str, model_id: str) -> dict[str, Any]: spec = REPO_SPECS[repo_key] full_files, chunk_files = load_real_repo_files(repo_key) vectors = build_real_test_vectors(repo_key) chosen_files = full_files if mode == "whole_repo_1m" else chunk_files packed_source = "\n\n".join(f"=== FILE: {rel} ===\n{content}" for rel, content in chosen_files.items()) packed_bpe_tokens = exact_beam_token_count(packed_source) sys_prompt = ( f"You are BeamBridge's Sovereign FinOps Failover Engine. {MCP_COMPAT_PREAMBLE} " "Use Python standard library (`decimal`, `math`, `re`, `json`, `typing`) ONLY. " "STRICT RULE: You are FORBIDDEN from importing `moneyed`, `price_parser`, `schwifty`, or `stdnum`! " "Return ONLY valid standalone Python code inside one ```python ... ``` block." ) user_prompt = ( f"### TARGET REPOSITORY: {spec['name']} (Git commit `{spec['git_sha']}`)\n" f"### REQUIRED MCP TOOLS SPECIFICATION\n{spec['tools_contract']}\n\n" f"### REPOSITORY SOURCE CONTEXT ({len(chosen_files)} files, {packed_bpe_tokens:,} Beam BPE tokens)\n" f"{packed_source}\n\n" "STRICT ZERO-DEPENDENCY RULE: DO NOT import `schwifty`, `moneyed`, `price_parser`, or `stdnum`! " "Output ONLY the complete standalone Python MCP module in a single ```python ... ``` block." ) messages: list[dict[str, Any]] = [ {"role": "system", "content": sys_prompt}, {"role": "user", "content": user_prompt}, ] max_turns = 3 if mode == "whole_repo_1m" else 1 turns_ledger: list[dict[str, Any]] = [] best_code = "" best_eval: dict[str, Any] = {"passed": -1, "total": len(vectors), "pass_rate_pct": 0.0, "case_results": []} for turn_idx in range(1, max_turns + 1): print(f" [{repo_key} | {mode}] Turn {turn_idx}/{max_turns} calling {model_id} ...", flush=True) api_res = call_openrouter(api_key=api_key, model=model_id, messages=messages, max_tokens=12288) code = safe_extract_python(api_res["content"]) ev = evaluate_generated_code_subprocess(code, vectors) turns_ledger.append({ "turn": turn_idx, "request_id": api_res["request_id"], "provider": api_res["provider"], "model": api_res["model"], "latency_ms": api_res["latency_ms"], "usage": api_res["usage"], "reasoning_excerpt": (api_res["reasoning"] or "")[:1800], "passed": ev["passed"], "total": ev["total"], "pass_rate_pct": ev["pass_rate_pct"], "error_summary": ev.get("syntax_or_policy_error"), }) print( f" -> id={api_res['request_id']} | passed={ev['passed']}/{ev['total']} ({ev['pass_rate_pct']}%) " f"| tokens={api_res['usage']['total_tokens']} (reasoning={api_res['usage']['reasoning_tokens']}) " f"| {api_res['latency_ms']}ms", flush=True, ) if ev["passed"] > best_eval["passed"]: best_eval = ev best_code = code if ev["passed"] == ev["total"] or turn_idx == max_turns: break sample_fails = ev.get("failures", [])[:10] err_note = f"\nSyntax/Load Error: {ev.get('syntax_or_policy_error')}\n" if ev.get("syntax_or_policy_error") else "" messages = [ {"role": "system", "content": sys_prompt}, { "role": "user", "content": ( f"Your standalone MCP module for `{spec['name']}` passed {ev['passed']}/{ev['total']} ground-truth tests.{err_note}\n" f"### YOUR CURRENT PYTHON MODULE\n```python\n{code}\n```\n\n" f"### FAILING ASSERTIONS TO FIX\n{json.dumps(sample_fails, indent=2)}\n\n" f"### REFERENCE REPOSITORY CONTEXT\n{packed_source}\n\n" f"Fix ALL discrepancies without importing `{repo_key}` or breaking passing cases. " "Return ONLY the complete corrected Python module in a single ```python ... ``` block." ), }, ] return { "repo_key": repo_key, "mode": mode, "files_included": len(chosen_files), "exact_repo_bpe_tokens": packed_bpe_tokens, "turns": turns_ledger, "final_passed": best_eval["passed"], "final_total": best_eval["total"], "final_pass_rate_pct": best_eval["pass_rate_pct"], "total_cost_usd": round(sum(t["usage"]["cost_usd"] for t in turns_ledger), 6), "total_prompt_tokens": sum(t["usage"]["prompt_tokens"] for t in turns_ledger), "total_completion_tokens": sum(t["usage"]["completion_tokens"] for t in turns_ledger), "total_reasoning_tokens": sum(t["usage"]["reasoning_tokens"] for t in turns_ledger), "total_latency_ms": round(sum(t["latency_ms"] for t in turns_ledger), 1), "best_code": best_code, "case_results": best_eval.get("case_results", []), } def main() -> None: api_key = os.environ["OPENROUTER_API_KEY"].strip() runs_dir = ROOT / "runs" mcp_dir = ROOT / "modernized_mcp" ckpt_path = runs_dir / "checkpoint_campaigns.json" ckpt = json.loads(ckpt_path.read_text(encoding="utf-8")) failover_model = "nvidia/nemotron-3-ultra-550b-a55b:free" # 1. Re-run whole_repo for schwifty & python_stdnum with proper 12,288 max_tokens! for rk in ("schwifty", "python_stdnum"): if ckpt["whole_repo"].get(rk, {}).get("final_passed", 0) < 38: print(f"\n=== Completing Whole-Repo + Self-Healing for {rk} ===", flush=True) wr = run_failover_modernization(api_key, rk, "whole_repo_1m", failover_model) ckpt["whole_repo"][rk] = wr header = ( f'"""\n' f'AUTONOMOUSLY SYNTHESIZED & DIFFERENTIAL-VERIFIED MCP SERVER\n' f'Source Repo: {REPO_SPECS[rk]["github_url"]} @ {REPO_SPECS[rk]["git_sha"]}\n' f'Model Route: {wr["turns"][-1]["model"]} (FinOps Failover after Beam-1.0 credit cap)\n' f'OpenRouter Request IDs: {[t["request_id"] for t in wr["turns"]]}\n' f'Verified Differential Parity: {wr["final_passed"]}/{wr["final_total"]} ({wr["final_pass_rate_pct"]}%)\n' f'"""\n' ) (mcp_dir / f"{rk}_mcp.py").write_text(header + wr["best_code"] + "\n", encoding="utf-8") ckpt_path.write_text(json.dumps(ckpt, indent=2), encoding="utf-8") # 2. Re-run chunked_top1 for price_parser & schwifty cleanly without 1132-token cutoff! for rk in ("price_parser", "schwifty"): if ckpt["chunked"].get(rk, {}).get("final_passed", 0) == 0: print(f"\n=== Completing Clean Chunked Baseline for {rk} ===", flush=True) ch = run_failover_modernization(api_key, rk, "chunked_top1", failover_model) ckpt["chunked"][rk] = ch ckpt_path.write_text(json.dumps(ckpt, indent=2), encoding="utf-8") # 3. Clean Multi-Model Head-to-Head on price_parser (16,495 Beam BPE tokens) with 12,288 max_tokens! print("\n=== Completing Clean Multi-Model Head-to-Head Benchmark on price_parser ===", flush=True) spec = REPO_SPECS["price_parser"] full_files, _ = load_real_repo_files("price_parser") vectors = build_real_test_vectors("price_parser") packed_source = "\n\n".join(f"=== FILE: {rel} ===\n{content}" for rel, content in full_files.items()) sys_prompt = ( f"Synthesize a SINGLE, COMPLETE, 100% STANDALONE Python module. {MCP_COMPAT_PREAMBLE} " "Use Python stdlib ONLY. DO NOT import `price_parser` or any forbidden legacy package. " "Return ONLY Python code inside ```python ... ```." ) user_prompt = ( f"### TARGET REPOSITORY: {spec['name']} (Git commit `{spec['git_sha']}`)\n" f"### REQUIRED MCP TOOLS SPECIFICATION\n{spec['tools_contract']}\n\n" f"### REPOSITORY SOURCE CONTEXT\n{packed_source}\n\n" "Output ONLY the complete standalone Python MCP module in a single ```python ... ``` block." ) msgs = [{"role": "system", "content": sys_prompt}, {"role": "user", "content": user_prompt}] pp_beam_t1 = ckpt["whole_repo"]["price_parser"]["turns"][0] pp_beam_t2 = ckpt["whole_repo"]["price_parser"]["turns"][1] h2h: list[dict[str, Any]] = [ { "benchmark_repo": "price_parser", "model": "reflection/beam-1.0-501b-a23b (Turn 1 Zero-Shot)", "request_id": pp_beam_t1["request_id"], "provider": pp_beam_t1["provider"], "latency_ms": pp_beam_t1["latency_ms"], "prompt_tokens": pp_beam_t1["usage"]["prompt_tokens"], "completion_tokens": pp_beam_t1["usage"]["completion_tokens"], "reasoning_tokens": pp_beam_t1["usage"]["reasoning_tokens"], "total_tokens": pp_beam_t1["usage"]["total_tokens"], "cost_usd": round(pp_beam_t1["usage"]["cost_usd"], 6), "zero_shot_passed": pp_beam_t1["passed"], "total_vectors": pp_beam_t1["total"], "zero_shot_parity_pct": pp_beam_t1["pass_rate_pct"], }, { "benchmark_repo": "price_parser", "model": "reflection/beam-1.0-501b-a23b (+Self-Heal Turn 2)", "request_id": pp_beam_t2["request_id"], "provider": pp_beam_t2["provider"], "latency_ms": round(pp_beam_t1["latency_ms"] + pp_beam_t2["latency_ms"], 1), "prompt_tokens": pp_beam_t1["usage"]["prompt_tokens"] + pp_beam_t2["usage"]["prompt_tokens"], "completion_tokens": pp_beam_t1["usage"]["completion_tokens"] + pp_beam_t2["usage"]["completion_tokens"], "reasoning_tokens": pp_beam_t1["usage"]["reasoning_tokens"] + pp_beam_t2["usage"]["reasoning_tokens"], "total_tokens": pp_beam_t1["usage"]["total_tokens"] + pp_beam_t2["usage"]["total_tokens"], "cost_usd": round(pp_beam_t1["usage"]["cost_usd"] + pp_beam_t2["usage"]["cost_usd"], 6), "zero_shot_passed": pp_beam_t2["passed"], "total_vectors": pp_beam_t2["total"], "zero_shot_parity_pct": pp_beam_t2["pass_rate_pct"], }, ] for comp_model in ("nvidia/nemotron-3-ultra-550b-a55b:free", "poolside/laguna-s-2.1:free", "cohere/north-mini-code:free"): print(f" [H2H] Calling {comp_model} with max_tokens=12288 ...", flush=True) try: res = call_openrouter(api_key=api_key, model=comp_model, messages=msgs, max_tokens=12288) code = safe_extract_python(res["content"]) ev = evaluate_generated_code_subprocess(code, vectors) entry = { "benchmark_repo": "price_parser", "model": comp_model, "request_id": res["request_id"], "provider": res["provider"], "latency_ms": res["latency_ms"], "prompt_tokens": res["usage"]["prompt_tokens"], "completion_tokens": res["usage"]["completion_tokens"], "reasoning_tokens": res["usage"]["reasoning_tokens"], "total_tokens": res["usage"]["total_tokens"], "cost_usd": round(res["usage"]["cost_usd"], 6), "zero_shot_passed": ev["passed"], "total_vectors": ev["total"], "zero_shot_parity_pct": ev["pass_rate_pct"], } print(f" -> {comp_model}: {ev['passed']}/{ev['total']} ({ev['pass_rate_pct']}%) | tok={entry['completion_tokens']} | {entry['latency_ms']}ms", flush=True) h2h.append(entry) except Exception as exc: print(f" -> {comp_model} ERR: {exc}", flush=True) ckpt["h2h"] = h2h ckpt_path.write_text(json.dumps(ckpt, indent=2), encoding="utf-8") # Build final verifiable_empirical_ledger.json including raw disk repo stats! ledger = { "generated_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "primary_model": "reflection/beam-1.0-501b-a23b", "failover_model": failover_model, "tokenizer_vocab_size": 151681, "repositories": {}, "head_to_head_benchmark": h2h, } for rk in ("py_moneyed", "price_parser", "schwifty", "python_stdnum"): wr = ckpt["whole_repo"][rk] ch = ckpt["chunked"][rk] raw_stats = get_raw_repo_disk_stats(rk) chunk_pass_by_idx = {c["index"]: c["passed"] for c in ch.get("case_results", [])} merged_cases = [{**c, "chunked_rag_passed": chunk_pass_by_idx.get(c["index"], False)} for c in wr.get("case_results", [])] cert_hash = hashlib.sha256(json.dumps(merged_cases, sort_keys=True).encode("utf-8")).hexdigest() ledger["repositories"][rk] = { "spec": {k: v for k, v in REPO_SPECS[rk].items() if k != "tools_contract"}, "raw_github_disk_stats": raw_stats, "whole_repo_1m": {k: v for k, v in wr.items() if k not in ("best_code", "case_results")}, "chunked_top1": {k: v for k, v in ch.items() if k not in ("best_code", "case_results")}, "certificate_sha256": cert_hash, "cases": merged_cases, } ledger_path = runs_dir / "verifiable_empirical_ledger.json" ledger_path.write_text(json.dumps(ledger, indent=2), encoding="utf-8") print(f"\nALL DONE! Saved verified empirical ledger -> {ledger_path}", flush=True) if __name__ == "__main__": main()