Spaces:
Running
Running
| """ | |
| Completes FinOps Failover Whole-Repo Self-Healing (`schwifty`, `python_stdnum`), | |
| clean `chunked_top1` baselines (`price_parser`, `schwifty`), and Multi-Model Head-to-Head Benchmark | |
| with `max_tokens=12288` so `:free` 550B models never hit 4096-token markdown truncation. | |
| Preserves all completed `reflection/beam-1.0-501b-a23b` checkpoints (`py_moneyed` 40/40 & 37/40, `price_parser` 40/40). | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import os | |
| import re | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from typing import Any | |
| ROOT = Path(__file__).resolve().parent.parent | |
| if str(ROOT) not in sys.path: | |
| sys.path.insert(0, str(ROOT)) | |
| from backend.real_oracles import build_real_test_vectors, exact_beam_token_count, get_raw_repo_disk_stats, load_real_repo_files | |
| from execute_live_beam_campaigns import ( | |
| MCP_COMPAT_PREAMBLE, | |
| REPO_SPECS, | |
| call_openrouter, | |
| evaluate_generated_code_subprocess, | |
| ) | |
| def safe_extract_python(raw_text: str) -> str: | |
| m = re.search(r"```python\s*(.*?)```", raw_text, flags=re.DOTALL | re.IGNORECASE) | |
| if m: | |
| return m.group(1).strip() | |
| if "```python" in raw_text: | |
| return raw_text.split("```python", 1)[1].rsplit("```", 1)[0].strip() | |
| m2 = re.search(r"```\s*(.*?)```", raw_text, flags=re.DOTALL) | |
| if m2: | |
| return m2.group(1).strip() | |
| return raw_text.strip() | |
| def run_failover_modernization(api_key: str, repo_key: str, mode: str, model_id: str) -> dict[str, Any]: | |
| spec = REPO_SPECS[repo_key] | |
| full_files, chunk_files = load_real_repo_files(repo_key) | |
| vectors = build_real_test_vectors(repo_key) | |
| chosen_files = full_files if mode == "whole_repo_1m" else chunk_files | |
| packed_source = "\n\n".join(f"=== FILE: {rel} ===\n{content}" for rel, content in chosen_files.items()) | |
| packed_bpe_tokens = exact_beam_token_count(packed_source) | |
| sys_prompt = ( | |
| f"You are BeamBridge's Sovereign FinOps Failover Engine. {MCP_COMPAT_PREAMBLE} " | |
| "Use Python standard library (`decimal`, `math`, `re`, `json`, `typing`) ONLY. " | |
| "STRICT RULE: You are FORBIDDEN from importing `moneyed`, `price_parser`, `schwifty`, or `stdnum`! " | |
| "Return ONLY valid standalone Python code inside one ```python ... ``` block." | |
| ) | |
| user_prompt = ( | |
| f"### TARGET REPOSITORY: {spec['name']} (Git commit `{spec['git_sha']}`)\n" | |
| f"### REQUIRED MCP TOOLS SPECIFICATION\n{spec['tools_contract']}\n\n" | |
| f"### REPOSITORY SOURCE CONTEXT ({len(chosen_files)} files, {packed_bpe_tokens:,} Beam BPE tokens)\n" | |
| f"{packed_source}\n\n" | |
| "STRICT ZERO-DEPENDENCY RULE: DO NOT import `schwifty`, `moneyed`, `price_parser`, or `stdnum`! " | |
| "Output ONLY the complete standalone Python MCP module in a single ```python ... ``` block." | |
| ) | |
| messages: list[dict[str, Any]] = [ | |
| {"role": "system", "content": sys_prompt}, | |
| {"role": "user", "content": user_prompt}, | |
| ] | |
| max_turns = 3 if mode == "whole_repo_1m" else 1 | |
| turns_ledger: list[dict[str, Any]] = [] | |
| best_code = "" | |
| best_eval: dict[str, Any] = {"passed": -1, "total": len(vectors), "pass_rate_pct": 0.0, "case_results": []} | |
| for turn_idx in range(1, max_turns + 1): | |
| print(f" [{repo_key} | {mode}] Turn {turn_idx}/{max_turns} calling {model_id} ...", flush=True) | |
| api_res = call_openrouter(api_key=api_key, model=model_id, messages=messages, max_tokens=12288) | |
| code = safe_extract_python(api_res["content"]) | |
| ev = evaluate_generated_code_subprocess(code, vectors) | |
| turns_ledger.append({ | |
| "turn": turn_idx, | |
| "request_id": api_res["request_id"], | |
| "provider": api_res["provider"], | |
| "model": api_res["model"], | |
| "latency_ms": api_res["latency_ms"], | |
| "usage": api_res["usage"], | |
| "reasoning_excerpt": (api_res["reasoning"] or "")[:1800], | |
| "passed": ev["passed"], | |
| "total": ev["total"], | |
| "pass_rate_pct": ev["pass_rate_pct"], | |
| "error_summary": ev.get("syntax_or_policy_error"), | |
| }) | |
| print( | |
| f" -> id={api_res['request_id']} | passed={ev['passed']}/{ev['total']} ({ev['pass_rate_pct']}%) " | |
| f"| tokens={api_res['usage']['total_tokens']} (reasoning={api_res['usage']['reasoning_tokens']}) " | |
| f"| {api_res['latency_ms']}ms", | |
| flush=True, | |
| ) | |
| if ev["passed"] > best_eval["passed"]: | |
| best_eval = ev | |
| best_code = code | |
| if ev["passed"] == ev["total"] or turn_idx == max_turns: | |
| break | |
| sample_fails = ev.get("failures", [])[:10] | |
| err_note = f"\nSyntax/Load Error: {ev.get('syntax_or_policy_error')}\n" if ev.get("syntax_or_policy_error") else "" | |
| messages = [ | |
| {"role": "system", "content": sys_prompt}, | |
| { | |
| "role": "user", | |
| "content": ( | |
| f"Your standalone MCP module for `{spec['name']}` passed {ev['passed']}/{ev['total']} ground-truth tests.{err_note}\n" | |
| f"### YOUR CURRENT PYTHON MODULE\n```python\n{code}\n```\n\n" | |
| f"### FAILING ASSERTIONS TO FIX\n{json.dumps(sample_fails, indent=2)}\n\n" | |
| f"### REFERENCE REPOSITORY CONTEXT\n{packed_source}\n\n" | |
| f"Fix ALL discrepancies without importing `{repo_key}` or breaking passing cases. " | |
| "Return ONLY the complete corrected Python module in a single ```python ... ``` block." | |
| ), | |
| }, | |
| ] | |
| return { | |
| "repo_key": repo_key, | |
| "mode": mode, | |
| "files_included": len(chosen_files), | |
| "exact_repo_bpe_tokens": packed_bpe_tokens, | |
| "turns": turns_ledger, | |
| "final_passed": best_eval["passed"], | |
| "final_total": best_eval["total"], | |
| "final_pass_rate_pct": best_eval["pass_rate_pct"], | |
| "total_cost_usd": round(sum(t["usage"]["cost_usd"] for t in turns_ledger), 6), | |
| "total_prompt_tokens": sum(t["usage"]["prompt_tokens"] for t in turns_ledger), | |
| "total_completion_tokens": sum(t["usage"]["completion_tokens"] for t in turns_ledger), | |
| "total_reasoning_tokens": sum(t["usage"]["reasoning_tokens"] for t in turns_ledger), | |
| "total_latency_ms": round(sum(t["latency_ms"] for t in turns_ledger), 1), | |
| "best_code": best_code, | |
| "case_results": best_eval.get("case_results", []), | |
| } | |
| def main() -> None: | |
| api_key = os.environ["OPENROUTER_API_KEY"].strip() | |
| runs_dir = ROOT / "runs" | |
| mcp_dir = ROOT / "modernized_mcp" | |
| ckpt_path = runs_dir / "checkpoint_campaigns.json" | |
| ckpt = json.loads(ckpt_path.read_text(encoding="utf-8")) | |
| failover_model = "nvidia/nemotron-3-ultra-550b-a55b:free" | |
| # 1. Re-run whole_repo for schwifty & python_stdnum with proper 12,288 max_tokens! | |
| for rk in ("schwifty", "python_stdnum"): | |
| if ckpt["whole_repo"].get(rk, {}).get("final_passed", 0) < 38: | |
| print(f"\n=== Completing Whole-Repo + Self-Healing for {rk} ===", flush=True) | |
| wr = run_failover_modernization(api_key, rk, "whole_repo_1m", failover_model) | |
| ckpt["whole_repo"][rk] = wr | |
| header = ( | |
| f'"""\n' | |
| f'AUTONOMOUSLY SYNTHESIZED & DIFFERENTIAL-VERIFIED MCP SERVER\n' | |
| f'Source Repo: {REPO_SPECS[rk]["github_url"]} @ {REPO_SPECS[rk]["git_sha"]}\n' | |
| f'Model Route: {wr["turns"][-1]["model"]} (FinOps Failover after Beam-1.0 credit cap)\n' | |
| f'OpenRouter Request IDs: {[t["request_id"] for t in wr["turns"]]}\n' | |
| f'Verified Differential Parity: {wr["final_passed"]}/{wr["final_total"]} ({wr["final_pass_rate_pct"]}%)\n' | |
| f'"""\n' | |
| ) | |
| (mcp_dir / f"{rk}_mcp.py").write_text(header + wr["best_code"] + "\n", encoding="utf-8") | |
| ckpt_path.write_text(json.dumps(ckpt, indent=2), encoding="utf-8") | |
| # 2. Re-run chunked_top1 for price_parser & schwifty cleanly without 1132-token cutoff! | |
| for rk in ("price_parser", "schwifty"): | |
| if ckpt["chunked"].get(rk, {}).get("final_passed", 0) == 0: | |
| print(f"\n=== Completing Clean Chunked Baseline for {rk} ===", flush=True) | |
| ch = run_failover_modernization(api_key, rk, "chunked_top1", failover_model) | |
| ckpt["chunked"][rk] = ch | |
| ckpt_path.write_text(json.dumps(ckpt, indent=2), encoding="utf-8") | |
| # 3. Clean Multi-Model Head-to-Head on price_parser (16,495 Beam BPE tokens) with 12,288 max_tokens! | |
| print("\n=== Completing Clean Multi-Model Head-to-Head Benchmark on price_parser ===", flush=True) | |
| spec = REPO_SPECS["price_parser"] | |
| full_files, _ = load_real_repo_files("price_parser") | |
| vectors = build_real_test_vectors("price_parser") | |
| packed_source = "\n\n".join(f"=== FILE: {rel} ===\n{content}" for rel, content in full_files.items()) | |
| sys_prompt = ( | |
| f"Synthesize a SINGLE, COMPLETE, 100% STANDALONE Python module. {MCP_COMPAT_PREAMBLE} " | |
| "Use Python stdlib ONLY. DO NOT import `price_parser` or any forbidden legacy package. " | |
| "Return ONLY Python code inside ```python ... ```." | |
| ) | |
| user_prompt = ( | |
| f"### TARGET REPOSITORY: {spec['name']} (Git commit `{spec['git_sha']}`)\n" | |
| f"### REQUIRED MCP TOOLS SPECIFICATION\n{spec['tools_contract']}\n\n" | |
| f"### REPOSITORY SOURCE CONTEXT\n{packed_source}\n\n" | |
| "Output ONLY the complete standalone Python MCP module in a single ```python ... ``` block." | |
| ) | |
| msgs = [{"role": "system", "content": sys_prompt}, {"role": "user", "content": user_prompt}] | |
| pp_beam_t1 = ckpt["whole_repo"]["price_parser"]["turns"][0] | |
| pp_beam_t2 = ckpt["whole_repo"]["price_parser"]["turns"][1] | |
| h2h: list[dict[str, Any]] = [ | |
| { | |
| "benchmark_repo": "price_parser", | |
| "model": "reflection/beam-1.0-501b-a23b (Turn 1 Zero-Shot)", | |
| "request_id": pp_beam_t1["request_id"], | |
| "provider": pp_beam_t1["provider"], | |
| "latency_ms": pp_beam_t1["latency_ms"], | |
| "prompt_tokens": pp_beam_t1["usage"]["prompt_tokens"], | |
| "completion_tokens": pp_beam_t1["usage"]["completion_tokens"], | |
| "reasoning_tokens": pp_beam_t1["usage"]["reasoning_tokens"], | |
| "total_tokens": pp_beam_t1["usage"]["total_tokens"], | |
| "cost_usd": round(pp_beam_t1["usage"]["cost_usd"], 6), | |
| "zero_shot_passed": pp_beam_t1["passed"], | |
| "total_vectors": pp_beam_t1["total"], | |
| "zero_shot_parity_pct": pp_beam_t1["pass_rate_pct"], | |
| }, | |
| { | |
| "benchmark_repo": "price_parser", | |
| "model": "reflection/beam-1.0-501b-a23b (+Self-Heal Turn 2)", | |
| "request_id": pp_beam_t2["request_id"], | |
| "provider": pp_beam_t2["provider"], | |
| "latency_ms": round(pp_beam_t1["latency_ms"] + pp_beam_t2["latency_ms"], 1), | |
| "prompt_tokens": pp_beam_t1["usage"]["prompt_tokens"] + pp_beam_t2["usage"]["prompt_tokens"], | |
| "completion_tokens": pp_beam_t1["usage"]["completion_tokens"] + pp_beam_t2["usage"]["completion_tokens"], | |
| "reasoning_tokens": pp_beam_t1["usage"]["reasoning_tokens"] + pp_beam_t2["usage"]["reasoning_tokens"], | |
| "total_tokens": pp_beam_t1["usage"]["total_tokens"] + pp_beam_t2["usage"]["total_tokens"], | |
| "cost_usd": round(pp_beam_t1["usage"]["cost_usd"] + pp_beam_t2["usage"]["cost_usd"], 6), | |
| "zero_shot_passed": pp_beam_t2["passed"], | |
| "total_vectors": pp_beam_t2["total"], | |
| "zero_shot_parity_pct": pp_beam_t2["pass_rate_pct"], | |
| }, | |
| ] | |
| for comp_model in ("nvidia/nemotron-3-ultra-550b-a55b:free", "poolside/laguna-s-2.1:free", "cohere/north-mini-code:free"): | |
| print(f" [H2H] Calling {comp_model} with max_tokens=12288 ...", flush=True) | |
| try: | |
| res = call_openrouter(api_key=api_key, model=comp_model, messages=msgs, max_tokens=12288) | |
| code = safe_extract_python(res["content"]) | |
| ev = evaluate_generated_code_subprocess(code, vectors) | |
| entry = { | |
| "benchmark_repo": "price_parser", | |
| "model": comp_model, | |
| "request_id": res["request_id"], | |
| "provider": res["provider"], | |
| "latency_ms": res["latency_ms"], | |
| "prompt_tokens": res["usage"]["prompt_tokens"], | |
| "completion_tokens": res["usage"]["completion_tokens"], | |
| "reasoning_tokens": res["usage"]["reasoning_tokens"], | |
| "total_tokens": res["usage"]["total_tokens"], | |
| "cost_usd": round(res["usage"]["cost_usd"], 6), | |
| "zero_shot_passed": ev["passed"], | |
| "total_vectors": ev["total"], | |
| "zero_shot_parity_pct": ev["pass_rate_pct"], | |
| } | |
| print(f" -> {comp_model}: {ev['passed']}/{ev['total']} ({ev['pass_rate_pct']}%) | tok={entry['completion_tokens']} | {entry['latency_ms']}ms", flush=True) | |
| h2h.append(entry) | |
| except Exception as exc: | |
| print(f" -> {comp_model} ERR: {exc}", flush=True) | |
| ckpt["h2h"] = h2h | |
| ckpt_path.write_text(json.dumps(ckpt, indent=2), encoding="utf-8") | |
| # Build final verifiable_empirical_ledger.json including raw disk repo stats! | |
| ledger = { | |
| "generated_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), | |
| "primary_model": "reflection/beam-1.0-501b-a23b", | |
| "failover_model": failover_model, | |
| "tokenizer_vocab_size": 151681, | |
| "repositories": {}, | |
| "head_to_head_benchmark": h2h, | |
| } | |
| for rk in ("py_moneyed", "price_parser", "schwifty", "python_stdnum"): | |
| wr = ckpt["whole_repo"][rk] | |
| ch = ckpt["chunked"][rk] | |
| raw_stats = get_raw_repo_disk_stats(rk) | |
| chunk_pass_by_idx = {c["index"]: c["passed"] for c in ch.get("case_results", [])} | |
| merged_cases = [{**c, "chunked_rag_passed": chunk_pass_by_idx.get(c["index"], False)} for c in wr.get("case_results", [])] | |
| cert_hash = hashlib.sha256(json.dumps(merged_cases, sort_keys=True).encode("utf-8")).hexdigest() | |
| ledger["repositories"][rk] = { | |
| "spec": {k: v for k, v in REPO_SPECS[rk].items() if k != "tools_contract"}, | |
| "raw_github_disk_stats": raw_stats, | |
| "whole_repo_1m": {k: v for k, v in wr.items() if k not in ("best_code", "case_results")}, | |
| "chunked_top1": {k: v for k, v in ch.items() if k not in ("best_code", "case_results")}, | |
| "certificate_sha256": cert_hash, | |
| "cases": merged_cases, | |
| } | |
| ledger_path = runs_dir / "verifiable_empirical_ledger.json" | |
| ledger_path.write_text(json.dumps(ledger, indent=2), encoding="utf-8") | |
| print(f"\nALL DONE! Saved verified empirical ledger -> {ledger_path}", flush=True) | |
| if __name__ == "__main__": | |
| main() | |