Spaces:
Running
Running
Self-healing text-to-SQL: landing page + full source
Browse files- README.md +73 -5
- index.html +126 -17
- src/selfheal_sql/audit_eval.py +107 -0
- src/selfheal_sql/build_hf_dataset.py +105 -0
- src/selfheal_sql/execcheck.py +152 -0
- src/selfheal_sql/gate_decision.py +115 -0
- src/selfheal_sql/heal.py +270 -0
- src/selfheal_sql/launch_job.py +76 -0
- src/selfheal_sql/make_gate_large.py +101 -0
- src/selfheal_sql/make_splits.py +123 -0
- src/selfheal_sql/metrics.py +102 -0
- src/selfheal_sql/student.py +149 -0
- src/selfheal_sql/train_dpo.py +211 -0
- third_party/agenttune_gate/PROVENANCE.txt +3 -0
- third_party/agenttune_gate/deployment_gate.py +447 -0
README.md
CHANGED
|
@@ -1,10 +1,78 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: static
|
| 7 |
pinned: false
|
|
|
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Self-Healing Text-to-SQL
|
| 3 |
+
emoji: 🩹
|
| 4 |
+
colorFrom: red
|
| 5 |
+
colorTo: yellow
|
| 6 |
sdk: static
|
| 7 |
pinned: false
|
| 8 |
+
license: cc-by-sa-4.0
|
| 9 |
---
|
| 10 |
|
| 11 |
+
# Self-Healing Text-to-SQL (Lexsi Hackathon, Track 3 — PS #8)
|
| 12 |
+
|
| 13 |
+
A fully automated self-healing loop for a **Tiny Aya Global (3.35B)** text-to-SQL
|
| 14 |
+
agent on **Spider**: detect failing runs by *executing* the SQL, classify root causes,
|
| 15 |
+
generate **execution-verified** corrective training pairs with **Aya Expanse 32B**,
|
| 16 |
+
retrain adapter-only **DPO** through **AgentTune**, and gate redeployment with
|
| 17 |
+
**AgentTune's DeploymentGate**. Outcomes independently verified with **AuditKIT**.
|
| 18 |
+
|
| 19 |
+
```
|
| 20 |
+
student (Tiny Aya) writes SQL ──▶ execcheck runs it vs gold ──▶ failures
|
| 21 |
+
▲ │
|
| 22 |
+
│ classify (SQL taxonomy, Aya-32B)
|
| 23 |
+
DPO LoRA adapter │
|
| 24 |
+
(AgentTune run_retrain) generate fix (Aya-32B, never sees gold)
|
| 25 |
+
▲ │
|
| 26 |
+
│ execute fix ─ correct? keep : retry ≤3
|
| 27 |
+
158 verified pairs ◀──────────────────────────────────────────────┘
|
| 28 |
+
│
|
| 29 |
+
DeploymentGate (AgentTune, held-out gate split) ──▶ deploy / keep old
|
| 30 |
+
│
|
| 31 |
+
final report on unseen test split (+ AuditKIT re-verification)
|
| 32 |
+
```
|
| 33 |
+
|
| 34 |
+
## Layout
|
| 35 |
+
|
| 36 |
+
| path | what |
|
| 37 |
+
|---|---|
|
| 38 |
+
| `src/selfheal_sql/execcheck.py` | Execution checker — the single source of truth for pass/fail |
|
| 39 |
+
| `src/selfheal_sql/make_splits.py` | Database-disjoint pool/gate/test splits from Spider |
|
| 40 |
+
| `src/selfheal_sql/student.py` | Student harness: prompt → SQL → execute → grade → log |
|
| 41 |
+
| `src/selfheal_sql/heal.py` | Classify failures + generate execution-verified DPO pairs |
|
| 42 |
+
| `src/selfheal_sql/train_dpo.py` | Adapter-only DPO via AgentTune `RetrainConfig`/`build_retrainer` |
|
| 43 |
+
| `src/selfheal_sql/gate_decision.py` | AgentTune `DeploymentGate.evaluate_decision` on the gate split |
|
| 44 |
+
| `src/selfheal_sql/audit_eval.py` | AuditKIT `agent_eval` with a fresh-execution outcome oracle |
|
| 45 |
+
| `src/selfheal_sql/metrics.py` | Before/after report (accuracy, heal/regression rate, McNemar) |
|
| 46 |
+
| `src/selfheal_sql/launch_job.py` | Runs any of the above on HF Jobs GPUs |
|
| 47 |
+
| `src/selfheal_sql/build_hf_dataset.py` | Packages splits + SQLite DBs to the HF dataset repo |
|
| 48 |
+
| `third_party/agenttune_gate/` | AgentTune's DeploymentGate vendored verbatim (provenance inside) |
|
| 49 |
+
| `writeup/` | HF Article draft |
|
| 50 |
+
|
| 51 |
+
## Hugging Face artifacts
|
| 52 |
+
|
| 53 |
+
- Dataset (splits + 149 Spider DBs): `HideIron/selfheal-sql-spider`
|
| 54 |
+
- Every run log, heal pairs, gate decision, metrics: `HideIron/selfheal-sql-runs`
|
| 55 |
+
- DPO adapter: `HideIron/tiny-aya-global-sql-selfheal`
|
| 56 |
+
|
| 57 |
+
## Reproduce
|
| 58 |
+
|
| 59 |
+
```bash
|
| 60 |
+
# 1. splits (needs the official Spider download)
|
| 61 |
+
python src/selfheal_sql/make_splits.py <spider_data_dir> data/splits 600 100 500 0
|
| 62 |
+
# 2. baseline on an HF Jobs A10G
|
| 63 |
+
python src/selfheal_sql/launch_job.py --name baseline-pool -- student.py --split pool
|
| 64 |
+
# 3. heal (local; needs HF_TOKEN for the Aya-32B router calls)
|
| 65 |
+
python src/selfheal_sql/heal.py --split-file data/splits/pool.jsonl --out data/heal
|
| 66 |
+
# 4. DPO retrain (AgentTune on an A10G)
|
| 67 |
+
python src/selfheal_sql/launch_job.py --name dpo-full --image python:3.12 \
|
| 68 |
+
--pip "agenttune huggingface_hub" -- train_dpo.py
|
| 69 |
+
# 5. healed evals, gate, metrics
|
| 70 |
+
python src/selfheal_sql/launch_job.py --name healed-gate -- student.py --split gate \
|
| 71 |
+
--adapter HideIron/tiny-aya-global-sql-selfheal
|
| 72 |
+
python src/selfheal_sql/gate_decision.py
|
| 73 |
+
python src/selfheal_sql/metrics.py --pair test:baseline-test:healed-test
|
| 74 |
+
```
|
| 75 |
+
|
| 76 |
+
Models: Tiny Aya Global (student) and Aya Expanse 32B (healer) — both open-weights
|
| 77 |
+
Cohere Labs models from the hackathon's allowed list. Spider is CC BY-SA 4.0.
|
| 78 |
+
AgentTune/AuditKIT are Lexsi Labs (LSAL / their respective licenses).
|
index.html
CHANGED
|
@@ -1,19 +1,128 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html>
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
</html>
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 6 |
+
<title>Self-Healing Text-to-SQL</title>
|
| 7 |
+
<style>
|
| 8 |
+
:root{
|
| 9 |
+
--bg:#f7f6f3; --card:#ffffff; --ink:#1a1a1a; --muted:#5f5c57; --line:#e4e1db;
|
| 10 |
+
--accent:#b4552d; --good:#2d7a4f; --bad:#a63d3d; --code:#f0eee9;
|
| 11 |
+
}
|
| 12 |
+
@media (prefers-color-scheme: dark){ :root:not([data-theme="light"]){
|
| 13 |
+
--bg:#171614; --card:#201f1c; --ink:#ece9e4; --muted:#a39e96; --line:#35332e;
|
| 14 |
+
--accent:#e08050; --good:#5fbf8a; --bad:#d97f7f; --code:#2a2925;
|
| 15 |
+
}}
|
| 16 |
+
:root[data-theme="dark"]{
|
| 17 |
+
--bg:#171614; --card:#201f1c; --ink:#ece9e4; --muted:#a39e96; --line:#35332e;
|
| 18 |
+
--accent:#e08050; --good:#5fbf8a; --bad:#d97f7f; --code:#2a2925;
|
| 19 |
+
}
|
| 20 |
+
*{box-sizing:border-box}
|
| 21 |
+
body{margin:0;background:var(--bg);color:var(--ink);
|
| 22 |
+
font:16px/1.6 Georgia,'Times New Roman',serif;}
|
| 23 |
+
.wrap{max-width:860px;margin:0 auto;padding:48px 16px 80px}
|
| 24 |
+
h1{font-size:2rem;line-height:1.2;margin:0 0 4px}
|
| 25 |
+
h2{font-size:1.25rem;margin:40px 0 12px;border-bottom:1px solid var(--line);padding-bottom:6px}
|
| 26 |
+
.sub{color:var(--muted);margin:0 0 28px}
|
| 27 |
+
.badge{display:inline-block;background:var(--accent);color:#fff;border-radius:4px;
|
| 28 |
+
padding:1px 8px;font:600 .75rem/1.6 system-ui,sans-serif;vertical-align:middle}
|
| 29 |
+
p{margin:12px 0}
|
| 30 |
+
pre{background:var(--code);border:1px solid var(--line);border-radius:8px;
|
| 31 |
+
padding:14px;overflow-x:auto;font:13px/1.5 ui-monospace,Consolas,monospace}
|
| 32 |
+
table{border-collapse:collapse;width:100%;font:14px/1.5 system-ui,sans-serif;margin:16px 0}
|
| 33 |
+
th,td{border:1px solid var(--line);padding:7px 10px;text-align:left}
|
| 34 |
+
th{background:var(--code)}
|
| 35 |
+
td.num{text-align:right;font-variant-numeric:tabular-nums}
|
| 36 |
+
.good{color:var(--good);font-weight:600}.bad{color:var(--bad);font-weight:600}
|
| 37 |
+
a{color:var(--accent)}
|
| 38 |
+
ul{padding-left:22px}
|
| 39 |
+
.card{background:var(--card);border:1px solid var(--line);border-radius:10px;
|
| 40 |
+
padding:4px 18px;margin:18px 0}
|
| 41 |
+
.foot{color:var(--muted);font-size:.85rem;margin-top:48px;border-top:1px solid var(--line);padding-top:14px}
|
| 42 |
+
</style>
|
| 43 |
+
</head>
|
| 44 |
+
<body>
|
| 45 |
+
<div class="wrap">
|
| 46 |
+
<h1>Self-Healing Text-to-SQL <span class="badge">PS #8</span></h1>
|
| 47 |
+
<p class="sub">Lexsi Labs Hackathon · Track 3 (Agentic Fine-tuning) · AgentTune + AuditKIT ·
|
| 48 |
+
Cohere Labs open models</p>
|
| 49 |
+
|
| 50 |
+
<p>A fully automated self-healing loop for a <strong>Tiny Aya Global (3.35B)</strong>
|
| 51 |
+
text-to-SQL agent on Spider: failures are detected by <em>executing</em> the SQL,
|
| 52 |
+
classified into SQL root causes, corrected by <strong>Aya Expanse 32B</strong>
|
| 53 |
+
(every fix verified by execution before it becomes training data — the healer never
|
| 54 |
+
sees the gold query), retrained adapter-only through <strong>AgentTune</strong>, and
|
| 55 |
+
deployment is decided by <strong>AgentTune's DeploymentGate</strong> on a held-out
|
| 56 |
+
split. Outcomes independently re-verified with <strong>AuditKIT</strong>.
|
| 57 |
+
No human wrote or fixed a single SQL query.</p>
|
| 58 |
+
|
| 59 |
+
<h2>The loop</h2>
|
| 60 |
+
<pre>
|
| 61 |
+
student (Tiny Aya 3.35B) writes SQL ──▶ execution checker runs it vs gold ──▶ failures
|
| 62 |
+
▲ │
|
| 63 |
+
│ classify (SQL taxonomy, Aya Expanse 32B)
|
| 64 |
+
LoRA adapter (AgentTune │
|
| 65 |
+
DPO / BCO retrain) generate fix (Aya Expanse 32B, never sees gold)
|
| 66 |
+
▲ │
|
| 67 |
+
│ execute fix ── correct? keep : retry ≤3
|
| 68 |
+
158 execution-verified pairs ◀─────────────────────────────────────────────────┘
|
| 69 |
+
│
|
| 70 |
+
AgentTune DeploymentGate (held-out gate split) ──▶ deploy / keep old
|
| 71 |
+
│
|
| 72 |
+
final report on unseen test split (+ AuditKIT re-verification)
|
| 73 |
+
</pre>
|
| 74 |
+
|
| 75 |
+
<h2>Headline results</h2>
|
| 76 |
+
<div class="card">
|
| 77 |
+
<p><strong>The loop ran end-to-end — and the gate did its job.</strong>
|
| 78 |
+
158 of 253 pool failures (62.5%) were converted into execution-verified corrective
|
| 79 |
+
training pairs. Retraining consistently <em>regressed</em> held-out accuracy
|
| 80 |
+
(a documented DPO failure mode on near-identical SQL pairs, plus catastrophic
|
| 81 |
+
forgetting), and the DeploymentGate correctly <span class="bad">BLOCKED</span> every
|
| 82 |
+
candidate adapter, keeping the baseline deployed. The best candidate
|
| 83 |
+
(BCO + success-replay) recovered to within 2pp of baseline.</p>
|
| 84 |
+
</div>
|
| 85 |
+
|
| 86 |
+
<table>
|
| 87 |
+
<tr><th>Stage</th><th>Metric</th><th class="num">Value</th></tr>
|
| 88 |
+
<tr><td>Baseline, pool (600)</td><td>execution accuracy</td><td class="num">57.8%</td></tr>
|
| 89 |
+
<tr><td>Baseline, gate (100)</td><td>execution accuracy</td><td class="num">62.0%</td></tr>
|
| 90 |
+
<tr><td>Baseline, <strong>test (500, unseen)</strong></td><td>execution accuracy</td><td class="num"><strong>45.6%</strong></td></tr>
|
| 91 |
+
<tr><td>Heal stage</td><td>verified-fix yield</td><td class="num good">158 / 253 (62.5%)</td></tr>
|
| 92 |
+
<tr><td>DPO adapter, gate</td><td>execution accuracy</td><td class="num">58.0% <span class="bad">BLOCKED</span></td></tr>
|
| 93 |
+
<tr><td>BCO + replay adapter, gate</td><td>execution accuracy</td><td class="num">60.0% <span class="bad">BLOCKED</span></td></tr>
|
| 94 |
+
<tr><td>AuditKIT re-verification, test</td><td>agreement with harness</td><td class="num good">500 / 500</td></tr>
|
| 95 |
+
</table>
|
| 96 |
+
<p>Full before/after tables (valid-SQL rates, per-status breakdowns, McNemar exact
|
| 97 |
+
tests, 400-question enlarged gate) are in the article and the runs repo below.</p>
|
| 98 |
+
|
| 99 |
+
<h2>Artifacts</h2>
|
| 100 |
+
<ul>
|
| 101 |
+
<li>Dataset — splits + 149 Spider DBs:
|
| 102 |
+
<a href="https://huggingface.co/datasets/HideIron/selfheal-sql-spider">HideIron/selfheal-sql-spider</a></li>
|
| 103 |
+
<li>Every run log, heal pairs, gate decisions, metrics:
|
| 104 |
+
<a href="https://huggingface.co/datasets/HideIron/selfheal-sql-runs">HideIron/selfheal-sql-runs</a></li>
|
| 105 |
+
<li>Adapters:
|
| 106 |
+
<a href="https://huggingface.co/HideIron/tiny-aya-global-sql-selfheal">DPO</a> ·
|
| 107 |
+
<a href="https://huggingface.co/HideIron/tiny-aya-global-sql-selfheal-bco">BCO</a> ·
|
| 108 |
+
<a href="https://huggingface.co/HideIron/tiny-aya-global-sql-selfheal-bco-replay">BCO + replay</a></li>
|
| 109 |
+
<li>Code — browse the <em>Files</em> tab of this Space (<code>src/selfheal_sql/</code>);
|
| 110 |
+
reproduce instructions in <code>README.md</code></li>
|
| 111 |
+
</ul>
|
| 112 |
+
|
| 113 |
+
<h2>Why the gate blocking is the point</h2>
|
| 114 |
+
<p>PS #8 asks for redeployment <em>gated on measured accuracy</em>. Preference-tuning
|
| 115 |
+
on minimal-edit SQL pairs hits <em>likelihood displacement</em> (chosen and rejected
|
| 116 |
+
share ~90% of tokens; pushing the rejected down drags the chosen with it) and training
|
| 117 |
+
only on failures causes forgetting of the 347 already-correct answers. The gate caught
|
| 118 |
+
every resulting regression on held-out data and refused to ship — exactly what a
|
| 119 |
+
production self-healing system must do. The analysis, and the BCO + success-replay
|
| 120 |
+
remedy that recovers most of the gap, are in the article.</p>
|
| 121 |
+
|
| 122 |
+
<div class="foot">Solo project · models: Tiny Aya Global (student) and Aya Expanse 32B
|
| 123 |
+
(healer), both open-weights Cohere Labs models from the hackathon's allowed list ·
|
| 124 |
+
Spider is CC BY-SA 4.0 · AgentTune/AuditKIT © Lexsi Labs (DeploymentGate vendored
|
| 125 |
+
verbatim @ 1dfd054, provenance in <code>third_party/</code>).</div>
|
| 126 |
+
</div>
|
| 127 |
+
</body>
|
| 128 |
</html>
|
src/selfheal_sql/audit_eval.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""AuditKIT agent_eval over a student run: independent outcome verification.
|
| 2 |
+
|
| 3 |
+
Each logged run row becomes a recorded AgentEpisode (user prompt -> assistant raw
|
| 4 |
+
output). The outcome oracle is a CustomPredicate that RE-EXECUTES the model's SQL
|
| 5 |
+
against the real database and compares with the gold result — AuditKIT's verdict is a
|
| 6 |
+
fresh execution, not a read of our logged status. We then cross-check AuditKIT's
|
| 7 |
+
verdicts against the harness log (they must agree 100%).
|
| 8 |
+
|
| 9 |
+
Usage: python audit_eval.py --run baseline-test --split-file ../../data/splits/test.jsonl
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import argparse
|
| 15 |
+
import json
|
| 16 |
+
import os
|
| 17 |
+
import sys
|
| 18 |
+
from collections import Counter
|
| 19 |
+
|
| 20 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 21 |
+
sys.path.insert(0, HERE)
|
| 22 |
+
from execcheck import check # noqa: E402
|
| 23 |
+
from student import PROMPT # noqa: E402
|
| 24 |
+
|
| 25 |
+
RUNS_REPO = "HideIron/selfheal-sql-runs"
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def main() -> None:
|
| 29 |
+
ap = argparse.ArgumentParser()
|
| 30 |
+
ap.add_argument("--run", required=True)
|
| 31 |
+
ap.add_argument("--split-file", required=True)
|
| 32 |
+
ap.add_argument("--out", default=None)
|
| 33 |
+
ap.add_argument("--no-push", action="store_true")
|
| 34 |
+
args = ap.parse_args()
|
| 35 |
+
|
| 36 |
+
from huggingface_hub import HfApi, hf_hub_download
|
| 37 |
+
|
| 38 |
+
from auditkit.agent_eval import (
|
| 39 |
+
AgentCase,
|
| 40 |
+
AgentEvalRunner,
|
| 41 |
+
AgentEvalSpec,
|
| 42 |
+
CustomPredicate,
|
| 43 |
+
episode_from_openai_messages,
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
rp = hf_hub_download(RUNS_REPO, f"runs/{args.run}/results.jsonl", repo_type="dataset")
|
| 47 |
+
rows = [json.loads(l) for l in open(rp, encoding="utf-8")]
|
| 48 |
+
splits = {r["qid"]: r for r in map(json.loads, open(args.split_file, encoding="utf-8"))}
|
| 49 |
+
|
| 50 |
+
def exec_predicate(episode, case):
|
| 51 |
+
s = splits[case.id]
|
| 52 |
+
res = check(s["db_path"], episode.final_answer or "", s["gold_sql"])
|
| 53 |
+
return bool(res.correct)
|
| 54 |
+
|
| 55 |
+
episodes, cases = [], []
|
| 56 |
+
for r in rows:
|
| 57 |
+
s = splits[r["qid"]]
|
| 58 |
+
episodes.append(
|
| 59 |
+
episode_from_openai_messages(
|
| 60 |
+
[
|
| 61 |
+
{"role": "user",
|
| 62 |
+
"content": PROMPT.format(schema=s["schema"], question=s["question"])},
|
| 63 |
+
{"role": "assistant", "content": r["raw_output"]},
|
| 64 |
+
],
|
| 65 |
+
case_id=r["qid"],
|
| 66 |
+
final_answer=r["raw_output"],
|
| 67 |
+
)
|
| 68 |
+
)
|
| 69 |
+
cases.append(
|
| 70 |
+
AgentCase(
|
| 71 |
+
id=r["qid"], task=s["question"], category="text_to_sql",
|
| 72 |
+
outcome=CustomPredicate(exec_predicate, version="execcheck-v1",
|
| 73 |
+
name="spider_execution_accuracy"),
|
| 74 |
+
metadata={"db_id": s["db_id"]},
|
| 75 |
+
)
|
| 76 |
+
)
|
| 77 |
+
|
| 78 |
+
spec = AgentEvalSpec(cases=cases, mode="recorded", episodes=episodes,
|
| 79 |
+
scorers=[], trials=1)
|
| 80 |
+
result = AgentEvalRunner().run(spec)
|
| 81 |
+
|
| 82 |
+
verdicts = {row.case_id: row.outcome["verdict"] for row in result.rows}
|
| 83 |
+
logged = {r["qid"]: ("success" if r["status"] == "correct" else "failure") for r in rows}
|
| 84 |
+
agree = sum(verdicts[q] == logged[q] for q in verdicts)
|
| 85 |
+
out = {
|
| 86 |
+
"run": args.run,
|
| 87 |
+
"auditkit_summary": result.summary(),
|
| 88 |
+
"verdict_counts": dict(Counter(verdicts.values())),
|
| 89 |
+
"n": len(verdicts),
|
| 90 |
+
"success_rate": round(sum(v == "success" for v in verdicts.values()) / len(verdicts), 4),
|
| 91 |
+
"crosscheck_agree_with_harness": f"{agree}/{len(verdicts)}",
|
| 92 |
+
"oracle": "CustomPredicate(spider_execution_accuracy, version=execcheck-v1) — fresh re-execution",
|
| 93 |
+
}
|
| 94 |
+
out_dir = args.out or os.path.join(HERE, "..", "..", "data", "audit", args.run)
|
| 95 |
+
os.makedirs(out_dir, exist_ok=True)
|
| 96 |
+
with open(os.path.join(out_dir, "auditkit_report.json"), "w", encoding="utf-8") as f:
|
| 97 |
+
json.dump(out, f, indent=2, default=str)
|
| 98 |
+
print(json.dumps(out, indent=2, default=str))
|
| 99 |
+
|
| 100 |
+
if not args.no_push:
|
| 101 |
+
HfApi().upload_folder(repo_id=RUNS_REPO, repo_type="dataset", folder_path=out_dir,
|
| 102 |
+
path_in_repo=f"audit/{args.run}",
|
| 103 |
+
commit_message=f"auditkit report {args.run}")
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
if __name__ == "__main__":
|
| 107 |
+
main()
|
src/selfheal_sql/build_hf_dataset.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Stage and upload the splits + the SQLite databases they use to a private HF dataset repo.
|
| 2 |
+
|
| 3 |
+
Rows are rewritten so db_path is RELATIVE to the dataset root
|
| 4 |
+
(databases/<db_id>/<db_id>.sqlite): the same rows then work on the laptop and inside an
|
| 5 |
+
HF Job that mounts the repo.
|
| 6 |
+
|
| 7 |
+
Usage: python build_hf_dataset.py <repo_id> [--no-upload]
|
| 8 |
+
Token is read from the HF_TOKEN environment variable (never from a file).
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
import json
|
| 14 |
+
import os
|
| 15 |
+
import shutil
|
| 16 |
+
import sys
|
| 17 |
+
|
| 18 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 19 |
+
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
| 20 |
+
SPLITS = os.path.join(ROOT, "data", "splits")
|
| 21 |
+
STAGE = os.path.join(ROOT, "data", "hf_dataset")
|
| 22 |
+
|
| 23 |
+
CARD = """---
|
| 24 |
+
license: cc-by-sa-4.0
|
| 25 |
+
task_categories:
|
| 26 |
+
- text-generation
|
| 27 |
+
language:
|
| 28 |
+
- en
|
| 29 |
+
tags:
|
| 30 |
+
- text-to-sql
|
| 31 |
+
- spider
|
| 32 |
+
- self-healing
|
| 33 |
+
pretty_name: Self-heal SQL - Spider splits
|
| 34 |
+
---
|
| 35 |
+
|
| 36 |
+
# Self-heal SQL: database-disjoint Spider splits
|
| 37 |
+
|
| 38 |
+
Splits and SQLite databases used for a self-healing text-to-SQL agent
|
| 39 |
+
(detect failures -> classify -> generate corrective examples -> DPO retrain -> gate).
|
| 40 |
+
|
| 41 |
+
Derived from **Spider** (Yu et al., 2018, https://yale-lily.github.io/spider), CC BY-SA 4.0.
|
| 42 |
+
|
| 43 |
+
| split | source | purpose |
|
| 44 |
+
|---|---|---|
|
| 45 |
+
| `pool` | Spider train, DB group A | failure pool; its failures become training data |
|
| 46 |
+
| `gate` | Spider train, DB group B | held-out deployment gate; never used to build training data |
|
| 47 |
+
| `test` | Spider dev | headline before/after numbers |
|
| 48 |
+
|
| 49 |
+
No database appears in more than one split (asserted at build time). Rows whose gold query
|
| 50 |
+
errors, times out or returns zero rows were dropped.
|
| 51 |
+
|
| 52 |
+
Row fields: `qid, split, db_id, question, gold_sql, db_path, schema`.
|
| 53 |
+
`db_path` is relative to the repo root.
|
| 54 |
+
"""
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def main() -> None:
|
| 58 |
+
repo_id = sys.argv[1]
|
| 59 |
+
upload = "--no-upload" not in sys.argv
|
| 60 |
+
|
| 61 |
+
if os.path.isdir(STAGE):
|
| 62 |
+
shutil.rmtree(STAGE)
|
| 63 |
+
os.makedirs(os.path.join(STAGE, "splits"))
|
| 64 |
+
|
| 65 |
+
used: dict[str, str] = {}
|
| 66 |
+
for name in ("pool", "gate", "test"):
|
| 67 |
+
rows = []
|
| 68 |
+
with open(os.path.join(SPLITS, f"{name}.jsonl"), encoding="utf-8") as f:
|
| 69 |
+
for line in f:
|
| 70 |
+
r = json.loads(line)
|
| 71 |
+
used[r["db_id"]] = r["db_path"]
|
| 72 |
+
r["db_path"] = f"databases/{r['db_id']}/{r['db_id']}.sqlite"
|
| 73 |
+
rows.append(r)
|
| 74 |
+
with open(os.path.join(STAGE, "splits", f"{name}.jsonl"), "w", encoding="utf-8") as f:
|
| 75 |
+
for r in rows:
|
| 76 |
+
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
| 77 |
+
print(f"{name}: {len(rows)} rows")
|
| 78 |
+
|
| 79 |
+
total = 0
|
| 80 |
+
for db_id, src in used.items():
|
| 81 |
+
dst_dir = os.path.join(STAGE, "databases", db_id)
|
| 82 |
+
os.makedirs(dst_dir)
|
| 83 |
+
dst = os.path.join(dst_dir, f"{db_id}.sqlite")
|
| 84 |
+
shutil.copyfile(src, dst)
|
| 85 |
+
total += os.path.getsize(dst)
|
| 86 |
+
print(f"{len(used)} databases, {total / 2**20:.1f} MiB")
|
| 87 |
+
|
| 88 |
+
with open(os.path.join(STAGE, "README.md"), "w", encoding="utf-8") as f:
|
| 89 |
+
f.write(CARD)
|
| 90 |
+
|
| 91 |
+
if not upload:
|
| 92 |
+
print("staged at", STAGE, "(not uploaded)")
|
| 93 |
+
return
|
| 94 |
+
|
| 95 |
+
from huggingface_hub import HfApi
|
| 96 |
+
|
| 97 |
+
api = HfApi(token=os.environ["HF_TOKEN"])
|
| 98 |
+
api.create_repo(repo_id, repo_type="dataset", private=True, exist_ok=True)
|
| 99 |
+
api.upload_folder(repo_id=repo_id, repo_type="dataset", folder_path=STAGE,
|
| 100 |
+
commit_message="Spider-derived DB-disjoint splits + databases")
|
| 101 |
+
print("uploaded: https://huggingface.co/datasets/" + repo_id, "(private)")
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
if __name__ == "__main__":
|
| 105 |
+
main()
|
src/selfheal_sql/execcheck.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Execution checker for text-to-SQL: the single source of truth for pass/fail.
|
| 2 |
+
|
| 3 |
+
A predicted query is CORRECT iff executing it on the question's SQLite database
|
| 4 |
+
returns the same result as the gold query (Spider-style execution accuracy).
|
| 5 |
+
|
| 6 |
+
Outcome statuses (feed the self-heal Failure objects):
|
| 7 |
+
correct - result matches gold
|
| 8 |
+
wrong_result - ran fine, result differs from gold
|
| 9 |
+
sql_error - sqlite raised (bad column/table/syntax ...); error_message holds it
|
| 10 |
+
timeout - exceeded the time budget (likely a runaway / cartesian join)
|
| 11 |
+
empty_sql - nothing usable was extracted from the model output
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import re
|
| 17 |
+
import sqlite3
|
| 18 |
+
import time
|
| 19 |
+
from dataclasses import asdict, dataclass
|
| 20 |
+
|
| 21 |
+
TIMEOUT_S = 10.0
|
| 22 |
+
MAX_ROWS = 100_000 # guard against pathological result sets
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
@dataclass
|
| 26 |
+
class ExecResult:
|
| 27 |
+
status: str # correct | wrong_result | sql_error | timeout | empty_sql
|
| 28 |
+
error_message: str = ""
|
| 29 |
+
pred_rows: int | None = None
|
| 30 |
+
gold_rows: int | None = None
|
| 31 |
+
|
| 32 |
+
@property
|
| 33 |
+
def correct(self) -> bool:
|
| 34 |
+
return self.status == "correct"
|
| 35 |
+
|
| 36 |
+
def to_dict(self) -> dict:
|
| 37 |
+
return asdict(self)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
_FENCE = re.compile(r"```(?:sql)?\s*(.*?)```", re.DOTALL | re.IGNORECASE)
|
| 41 |
+
_THINK = re.compile(r"<think>.*?</think>", re.DOTALL | re.IGNORECASE)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def extract_sql(text: str) -> str:
|
| 45 |
+
"""Pull a SQL statement out of raw model output.
|
| 46 |
+
|
| 47 |
+
Handles <think> blocks, ```sql fences, and a trailing explanation. Returns ""
|
| 48 |
+
if nothing that looks like SQL is found.
|
| 49 |
+
"""
|
| 50 |
+
if not text:
|
| 51 |
+
return ""
|
| 52 |
+
text = _THINK.sub("", text)
|
| 53 |
+
m = _FENCE.search(text)
|
| 54 |
+
if m:
|
| 55 |
+
text = m.group(1)
|
| 56 |
+
text = text.strip()
|
| 57 |
+
# Start at the first SELECT/WITH; drop anything before it.
|
| 58 |
+
m = re.search(r"\b(select|with)\b", text, re.IGNORECASE)
|
| 59 |
+
if not m:
|
| 60 |
+
return ""
|
| 61 |
+
text = text[m.start():]
|
| 62 |
+
# Keep only the first statement.
|
| 63 |
+
text = text.split(";")[0].strip()
|
| 64 |
+
return text
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _connect(db_path: str) -> sqlite3.Connection:
|
| 68 |
+
# Read-only: a model-written query must never modify the benchmark databases.
|
| 69 |
+
uri = "file:" + db_path.replace("\\", "/") + "?mode=ro"
|
| 70 |
+
conn = sqlite3.connect(uri, uri=True)
|
| 71 |
+
conn.text_factory = lambda b: b.decode("utf-8", errors="ignore")
|
| 72 |
+
return conn
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _run(conn: sqlite3.Connection, sql: str, timeout_s: float):
|
| 76 |
+
deadline = time.monotonic() + timeout_s
|
| 77 |
+
|
| 78 |
+
def _abort() -> int:
|
| 79 |
+
return 1 if time.monotonic() > deadline else 0
|
| 80 |
+
|
| 81 |
+
conn.set_progress_handler(_abort, 10_000)
|
| 82 |
+
try:
|
| 83 |
+
cur = conn.execute(sql)
|
| 84 |
+
rows = cur.fetchmany(MAX_ROWS)
|
| 85 |
+
finally:
|
| 86 |
+
conn.set_progress_handler(None, 0)
|
| 87 |
+
return rows
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _norm_cell(v):
|
| 91 |
+
if isinstance(v, float):
|
| 92 |
+
return round(v, 6)
|
| 93 |
+
return v
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def _norm_rows(rows) -> list[tuple]:
|
| 97 |
+
return [tuple(_norm_cell(c) for c in r) for r in rows]
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def _results_match(pred, gold, ordered: bool) -> bool:
|
| 101 |
+
pred, gold = _norm_rows(pred), _norm_rows(gold)
|
| 102 |
+
if ordered:
|
| 103 |
+
return pred == gold
|
| 104 |
+
key = lambda r: tuple(repr(c) for c in r) # noqa: E731 (mixed types sort safely)
|
| 105 |
+
return sorted(pred, key=key) == sorted(gold, key=key)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def check(db_path: str, pred_sql: str, gold_sql: str, timeout_s: float = TIMEOUT_S) -> ExecResult:
|
| 109 |
+
"""Execute pred and gold on db_path and compare. Never raises."""
|
| 110 |
+
# Always extract: strips fences/<think>/trailing prose and keeps only the FIRST
|
| 111 |
+
# statement (a model that appends "; <explanation>" must not be marked wrong).
|
| 112 |
+
pred_sql = extract_sql(pred_sql)
|
| 113 |
+
if not pred_sql:
|
| 114 |
+
return ExecResult("empty_sql", "no SQL found in model output")
|
| 115 |
+
|
| 116 |
+
try:
|
| 117 |
+
conn = _connect(db_path)
|
| 118 |
+
except Exception as e: # noqa: BLE001
|
| 119 |
+
return ExecResult("sql_error", f"cannot open database: {e}")
|
| 120 |
+
|
| 121 |
+
try:
|
| 122 |
+
try:
|
| 123 |
+
gold = _run(conn, gold_sql, timeout_s * 3)
|
| 124 |
+
except Exception as e: # noqa: BLE001
|
| 125 |
+
return ExecResult("sql_error", f"GOLD query failed (bad benchmark row): {e}")
|
| 126 |
+
try:
|
| 127 |
+
pred = _run(conn, pred_sql, timeout_s)
|
| 128 |
+
except sqlite3.OperationalError as e:
|
| 129 |
+
msg = str(e)
|
| 130 |
+
if "interrupted" in msg.lower():
|
| 131 |
+
return ExecResult("timeout", f"query exceeded {timeout_s}s", None, len(gold))
|
| 132 |
+
return ExecResult("sql_error", msg, None, len(gold))
|
| 133 |
+
except Exception as e: # noqa: BLE001
|
| 134 |
+
return ExecResult("sql_error", f"{type(e).__name__}: {e}", None, len(gold))
|
| 135 |
+
|
| 136 |
+
ordered = bool(re.search(r"\border\s+by\b", gold_sql, re.IGNORECASE))
|
| 137 |
+
ok = _results_match(pred, gold, ordered)
|
| 138 |
+
return ExecResult("correct" if ok else "wrong_result", "", len(pred), len(gold))
|
| 139 |
+
finally:
|
| 140 |
+
conn.close()
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def schema_text(db_path: str) -> str:
|
| 144 |
+
"""CREATE TABLE statements for the prompt (the schema the agent is shown)."""
|
| 145 |
+
conn = _connect(db_path)
|
| 146 |
+
try:
|
| 147 |
+
rows = conn.execute(
|
| 148 |
+
"SELECT sql FROM sqlite_master WHERE type='table' AND sql IS NOT NULL ORDER BY name"
|
| 149 |
+
).fetchall()
|
| 150 |
+
finally:
|
| 151 |
+
conn.close()
|
| 152 |
+
return "\n".join(r[0].strip() + ";" for r in rows)
|
src/selfheal_sql/gate_decision.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Deployment gate: decide whether the retrained adapter ships, using AgentTune's
|
| 2 |
+
DeploymentGate verbatim (vendored at third_party/agenttune_gate, commit 1dfd054).
|
| 3 |
+
|
| 4 |
+
The gate is model-agnostic by design: `model_runner_fn(input_text) -> verdict` is
|
| 5 |
+
supplied by the caller and compared against `expected_verdict` per case ("keeps the
|
| 6 |
+
gate model-agnostic and testable without a GPU" — their docstring). Our runners look
|
| 7 |
+
up each gate question's execution-checker outcome from the student harness's logged
|
| 8 |
+
runs, so the verdict IS Spider execution accuracy.
|
| 9 |
+
|
| 10 |
+
Usage:
|
| 11 |
+
python gate_decision.py --old baseline-gate --new healed-gate --out ../../data/gate
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import argparse
|
| 17 |
+
import importlib.util
|
| 18 |
+
import json
|
| 19 |
+
import os
|
| 20 |
+
import sys
|
| 21 |
+
import types
|
| 22 |
+
from dataclasses import asdict
|
| 23 |
+
|
| 24 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 25 |
+
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
| 26 |
+
RUNS_REPO = "HideIron/selfheal-sql-runs"
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def load_gate_module():
|
| 30 |
+
"""Import the vendored deployment_gate.py without installing agenttune
|
| 31 |
+
(its only non-stdlib import is the PASS_VERDICTS constant, stubbed here 1:1)."""
|
| 32 |
+
pkg_names = ["agenttune", "agenttune.decide"]
|
| 33 |
+
for n in pkg_names:
|
| 34 |
+
if n not in sys.modules:
|
| 35 |
+
m = types.ModuleType(n)
|
| 36 |
+
m.__path__ = [] # mark as package
|
| 37 |
+
sys.modules[n] = m
|
| 38 |
+
st = types.ModuleType("agenttune.decide.state")
|
| 39 |
+
st.PASS_VERDICTS = frozenset({"PASS", "APPROVE", "COMPLETE"}) # verbatim
|
| 40 |
+
sys.modules["agenttune.decide.state"] = st
|
| 41 |
+
path = os.path.join(ROOT, "third_party", "agenttune_gate", "deployment_gate.py")
|
| 42 |
+
spec = importlib.util.spec_from_file_location("deployment_gate", path)
|
| 43 |
+
mod = importlib.util.module_from_spec(spec)
|
| 44 |
+
sys.modules["deployment_gate"] = mod # dataclasses resolves cls.__module__ here
|
| 45 |
+
spec.loader.exec_module(mod)
|
| 46 |
+
return mod
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def results_lookup(run_name: str) -> dict[str, str]:
|
| 50 |
+
from huggingface_hub import hf_hub_download
|
| 51 |
+
|
| 52 |
+
p = hf_hub_download(RUNS_REPO, f"runs/{run_name}/results.jsonl", repo_type="dataset")
|
| 53 |
+
out = {}
|
| 54 |
+
for line in open(p, encoding="utf-8"):
|
| 55 |
+
r = json.loads(line)
|
| 56 |
+
out[r["qid"]] = "CORRECT" if r["status"] == "correct" else r["status"].upper()
|
| 57 |
+
return out
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def main() -> None:
|
| 61 |
+
ap = argparse.ArgumentParser()
|
| 62 |
+
ap.add_argument("--old", default="baseline-gate")
|
| 63 |
+
ap.add_argument("--new", default="healed-gate")
|
| 64 |
+
ap.add_argument("--out", default=os.path.join(ROOT, "data", "gate"))
|
| 65 |
+
ap.add_argument("--task-regression-tol", type=float, default=0.0)
|
| 66 |
+
ap.add_argument("--no-push", action="store_true")
|
| 67 |
+
args = ap.parse_args()
|
| 68 |
+
|
| 69 |
+
gate_mod = load_gate_module()
|
| 70 |
+
old_v, new_v = results_lookup(args.old), results_lookup(args.new)
|
| 71 |
+
qids = sorted(old_v)
|
| 72 |
+
assert set(qids) == set(new_v), "old/new runs cover different questions"
|
| 73 |
+
|
| 74 |
+
test_set = [{"pipeline_id": q, "input_text": q, "expected_verdict": "CORRECT"}
|
| 75 |
+
for q in qids]
|
| 76 |
+
|
| 77 |
+
gate = gate_mod.DeploymentGate()
|
| 78 |
+
decision = gate.evaluate_decision(
|
| 79 |
+
test_set=test_set,
|
| 80 |
+
old_model_fn=lambda qid: old_v[qid],
|
| 81 |
+
new_model_fn=lambda qid: new_v[qid],
|
| 82 |
+
task_regression_tol=args.task_regression_tol,
|
| 83 |
+
)
|
| 84 |
+
comparison = gate.compare_models(
|
| 85 |
+
test_set=test_set,
|
| 86 |
+
old_model_fn=lambda qid: old_v[qid],
|
| 87 |
+
new_model_fn=lambda qid: new_v[qid],
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
out = {
|
| 91 |
+
"gate_decision": asdict(decision),
|
| 92 |
+
"ab_comparison": asdict(comparison),
|
| 93 |
+
"old_run": args.old, "new_run": args.new,
|
| 94 |
+
"per_question_flips": {
|
| 95 |
+
"fixed": [q for q in qids if old_v[q] != "CORRECT" and new_v[q] == "CORRECT"],
|
| 96 |
+
"broken": [q for q in qids if old_v[q] == "CORRECT" and new_v[q] != "CORRECT"],
|
| 97 |
+
},
|
| 98 |
+
"gate_source": "AgentTune deployment_gate.py @ 1dfd054 (vendored verbatim)",
|
| 99 |
+
}
|
| 100 |
+
os.makedirs(args.out, exist_ok=True)
|
| 101 |
+
with open(os.path.join(args.out, "gate_decision.json"), "w", encoding="utf-8") as f:
|
| 102 |
+
json.dump(out, f, indent=2)
|
| 103 |
+
print(json.dumps({k: out[k] for k in ("gate_decision", "ab_comparison")}, indent=2))
|
| 104 |
+
print("fixed:", len(out["per_question_flips"]["fixed"]),
|
| 105 |
+
"broken:", len(out["per_question_flips"]["broken"]))
|
| 106 |
+
|
| 107 |
+
if not args.no_push:
|
| 108 |
+
from huggingface_hub import HfApi
|
| 109 |
+
|
| 110 |
+
HfApi().upload_folder(repo_id=RUNS_REPO, repo_type="dataset", folder_path=args.out,
|
| 111 |
+
path_in_repo="gate", commit_message="gate decision")
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
if __name__ == "__main__":
|
| 115 |
+
main()
|
src/selfheal_sql/heal.py
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The heal step: classify each failure, generate a corrected SQL query, keep it only
|
| 2 |
+
if it ACTUALLY produces the gold result. Output: DPO preference pairs.
|
| 3 |
+
|
| 4 |
+
Replaces AgentTune's TrainingExampleGenerator, which is tool-call-specific (it drops
|
| 5 |
+
every failure whose root cause isn't wrong_tool/loop_collapse, asks the LLM for "the
|
| 6 |
+
EXACT JSON tool call", and trains on a placeholder prompt). The classification stage
|
| 7 |
+
keeps AgentTune's FailureClassifier contract ({root_cause, confidence, analysis}) but
|
| 8 |
+
with a SQL-specific taxonomy. The output rows carry chosen/rejected exactly as
|
| 9 |
+
agenttune.decide.closed_loop.contracts.TrainingExample expects, so the retrain step
|
| 10 |
+
consumes them through Lexsi's own examples_to_dpo_dataset / run_retrain.
|
| 11 |
+
|
| 12 |
+
The healer NEVER sees the gold SQL. Gold is used only to verify a proposed fix by
|
| 13 |
+
executing both and comparing results (execcheck). Failed fixes are retried with the
|
| 14 |
+
new error message, up to --max-attempts.
|
| 15 |
+
|
| 16 |
+
Runs locally (needs the Spider DBs on disk + the HF router for LLM calls).
|
| 17 |
+
|
| 18 |
+
Usage:
|
| 19 |
+
python heal.py --run baseline-pool600 --split-file ../../data/splits/pool.jsonl \
|
| 20 |
+
--out ../../data/heal [--limit 6] [--workers 6]
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
|
| 25 |
+
import argparse
|
| 26 |
+
import json
|
| 27 |
+
import os
|
| 28 |
+
import re
|
| 29 |
+
import sys
|
| 30 |
+
import threading
|
| 31 |
+
import time
|
| 32 |
+
import urllib.error
|
| 33 |
+
import urllib.request
|
| 34 |
+
from collections import Counter
|
| 35 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 36 |
+
|
| 37 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 38 |
+
sys.path.insert(0, HERE)
|
| 39 |
+
from execcheck import check, extract_sql # noqa: E402
|
| 40 |
+
from student import PROMPT # noqa: E402 (the student's prompt — reused verbatim)
|
| 41 |
+
|
| 42 |
+
ROUTER = "https://router.huggingface.co/v1/chat/completions"
|
| 43 |
+
HEALER_MODEL = "CohereLabs/aya-expanse-32b" # on the hackathon's allowed model list
|
| 44 |
+
RUNS_REPO = "HideIron/selfheal-sql-runs"
|
| 45 |
+
SCHEMA_CAP = 7000 # chars of schema shown to the healer
|
| 46 |
+
|
| 47 |
+
# SQL-specific root causes (replaces the tool-agent taxonomy in
|
| 48 |
+
# agenttune FailureClassifier.ROOT_CAUSES; same output contract).
|
| 49 |
+
ROOT_CAUSES = [
|
| 50 |
+
"schema_misread", # non-existent column/table, column on wrong table
|
| 51 |
+
"wrong_join", # wrong join path, missing join, wrong join keys
|
| 52 |
+
"wrong_aggregation", # count/sum/avg/group-by/having misuse
|
| 53 |
+
"missing_or_wrong_filter", # WHERE/HAVING condition missing or wrong
|
| 54 |
+
"value_mismatch", # wrong literal (e.g. 'US' vs 'United States')
|
| 55 |
+
"wrong_output_columns", # right rows, wrong selected columns/order/distinct
|
| 56 |
+
"overcomplicated_query", # needless subquery/join that changes the result
|
| 57 |
+
"other",
|
| 58 |
+
]
|
| 59 |
+
|
| 60 |
+
_counts_lock = threading.Lock()
|
| 61 |
+
TOKENS = [0, 0] # prompt, completion
|
| 62 |
+
CALLS = [0]
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def llm(messages: list[dict], temperature: float, max_tokens: int = 500) -> str:
|
| 66 |
+
"""One chat call to the HF router, with backoff on 429/5xx. Returns content."""
|
| 67 |
+
body = json.dumps(
|
| 68 |
+
{"model": HEALER_MODEL, "messages": messages, "temperature": temperature,
|
| 69 |
+
"max_tokens": max_tokens}
|
| 70 |
+
).encode()
|
| 71 |
+
last = None
|
| 72 |
+
for i in range(5):
|
| 73 |
+
req = urllib.request.Request(
|
| 74 |
+
ROUTER, data=body,
|
| 75 |
+
headers={"Authorization": "Bearer " + os.environ["HF_TOKEN"],
|
| 76 |
+
"Content-Type": "application/json"},
|
| 77 |
+
)
|
| 78 |
+
try:
|
| 79 |
+
with urllib.request.urlopen(req, timeout=180) as r:
|
| 80 |
+
d = json.load(r)
|
| 81 |
+
u = d.get("usage", {})
|
| 82 |
+
with _counts_lock:
|
| 83 |
+
TOKENS[0] += u.get("prompt_tokens", 0)
|
| 84 |
+
TOKENS[1] += u.get("completion_tokens", 0)
|
| 85 |
+
CALLS[0] += 1
|
| 86 |
+
return d["choices"][0]["message"]["content"]
|
| 87 |
+
except urllib.error.HTTPError as e:
|
| 88 |
+
last = f"HTTP {e.code}: {e.read()[:200]!r}"
|
| 89 |
+
if e.code in (429, 500, 502, 503, 504):
|
| 90 |
+
time.sleep(2 ** i)
|
| 91 |
+
continue
|
| 92 |
+
raise RuntimeError(last) from e
|
| 93 |
+
except Exception as e: # noqa: BLE001 (timeouts, resets)
|
| 94 |
+
last = repr(e)
|
| 95 |
+
time.sleep(2 ** i)
|
| 96 |
+
raise RuntimeError(f"router call failed after retries: {last}")
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def _json_block(text: str) -> dict:
|
| 100 |
+
m = re.search(r"\{.*\}", text, re.DOTALL)
|
| 101 |
+
return json.loads(m.group(0)) if m else {}
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def classify(fail: dict, srow: dict) -> dict:
|
| 105 |
+
"""SQL-specific root-cause classification (AgentTune ClassifiedFailure contract)."""
|
| 106 |
+
sys_p = (
|
| 107 |
+
"You are an expert SQL failure analyst. Classify the failure into EXACTLY ONE "
|
| 108 |
+
"root cause from this list:\n" + ", ".join(ROOT_CAUSES) + "\n\n"
|
| 109 |
+
'Return ONLY a JSON object: {"root_cause": "<one of the list>", '
|
| 110 |
+
'"confidence": 0.0-1.0, "analysis": "one short sentence"}'
|
| 111 |
+
)
|
| 112 |
+
user = (
|
| 113 |
+
f"Schema:\n{srow['schema'][:SCHEMA_CAP]}\n\nQuestion: {srow['question']}\n\n"
|
| 114 |
+
f"Model's SQL:\n{fail['raw_output'][:1200]}\n\n"
|
| 115 |
+
f"Outcome: {fail['status']}"
|
| 116 |
+
+ (f" — {fail['error_message']}" if fail.get("error_message") else "")
|
| 117 |
+
+ (
|
| 118 |
+
f" (query returned {fail['pred_rows']} rows, expected result has {fail['gold_rows']} rows)"
|
| 119 |
+
if fail.get("pred_rows") is not None and fail.get("gold_rows") is not None
|
| 120 |
+
else ""
|
| 121 |
+
)
|
| 122 |
+
)
|
| 123 |
+
try:
|
| 124 |
+
p = _json_block(llm([{"role": "system", "content": sys_p},
|
| 125 |
+
{"role": "user", "content": user}], 0.0, 250))
|
| 126 |
+
rc = p.get("root_cause", "other")
|
| 127 |
+
return {
|
| 128 |
+
"root_cause": rc if rc in ROOT_CAUSES else "other",
|
| 129 |
+
"confidence": float(p.get("confidence", 0.5)),
|
| 130 |
+
"analysis": str(p.get("analysis", ""))[:400],
|
| 131 |
+
"classify_error": None,
|
| 132 |
+
}
|
| 133 |
+
except Exception as e: # noqa: BLE001
|
| 134 |
+
return {"root_cause": "other", "confidence": 0.0, "analysis": "",
|
| 135 |
+
"classify_error": str(e)[:300]}
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def generate_fix(fail: dict, srow: dict, cls: dict, max_attempts: int) -> tuple[str | None, list]:
|
| 139 |
+
"""Ask the healer for corrected SQL; keep only a fix that executes to the gold
|
| 140 |
+
result. Retries feed the candidate's own error back. Returns (sql|None, attempts)."""
|
| 141 |
+
base = (
|
| 142 |
+
f"A SQLite query is wrong and you must fix it.\n\n"
|
| 143 |
+
f"Schema:\n{srow['schema'][:SCHEMA_CAP]}\n\n"
|
| 144 |
+
f"Question: {srow['question']}\n\n"
|
| 145 |
+
f"Wrong SQL:\n{fail['raw_output'][:1200]}\n\n"
|
| 146 |
+
f"Observed problem: {fail['status']}"
|
| 147 |
+
+ (f" — {fail['error_message']}" if fail.get("error_message") else "")
|
| 148 |
+
+ (f"\nLikely root cause: {cls['root_cause']} ({cls['analysis']})" if cls.get("analysis") else "")
|
| 149 |
+
+ "\n\nRules: make the MINIMAL edit that fixes the query. Keep the original's "
|
| 150 |
+
"formatting, aliasing style, capitalisation and column order; do not add joins, "
|
| 151 |
+
"aliases, extra columns or reformat. Change only what is wrong.\n"
|
| 152 |
+
"Return ONLY the corrected SQLite query, nothing else."
|
| 153 |
+
)
|
| 154 |
+
msgs = [{"role": "user", "content": base}]
|
| 155 |
+
attempts = []
|
| 156 |
+
for k in range(max_attempts):
|
| 157 |
+
try:
|
| 158 |
+
out = llm(msgs, 0.0 if k == 0 else 0.4, 400)
|
| 159 |
+
except Exception as e: # noqa: BLE001
|
| 160 |
+
attempts.append({"attempt": k + 1, "error": f"llm: {str(e)[:200]}"})
|
| 161 |
+
break
|
| 162 |
+
cand = extract_sql(out)
|
| 163 |
+
if not cand:
|
| 164 |
+
attempts.append({"attempt": k + 1, "error": "no SQL in healer output"})
|
| 165 |
+
feedback = "Your reply contained no SQL. Return ONLY the corrected SQLite query."
|
| 166 |
+
else:
|
| 167 |
+
res = check(srow["db_path"], cand, srow["gold_sql"])
|
| 168 |
+
attempts.append({"attempt": k + 1, "sql": cand, "status": res.status,
|
| 169 |
+
"error": res.error_message[:200]})
|
| 170 |
+
if res.correct:
|
| 171 |
+
return cand, attempts
|
| 172 |
+
feedback = (
|
| 173 |
+
f"Your query:\n{cand}\n\nIt is still wrong: {res.status}"
|
| 174 |
+
+ (f" — {res.error_message[:300]}" if res.error_message else "")
|
| 175 |
+
+ (
|
| 176 |
+
f" (it returned {res.pred_rows} rows; the correct result has {res.gold_rows} rows)"
|
| 177 |
+
if res.pred_rows is not None and res.gold_rows is not None
|
| 178 |
+
else ""
|
| 179 |
+
)
|
| 180 |
+
+ "\nTry again. Return ONLY the corrected SQLite query."
|
| 181 |
+
)
|
| 182 |
+
msgs = msgs + [{"role": "assistant", "content": out},
|
| 183 |
+
{"role": "user", "content": feedback}]
|
| 184 |
+
return None, attempts
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def heal_one(fail: dict, srow: dict, max_attempts: int) -> dict:
|
| 188 |
+
cls = classify(fail, srow)
|
| 189 |
+
fixed_sql, attempts = generate_fix(fail, srow, cls, max_attempts)
|
| 190 |
+
rejected = extract_sql(fail["raw_output"]) or fail["raw_output"].strip()
|
| 191 |
+
return {
|
| 192 |
+
"qid": fail["qid"], "db_id": fail["db_id"], "question": srow["question"],
|
| 193 |
+
"status_before": fail["status"], "error_before": fail.get("error_message", ""),
|
| 194 |
+
**{k: cls[k] for k in ("root_cause", "confidence", "analysis", "classify_error")},
|
| 195 |
+
"healed": fixed_sql is not None,
|
| 196 |
+
"attempts_used": len(attempts), "attempts": attempts,
|
| 197 |
+
# --- TrainingExample-ready fields (consumed by train_dpo.py) ---
|
| 198 |
+
"prompt_user_content": PROMPT.format(schema=srow["schema"], question=srow["question"]),
|
| 199 |
+
"chosen_sql": fixed_sql,
|
| 200 |
+
"rejected_sql": rejected,
|
| 201 |
+
"healer_model": HEALER_MODEL,
|
| 202 |
+
}
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def main() -> None:
|
| 206 |
+
ap = argparse.ArgumentParser()
|
| 207 |
+
ap.add_argument("--run", default="baseline-pool600", help="runs-repo folder with results.jsonl")
|
| 208 |
+
ap.add_argument("--split-file", required=True)
|
| 209 |
+
ap.add_argument("--out", required=True)
|
| 210 |
+
ap.add_argument("--limit", type=int, default=None)
|
| 211 |
+
ap.add_argument("--workers", type=int, default=6)
|
| 212 |
+
ap.add_argument("--max-attempts", type=int, default=3)
|
| 213 |
+
ap.add_argument("--no-push", action="store_true")
|
| 214 |
+
ap.add_argument("--tag", default="", help="suffix for the runs-repo push path")
|
| 215 |
+
args = ap.parse_args()
|
| 216 |
+
|
| 217 |
+
from huggingface_hub import HfApi, hf_hub_download
|
| 218 |
+
|
| 219 |
+
rp = hf_hub_download(RUNS_REPO, f"runs/{args.run}/results.jsonl", repo_type="dataset")
|
| 220 |
+
results = [json.loads(l) for l in open(rp, encoding="utf-8")]
|
| 221 |
+
splits = {r["qid"]: r for r in map(json.loads, open(args.split_file, encoding="utf-8"))}
|
| 222 |
+
fails = [r for r in results if r["status"] != "correct"]
|
| 223 |
+
if args.limit:
|
| 224 |
+
fails = fails[: args.limit]
|
| 225 |
+
print(f"[heal] {len(fails)} failures to heal (of {len(results)} runs)", flush=True)
|
| 226 |
+
|
| 227 |
+
t0 = time.time()
|
| 228 |
+
done = [0]
|
| 229 |
+
|
| 230 |
+
def work(f):
|
| 231 |
+
row = heal_one(f, splits[f["qid"]], args.max_attempts)
|
| 232 |
+
with _counts_lock:
|
| 233 |
+
done[0] += 1
|
| 234 |
+
if done[0] % 20 == 0:
|
| 235 |
+
print(f"[heal] {done[0]}/{len(fails)} ({time.time()-t0:.0f}s, "
|
| 236 |
+
f"{CALLS[0]} calls)", flush=True)
|
| 237 |
+
return row
|
| 238 |
+
|
| 239 |
+
with ThreadPoolExecutor(max_workers=args.workers) as ex:
|
| 240 |
+
rows = list(ex.map(work, fails))
|
| 241 |
+
|
| 242 |
+
os.makedirs(args.out, exist_ok=True)
|
| 243 |
+
with open(os.path.join(args.out, "pairs.jsonl"), "w", encoding="utf-8") as f:
|
| 244 |
+
for r in rows:
|
| 245 |
+
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
| 246 |
+
|
| 247 |
+
healed = [r for r in rows if r["healed"]]
|
| 248 |
+
summary = {
|
| 249 |
+
"run": args.run, "healer_model": HEALER_MODEL, "n_failures": len(fails),
|
| 250 |
+
"n_healed": len(healed), "heal_yield": round(len(healed) / max(len(fails), 1), 4),
|
| 251 |
+
"by_root_cause": dict(Counter(r["root_cause"] for r in rows)),
|
| 252 |
+
"healed_by_root_cause": dict(Counter(r["root_cause"] for r in healed)),
|
| 253 |
+
"attempts_histogram": dict(Counter(r["attempts_used"] for r in healed)),
|
| 254 |
+
"classify_errors": sum(1 for r in rows if r["classify_error"]),
|
| 255 |
+
"llm_calls": CALLS[0], "prompt_tokens": TOKENS[0], "completion_tokens": TOKENS[1],
|
| 256 |
+
"elapsed_s": round(time.time() - t0, 1), "max_attempts": args.max_attempts,
|
| 257 |
+
}
|
| 258 |
+
with open(os.path.join(args.out, "heal_summary.json"), "w", encoding="utf-8") as f:
|
| 259 |
+
json.dump(summary, f, indent=2)
|
| 260 |
+
print("[heal] SUMMARY", json.dumps(summary), flush=True)
|
| 261 |
+
|
| 262 |
+
if not args.no_push:
|
| 263 |
+
dest = f"heal/{args.run}{args.tag}"
|
| 264 |
+
HfApi().upload_folder(repo_id=RUNS_REPO, repo_type="dataset", folder_path=args.out,
|
| 265 |
+
path_in_repo=dest, commit_message=f"heal {dest}")
|
| 266 |
+
print("[heal] pushed to", RUNS_REPO, dest, flush=True)
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
if __name__ == "__main__":
|
| 270 |
+
main()
|
src/selfheal_sql/launch_job.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Run one of our scripts on an HF Job GPU.
|
| 2 |
+
|
| 3 |
+
Ships the code folder and mounts the private dataset repo at /data. The HF token is read
|
| 4 |
+
from the HF_TOKEN env var and passed as an encrypted job secret (never written to disk).
|
| 5 |
+
|
| 6 |
+
Usage:
|
| 7 |
+
python launch_job.py --name smoke --flavor a10g-small --timeout 1h \
|
| 8 |
+
--pip "transformers accelerate peft" -- student.py --split pool --limit 20 ...
|
| 9 |
+
Everything after "--" is the script and its arguments; --data/--out are added for you.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import argparse
|
| 15 |
+
import os
|
| 16 |
+
import shlex
|
| 17 |
+
import sys
|
| 18 |
+
import time
|
| 19 |
+
|
| 20 |
+
from huggingface_hub import HfApi, Volume, fetch_job_logs, inspect_job, run_job, sync_job_volume
|
| 21 |
+
|
| 22 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 23 |
+
DATA_REPO = "HideIron/selfheal-sql-spider"
|
| 24 |
+
RUNS_REPO = "HideIron/selfheal-sql-runs"
|
| 25 |
+
IMAGE = "pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel"
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def main() -> None:
|
| 29 |
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
| 30 |
+
ap = argparse.ArgumentParser()
|
| 31 |
+
ap.add_argument("--name", required=True)
|
| 32 |
+
ap.add_argument("--flavor", default="a10g-small")
|
| 33 |
+
ap.add_argument("--timeout", default="1h")
|
| 34 |
+
ap.add_argument("--image", default=IMAGE)
|
| 35 |
+
ap.add_argument("--pip", default="transformers accelerate peft huggingface_hub jinja2>=3.1.6")
|
| 36 |
+
ap.add_argument("--detach", action="store_true")
|
| 37 |
+
ap.add_argument("rest", nargs=argparse.REMAINDER)
|
| 38 |
+
a = ap.parse_args()
|
| 39 |
+
rest = a.rest[1:] if a.rest and a.rest[0] == "--" else a.rest
|
| 40 |
+
script, script_args = rest[0], rest[1:]
|
| 41 |
+
|
| 42 |
+
token = os.environ["HF_TOKEN"]
|
| 43 |
+
api = HfApi(token=token)
|
| 44 |
+
api.create_repo(RUNS_REPO, repo_type="dataset", private=True, exist_ok=True)
|
| 45 |
+
|
| 46 |
+
code = sync_job_volume(HERE, "/code", token=token)
|
| 47 |
+
data = Volume(type="dataset", source=DATA_REPO, mount_path="/data")
|
| 48 |
+
|
| 49 |
+
inner = (
|
| 50 |
+
"export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True && "
|
| 51 |
+
"pip install -q -U " + " ".join(shlex.quote(x) for x in a.pip.split()) + " && "
|
| 52 |
+
f"python /code/{script} --data /data --out /tmp/out "
|
| 53 |
+
f"--push-repo {RUNS_REPO} --push-path runs/{a.name} " + " ".join(shlex.quote(x) for x in script_args)
|
| 54 |
+
)
|
| 55 |
+
job = run_job(
|
| 56 |
+
image=a.image, command=["bash", "-lc", inner], flavor=a.flavor, timeout=a.timeout,
|
| 57 |
+
name=a.name, volumes=[code, data], secrets={"HF_TOKEN": token},
|
| 58 |
+
)
|
| 59 |
+
print("job:", job.url, flush=True)
|
| 60 |
+
if a.detach:
|
| 61 |
+
print("job_id:", job.id)
|
| 62 |
+
return
|
| 63 |
+
while True:
|
| 64 |
+
st = inspect_job(job_id=job.id).status
|
| 65 |
+
if st.stage in ("COMPLETED", "ERROR", "CANCELED", "DELETED"):
|
| 66 |
+
break
|
| 67 |
+
time.sleep(10)
|
| 68 |
+
print("stage:", st.stage, "|", st.message)
|
| 69 |
+
lines = [ln for ln in fetch_job_logs(job_id=job.id) if "[A" not in ln and "it/s]" not in ln and "B/s]" not in ln]
|
| 70 |
+
for line in lines[-80:]:
|
| 71 |
+
print(line)
|
| 72 |
+
sys.exit(0 if st.stage == "COMPLETED" else 1)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
if __name__ == "__main__":
|
| 76 |
+
main()
|
src/selfheal_sql/make_gate_large.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Enlarge the held-out gate split (100 -> N) so small accuracy differences are measurable.
|
| 2 |
+
|
| 3 |
+
The original gate has 100 questions (std. error ~ +/-5pp), too noisy to tell a 2pp
|
| 4 |
+
regression from chance. This keeps the original 100 rows (same qids, so old results stay
|
| 5 |
+
comparable) and adds more questions drawn from the SAME databases the gate already uses
|
| 6 |
+
(Spider-train DB group B), so:
|
| 7 |
+
* no database appears in pool or test (asserted),
|
| 8 |
+
* every database is already in the HF dataset repo (no DB upload needed),
|
| 9 |
+
* gold queries run and return >=1 row (same rule as make_splits.py).
|
| 10 |
+
|
| 11 |
+
Usage (from repo root):
|
| 12 |
+
python src/selfheal_sql/make_gate_large.py [--total 400] [--seed 1]
|
| 13 |
+
Writes data/splits/gate_large.jsonl (absolute db_path, like the other local splits) and
|
| 14 |
+
data/hf_dataset/splits/gate_large.jsonl (relative db_path, ready to upload).
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import argparse
|
| 20 |
+
import json
|
| 21 |
+
import os
|
| 22 |
+
import random
|
| 23 |
+
import sys
|
| 24 |
+
|
| 25 |
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
| 26 |
+
sys.path.insert(0, HERE)
|
| 27 |
+
from execcheck import check, schema_text # noqa: E402
|
| 28 |
+
from make_splits import usable # noqa: E402
|
| 29 |
+
|
| 30 |
+
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
| 31 |
+
SPIDER = os.path.join(ROOT, "data", "spider", "spider_data")
|
| 32 |
+
SPLITS = os.path.join(ROOT, "data", "splits")
|
| 33 |
+
STAGE = os.path.join(ROOT, "data", "hf_dataset", "splits")
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def read(path):
|
| 37 |
+
with open(path, encoding="utf-8") as f:
|
| 38 |
+
return [json.loads(line) for line in f]
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def main() -> None:
|
| 42 |
+
ap = argparse.ArgumentParser()
|
| 43 |
+
ap.add_argument("--total", type=int, default=400)
|
| 44 |
+
ap.add_argument("--seed", type=int, default=1)
|
| 45 |
+
args = ap.parse_args()
|
| 46 |
+
|
| 47 |
+
pool, gate, test = (read(os.path.join(SPLITS, f"{n}.jsonl")) for n in ("pool", "gate", "test"))
|
| 48 |
+
gate_dbs = {r["db_id"] for r in gate}
|
| 49 |
+
seen = {(r["db_id"], r["question"]) for r in gate}
|
| 50 |
+
|
| 51 |
+
with open(os.path.join(SPIDER, "train_spider.json"), encoding="utf-8") as f:
|
| 52 |
+
train = json.load(f)
|
| 53 |
+
# Candidates: train questions on the gate's databases, not already in the gate.
|
| 54 |
+
cands = [x for x in train if x["db_id"] in gate_dbs and (x["db_id"], x["question"]) not in seen]
|
| 55 |
+
random.Random(args.seed).shuffle(cands)
|
| 56 |
+
|
| 57 |
+
rows = [dict(r, split="gate_large") for r in gate] # keep original 100, same qids
|
| 58 |
+
schemas = {r["db_id"]: r["schema"] for r in gate}
|
| 59 |
+
n_keep = len(rows)
|
| 60 |
+
for it in cands:
|
| 61 |
+
if len(rows) >= args.total:
|
| 62 |
+
break
|
| 63 |
+
db_id = it["db_id"]
|
| 64 |
+
db_path = os.path.join(SPIDER, "database", db_id, f"{db_id}.sqlite")
|
| 65 |
+
if not os.path.exists(db_path) or not usable(db_path, it["query"]):
|
| 66 |
+
continue
|
| 67 |
+
schemas.setdefault(db_id, schema_text(db_path))
|
| 68 |
+
rows.append({
|
| 69 |
+
"qid": f"gate_{len(rows):04d}", "split": "gate_large", "db_id": db_id,
|
| 70 |
+
"question": it["question"], "gold_sql": it["query"],
|
| 71 |
+
"db_path": db_path, "schema": schemas[db_id],
|
| 72 |
+
})
|
| 73 |
+
|
| 74 |
+
# Hard guarantees, asserted not assumed.
|
| 75 |
+
dbs = {r["db_id"] for r in rows}
|
| 76 |
+
assert dbs == gate_dbs, "gate_large introduced a new database"
|
| 77 |
+
assert not dbs & {r["db_id"] for r in pool}, "gate_large shares a DB with pool"
|
| 78 |
+
assert not dbs & {r["db_id"] for r in test}, "gate_large shares a DB with test"
|
| 79 |
+
assert len({r["qid"] for r in rows}) == len(rows), "duplicate qid"
|
| 80 |
+
assert len({(r["db_id"], r["question"]) for r in rows}) == len(rows), "duplicate question"
|
| 81 |
+
bad = [r["qid"] for r in rows if not check(r["db_path"], r["gold_sql"], r["gold_sql"]).correct]
|
| 82 |
+
print(f"gate_large: {len(rows)} questions ({n_keep} original + {len(rows) - n_keep} new) "
|
| 83 |
+
f"over {len(dbs)} databases; gold-vs-gold failures: {len(bad)}")
|
| 84 |
+
assert not bad, bad[:5]
|
| 85 |
+
|
| 86 |
+
os.makedirs(SPLITS, exist_ok=True)
|
| 87 |
+
with open(os.path.join(SPLITS, "gate_large.jsonl"), "w", encoding="utf-8") as f:
|
| 88 |
+
for r in rows:
|
| 89 |
+
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
| 90 |
+
|
| 91 |
+
os.makedirs(STAGE, exist_ok=True)
|
| 92 |
+
with open(os.path.join(STAGE, "gate_large.jsonl"), "w", encoding="utf-8") as f:
|
| 93 |
+
for r in rows:
|
| 94 |
+
r2 = dict(r, db_path=f"databases/{r['db_id']}/{r['db_id']}.sqlite")
|
| 95 |
+
f.write(json.dumps(r2, ensure_ascii=False) + "\n")
|
| 96 |
+
print("wrote data/splits/gate_large.jsonl and data/hf_dataset/splits/gate_large.jsonl")
|
| 97 |
+
print("db-disjoint from pool/test: OK")
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
if __name__ == "__main__":
|
| 101 |
+
main()
|
src/selfheal_sql/make_splits.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Build the three DATABASE-DISJOINT splits from Spider.
|
| 2 |
+
|
| 3 |
+
pool (failure pool) - questions the untrained student attempts; its failures
|
| 4 |
+
feed the heal loop. Source: Spider train, DB group A.
|
| 5 |
+
gate (held-out gate) - DeploymentGate A/B set; never used to make training data.
|
| 6 |
+
Source: Spider train, DB group B (different DBs from pool).
|
| 7 |
+
test (final test) - headline before/after numbers.
|
| 8 |
+
Source: Spider dev (Spider's own dev DBs never appear in train).
|
| 9 |
+
|
| 10 |
+
Disjoint DBs mean a "heal" cannot pass by memorising a schema it already saw.
|
| 11 |
+
|
| 12 |
+
Rows whose gold query errors, times out, or returns ZERO rows are dropped: an empty
|
| 13 |
+
gold result would let any empty-result query count as correct.
|
| 14 |
+
|
| 15 |
+
Usage: python -I make_splits.py <spider_data_dir> <out_dir> [pool gate test seed]
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import json
|
| 21 |
+
import os
|
| 22 |
+
import random
|
| 23 |
+
import sys
|
| 24 |
+
|
| 25 |
+
from execcheck import check, schema_text # noqa: E402 (run from this folder)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def load(path):
|
| 29 |
+
with open(path, encoding="utf-8") as f:
|
| 30 |
+
return json.load(f)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def usable(db_path: str, gold_sql: str) -> bool:
|
| 34 |
+
# gold vs gold: must run, and must return at least one row
|
| 35 |
+
import sqlite3
|
| 36 |
+
|
| 37 |
+
from execcheck import _connect, _run # noqa: PLC0415
|
| 38 |
+
|
| 39 |
+
try:
|
| 40 |
+
conn = _connect(db_path)
|
| 41 |
+
try:
|
| 42 |
+
rows = _run(conn, gold_sql, 10.0)
|
| 43 |
+
finally:
|
| 44 |
+
conn.close()
|
| 45 |
+
except (sqlite3.Error, Exception): # noqa: BLE001
|
| 46 |
+
return False
|
| 47 |
+
return len(rows) > 0
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def build_rows(items, db_dir, want, rng, tag):
|
| 51 |
+
rng.shuffle(items)
|
| 52 |
+
rows, schemas = [], {}
|
| 53 |
+
for it in items:
|
| 54 |
+
if len(rows) >= want:
|
| 55 |
+
break
|
| 56 |
+
db_id = it["db_id"]
|
| 57 |
+
db_path = os.path.join(db_dir, db_id, f"{db_id}.sqlite")
|
| 58 |
+
if not os.path.exists(db_path) or not usable(db_path, it["query"]):
|
| 59 |
+
continue
|
| 60 |
+
if db_id not in schemas:
|
| 61 |
+
schemas[db_id] = schema_text(db_path)
|
| 62 |
+
rows.append(
|
| 63 |
+
{
|
| 64 |
+
"qid": f"{tag}_{len(rows):04d}",
|
| 65 |
+
"split": tag,
|
| 66 |
+
"db_id": db_id,
|
| 67 |
+
"question": it["question"],
|
| 68 |
+
"gold_sql": it["query"],
|
| 69 |
+
"db_path": db_path,
|
| 70 |
+
"schema": schemas[db_id],
|
| 71 |
+
}
|
| 72 |
+
)
|
| 73 |
+
return rows
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def main():
|
| 77 |
+
spider = sys.argv[1]
|
| 78 |
+
out = sys.argv[2]
|
| 79 |
+
n_pool, n_gate, n_test, seed = (
|
| 80 |
+
int(sys.argv[3]) if len(sys.argv) > 3 else 200,
|
| 81 |
+
int(sys.argv[4]) if len(sys.argv) > 4 else 100,
|
| 82 |
+
int(sys.argv[5]) if len(sys.argv) > 5 else 100,
|
| 83 |
+
int(sys.argv[6]) if len(sys.argv) > 6 else 0,
|
| 84 |
+
)
|
| 85 |
+
rng = random.Random(seed)
|
| 86 |
+
db_dir = os.path.join(spider, "database")
|
| 87 |
+
train = load(os.path.join(spider, "train_spider.json"))
|
| 88 |
+
dev = load(os.path.join(spider, "dev.json"))
|
| 89 |
+
|
| 90 |
+
# Split TRAIN databases into two disjoint groups (pool vs gate).
|
| 91 |
+
train_dbs = sorted({x["db_id"] for x in train})
|
| 92 |
+
rng.shuffle(train_dbs)
|
| 93 |
+
cut = int(len(train_dbs) * 0.65)
|
| 94 |
+
group_a, group_b = set(train_dbs[:cut]), set(train_dbs[cut:])
|
| 95 |
+
pool_items = [x for x in train if x["db_id"] in group_a]
|
| 96 |
+
gate_items = [x for x in train if x["db_id"] in group_b]
|
| 97 |
+
|
| 98 |
+
pool = build_rows(pool_items, db_dir, n_pool, rng, "pool")
|
| 99 |
+
gate = build_rows(gate_items, db_dir, n_gate, rng, "gate")
|
| 100 |
+
test = build_rows(list(dev), db_dir, n_test, rng, "test")
|
| 101 |
+
|
| 102 |
+
# Hard guarantees, asserted not assumed.
|
| 103 |
+
dbs = {k: {r["db_id"] for r in v} for k, v in (("pool", pool), ("gate", gate), ("test", test))}
|
| 104 |
+
assert not (dbs["pool"] & dbs["gate"]), "pool/gate share a database"
|
| 105 |
+
assert not (dbs["pool"] & dbs["test"]), "pool/test share a database"
|
| 106 |
+
assert not (dbs["gate"] & dbs["test"]), "gate/test share a database"
|
| 107 |
+
|
| 108 |
+
os.makedirs(out, exist_ok=True)
|
| 109 |
+
for name, rows in (("pool", pool), ("gate", gate), ("test", test)):
|
| 110 |
+
with open(os.path.join(out, f"{name}.jsonl"), "w", encoding="utf-8") as f:
|
| 111 |
+
for r in rows:
|
| 112 |
+
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
| 113 |
+
print(f"{name}: {len(rows)} questions over {len(dbs[name])} databases")
|
| 114 |
+
|
| 115 |
+
# Sanity: gold-vs-gold must be 100% correct under the checker.
|
| 116 |
+
bad = [r["qid"] for rows in (pool, gate, test) for r in rows
|
| 117 |
+
if not check(r["db_path"], r["gold_sql"], r["gold_sql"]).correct]
|
| 118 |
+
print("gold-vs-gold failures:", len(bad), bad[:5])
|
| 119 |
+
print("db-disjoint: OK")
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
if __name__ == "__main__":
|
| 123 |
+
main()
|
src/selfheal_sql/metrics.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Final before/after metrics for the write-up. Reads student-run results from the
|
| 2 |
+
runs repo, computes per-split accuracy, status breakdowns, heal/regression rates and
|
| 3 |
+
a paired significance test (McNemar exact), and writes one metrics.json.
|
| 4 |
+
|
| 5 |
+
Usage: python metrics.py --pairs before:after [...] e.g.
|
| 6 |
+
python metrics.py --pair test:baseline-test:healed-test --pair gate:baseline-gate:healed-gate
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import argparse
|
| 12 |
+
import json
|
| 13 |
+
import math
|
| 14 |
+
import os
|
| 15 |
+
from collections import Counter
|
| 16 |
+
|
| 17 |
+
RUNS_REPO = "HideIron/selfheal-sql-runs"
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def load(run: str) -> dict[str, dict]:
|
| 21 |
+
from huggingface_hub import hf_hub_download
|
| 22 |
+
|
| 23 |
+
p = hf_hub_download(RUNS_REPO, f"runs/{run}/results.jsonl", repo_type="dataset")
|
| 24 |
+
return {r["qid"]: r for r in map(json.loads, open(p, encoding="utf-8"))}
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def mcnemar_exact(b: int, c: int) -> float:
|
| 28 |
+
"""Two-sided exact McNemar p-value. b = fixed (wrong->right), c = broken."""
|
| 29 |
+
n = b + c
|
| 30 |
+
if n == 0:
|
| 31 |
+
return 1.0
|
| 32 |
+
k = min(b, c)
|
| 33 |
+
tail = sum(math.comb(n, i) for i in range(0, k + 1)) / 2**n
|
| 34 |
+
return min(1.0, 2 * tail)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def split_metrics(before: dict, after: dict) -> dict:
|
| 38 |
+
qids = sorted(before)
|
| 39 |
+
assert set(qids) == set(after)
|
| 40 |
+
n = len(qids)
|
| 41 |
+
acc_b = sum(before[q]["status"] == "correct" for q in qids) / n
|
| 42 |
+
acc_a = sum(after[q]["status"] == "correct" for q in qids) / n
|
| 43 |
+
fixed = [q for q in qids if before[q]["status"] != "correct" and after[q]["status"] == "correct"]
|
| 44 |
+
broken = [q for q in qids if before[q]["status"] == "correct" and after[q]["status"] != "correct"]
|
| 45 |
+
n_fail_before = sum(before[q]["status"] != "correct" for q in qids)
|
| 46 |
+
n_corr_before = n - n_fail_before
|
| 47 |
+
|
| 48 |
+
def valid_rate(d):
|
| 49 |
+
return sum(d[q]["status"] in ("correct", "wrong_result") for q in qids) / n
|
| 50 |
+
|
| 51 |
+
return {
|
| 52 |
+
"n": n,
|
| 53 |
+
"execution_accuracy_before": round(acc_b, 4),
|
| 54 |
+
"execution_accuracy_after": round(acc_a, 4),
|
| 55 |
+
"delta_pp": round((acc_a - acc_b) * 100, 2),
|
| 56 |
+
"valid_sql_rate_before": round(valid_rate(before), 4),
|
| 57 |
+
"valid_sql_rate_after": round(valid_rate(after), 4),
|
| 58 |
+
"status_before": dict(Counter(before[q]["status"] for q in qids)),
|
| 59 |
+
"status_after": dict(Counter(after[q]["status"] for q in qids)),
|
| 60 |
+
"n_fixed": len(fixed),
|
| 61 |
+
"n_broken": len(broken),
|
| 62 |
+
"heal_rate": round(len(fixed) / n_fail_before, 4) if n_fail_before else None,
|
| 63 |
+
"regression_rate": round(len(broken) / n_corr_before, 4) if n_corr_before else None,
|
| 64 |
+
"mcnemar_p": round(mcnemar_exact(len(fixed), len(broken)), 6),
|
| 65 |
+
"fixed_qids": fixed,
|
| 66 |
+
"broken_qids": broken,
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def main() -> None:
|
| 71 |
+
ap = argparse.ArgumentParser()
|
| 72 |
+
ap.add_argument("--pair", action="append", required=True,
|
| 73 |
+
help="label:before_run:after_run (repeatable)")
|
| 74 |
+
ap.add_argument("--out", default=None)
|
| 75 |
+
ap.add_argument("--no-push", action="store_true")
|
| 76 |
+
args = ap.parse_args()
|
| 77 |
+
|
| 78 |
+
report = {}
|
| 79 |
+
for spec in args.pair:
|
| 80 |
+
label, b, a = spec.split(":")
|
| 81 |
+
report[label] = {"before_run": b, "after_run": a, **split_metrics(load(b), load(a))}
|
| 82 |
+
r = report[label]
|
| 83 |
+
print(f"[{label}] {r['execution_accuracy_before']:.1%} -> "
|
| 84 |
+
f"{r['execution_accuracy_after']:.1%} ({r['delta_pp']:+.1f}pp, "
|
| 85 |
+
f"fixed {r['n_fixed']}, broke {r['n_broken']}, p={r['mcnemar_p']})")
|
| 86 |
+
|
| 87 |
+
out_dir = args.out or os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
| 88 |
+
"..", "..", "data", "metrics")
|
| 89 |
+
os.makedirs(out_dir, exist_ok=True)
|
| 90 |
+
path = os.path.join(out_dir, "metrics.json")
|
| 91 |
+
with open(path, "w", encoding="utf-8") as f:
|
| 92 |
+
json.dump(report, f, indent=2)
|
| 93 |
+
print("wrote", path)
|
| 94 |
+
if not args.no_push:
|
| 95 |
+
from huggingface_hub import HfApi
|
| 96 |
+
|
| 97 |
+
HfApi().upload_folder(repo_id=RUNS_REPO, repo_type="dataset", folder_path=out_dir,
|
| 98 |
+
path_in_repo="metrics", commit_message="final metrics")
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
if __name__ == "__main__":
|
| 102 |
+
main()
|
src/selfheal_sql/student.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Student harness: make the model write SQL for a split, execute it, grade it, log it.
|
| 2 |
+
|
| 3 |
+
One script runs BOTH the baseline (no adapter) and the healed model (--adapter), with the
|
| 4 |
+
same prompt, the same greedy decoding and the same parsing, so a before/after difference
|
| 5 |
+
comes from training and nothing else.
|
| 6 |
+
|
| 7 |
+
Usage (inside an HF Job; see launch_job.py):
|
| 8 |
+
python student.py --data /data --split pool --model CohereLabs/tiny-aya-global \
|
| 9 |
+
--out /out/baseline_pool [--adapter path_or_repo] [--limit 20] \
|
| 10 |
+
[--push-repo user/runs-repo --push-path runs/baseline_pool]
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import argparse
|
| 16 |
+
import json
|
| 17 |
+
import os
|
| 18 |
+
import sys
|
| 19 |
+
import time
|
| 20 |
+
from collections import Counter
|
| 21 |
+
|
| 22 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 23 |
+
from execcheck import check # noqa: E402
|
| 24 |
+
|
| 25 |
+
PROMPT = (
|
| 26 |
+
"You are an expert in SQLite. Given the database schema and a question, write ONE SQLite "
|
| 27 |
+
"query that answers the question. Output only the SQL query, with no explanation.\n\n"
|
| 28 |
+
"Database schema:\n{schema}\n\nQuestion: {question}\n\nSQL:"
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def load_rows(data_root: str, split: str, limit: int | None):
|
| 33 |
+
path = os.path.join(data_root, "splits", f"{split}.jsonl")
|
| 34 |
+
if not os.path.exists(path): # local layout: <root>/<split>.jsonl
|
| 35 |
+
path = os.path.join(data_root, f"{split}.jsonl")
|
| 36 |
+
rows = [json.loads(line) for line in open(path, encoding="utf-8")]
|
| 37 |
+
return rows[:limit] if limit else rows
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def build_prompt(tok, row) -> str:
|
| 41 |
+
msgs = [{"role": "user", "content": PROMPT.format(schema=row["schema"], question=row["question"])}]
|
| 42 |
+
return tok.apply_chat_template(msgs, add_generation_prompt=True, tokenize=False)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def main() -> None:
|
| 46 |
+
ap = argparse.ArgumentParser()
|
| 47 |
+
ap.add_argument("--data", required=True, help="dataset root (contains splits/ and databases/)")
|
| 48 |
+
ap.add_argument("--split", required=True)
|
| 49 |
+
ap.add_argument("--model", default="CohereLabs/tiny-aya-global")
|
| 50 |
+
ap.add_argument("--adapter", default=None, help="LoRA adapter path or HF repo id")
|
| 51 |
+
ap.add_argument("--out", required=True)
|
| 52 |
+
ap.add_argument("--limit", type=int, default=None)
|
| 53 |
+
ap.add_argument("--batch-size", type=int, default=16)
|
| 54 |
+
ap.add_argument("--max-new-tokens", type=int, default=256)
|
| 55 |
+
ap.add_argument("--push-repo", default=None)
|
| 56 |
+
ap.add_argument("--push-path", default=None)
|
| 57 |
+
args = ap.parse_args()
|
| 58 |
+
|
| 59 |
+
import torch
|
| 60 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 61 |
+
|
| 62 |
+
rows = load_rows(args.data, args.split, args.limit)
|
| 63 |
+
print(f"[student] {len(rows)} rows from split={args.split}", flush=True)
|
| 64 |
+
|
| 65 |
+
tok = AutoTokenizer.from_pretrained(args.model)
|
| 66 |
+
tok.padding_side = "left"
|
| 67 |
+
if tok.pad_token is None:
|
| 68 |
+
tok.pad_token = tok.eos_token
|
| 69 |
+
model = AutoModelForCausalLM.from_pretrained(args.model, dtype=torch.bfloat16, device_map="cuda")
|
| 70 |
+
if args.adapter:
|
| 71 |
+
from peft import PeftModel
|
| 72 |
+
|
| 73 |
+
model = PeftModel.from_pretrained(model, args.adapter)
|
| 74 |
+
model.eval()
|
| 75 |
+
print("[student] model loaded:", args.model, "adapter:", args.adapter, flush=True)
|
| 76 |
+
|
| 77 |
+
prompts = [build_prompt(tok, r) for r in rows]
|
| 78 |
+
order = sorted(range(len(rows)), key=lambda i: len(prompts[i])) # batch similar lengths
|
| 79 |
+
raw: dict[int, tuple[str, int, int]] = {}
|
| 80 |
+
gen_seconds = 0.0
|
| 81 |
+
for b in range(0, len(order), args.batch_size):
|
| 82 |
+
idx = order[b : b + args.batch_size]
|
| 83 |
+
enc = tok([prompts[i] for i in idx], return_tensors="pt", padding=True, add_special_tokens=False).to("cuda")
|
| 84 |
+
t0 = time.time()
|
| 85 |
+
with torch.inference_mode():
|
| 86 |
+
out = model.generate(
|
| 87 |
+
**enc, max_new_tokens=args.max_new_tokens, do_sample=False, pad_token_id=tok.pad_token_id
|
| 88 |
+
)
|
| 89 |
+
torch.cuda.synchronize()
|
| 90 |
+
gen_seconds += time.time() - t0
|
| 91 |
+
new = out[:, enc["input_ids"].shape[1] :]
|
| 92 |
+
for j, i in enumerate(idx):
|
| 93 |
+
ids = new[j]
|
| 94 |
+
n_new = int((ids != tok.pad_token_id).sum())
|
| 95 |
+
raw[i] = (tok.decode(ids, skip_special_tokens=True), int(enc["attention_mask"][j].sum()), n_new)
|
| 96 |
+
print(f"[student] {min(b + args.batch_size, len(order))}/{len(order)}", flush=True)
|
| 97 |
+
|
| 98 |
+
os.makedirs(args.out, exist_ok=True)
|
| 99 |
+
statuses: Counter = Counter()
|
| 100 |
+
n_new_tokens = 0
|
| 101 |
+
with open(os.path.join(args.out, "results.jsonl"), "w", encoding="utf-8") as f:
|
| 102 |
+
for i, r in enumerate(rows):
|
| 103 |
+
text, n_prompt, n_new = raw[i]
|
| 104 |
+
n_new_tokens += n_new
|
| 105 |
+
res = check(os.path.join(args.data, r["db_path"]), text, r["gold_sql"])
|
| 106 |
+
statuses[res.status] += 1
|
| 107 |
+
f.write(
|
| 108 |
+
json.dumps(
|
| 109 |
+
{
|
| 110 |
+
"qid": r["qid"], "split": r["split"], "db_id": r["db_id"],
|
| 111 |
+
"question": r["question"], "gold_sql": r["gold_sql"],
|
| 112 |
+
"raw_output": text, "status": res.status,
|
| 113 |
+
"error_message": res.error_message,
|
| 114 |
+
"pred_rows": res.pred_rows, "gold_rows": res.gold_rows,
|
| 115 |
+
"prompt_tokens": n_prompt, "new_tokens": n_new,
|
| 116 |
+
},
|
| 117 |
+
ensure_ascii=False,
|
| 118 |
+
)
|
| 119 |
+
+ "\n"
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
n = len(rows)
|
| 123 |
+
summary = {
|
| 124 |
+
"split": args.split, "model": args.model, "adapter": args.adapter, "n": n,
|
| 125 |
+
"execution_accuracy": statuses["correct"] / n,
|
| 126 |
+
"valid_sql_rate": (n - statuses["sql_error"] - statuses["empty_sql"] - statuses["timeout"]) / n,
|
| 127 |
+
"status_counts": dict(statuses),
|
| 128 |
+
"generation_seconds": round(gen_seconds, 2),
|
| 129 |
+
"new_tokens": n_new_tokens,
|
| 130 |
+
"tokens_per_second": round(n_new_tokens / gen_seconds, 1) if gen_seconds else None,
|
| 131 |
+
"max_new_tokens": args.max_new_tokens, "decoding": "greedy",
|
| 132 |
+
}
|
| 133 |
+
with open(os.path.join(args.out, "summary.json"), "w", encoding="utf-8") as f:
|
| 134 |
+
json.dump(summary, f, indent=2)
|
| 135 |
+
print("[student] SUMMARY", json.dumps(summary), flush=True)
|
| 136 |
+
|
| 137 |
+
if args.push_repo:
|
| 138 |
+
from huggingface_hub import HfApi
|
| 139 |
+
|
| 140 |
+
HfApi().upload_folder(
|
| 141 |
+
repo_id=args.push_repo, repo_type="dataset", folder_path=args.out,
|
| 142 |
+
path_in_repo=args.push_path or os.path.basename(args.out.rstrip("/")),
|
| 143 |
+
commit_message=f"student run {args.split}",
|
| 144 |
+
)
|
| 145 |
+
print("[student] pushed to", args.push_repo, args.push_path, flush=True)
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
if __name__ == "__main__":
|
| 149 |
+
main()
|
src/selfheal_sql/train_dpo.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""DPO retrain via AgentTune's closed-loop retrain path (run inside an HF GPU Job).
|
| 2 |
+
|
| 3 |
+
Consumes the healer's verified pairs, builds agenttune TrainingExample objects, and
|
| 4 |
+
trains an adapter-only LoRA through agenttune's own
|
| 5 |
+
examples_to_dpo_dataset -> build_retrainer (create_agentic_trainer) -> train()
|
| 6 |
+
exactly as the closed loop's run_retrain does (we call build_retrainer so we can save
|
| 7 |
+
the adapter explicitly afterwards).
|
| 8 |
+
|
| 9 |
+
Train/eval consistency: the student evaluates with the Cohere chat template
|
| 10 |
+
(student.build_prompt). examples_to_dpo_dataset flattens prompt messages as
|
| 11 |
+
"role: content", BUT with role="" it emits content verbatim — so we pre-apply the
|
| 12 |
+
chat template here and hand it the final string. Training sees exactly the text the
|
| 13 |
+
model sees at inference.
|
| 14 |
+
|
| 15 |
+
Launched by launch_job.py, which injects --data/--out/--push-repo/--push-path.
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import argparse
|
| 21 |
+
import json
|
| 22 |
+
import os
|
| 23 |
+
import sys
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def main() -> None:
|
| 27 |
+
ap = argparse.ArgumentParser()
|
| 28 |
+
ap.add_argument("--data", default=None) # injected by launcher; unused
|
| 29 |
+
ap.add_argument("--out", required=True)
|
| 30 |
+
ap.add_argument("--push-repo", default=None)
|
| 31 |
+
ap.add_argument("--push-path", default=None)
|
| 32 |
+
ap.add_argument("--pairs-repo", default="HideIron/selfheal-sql-runs")
|
| 33 |
+
ap.add_argument("--pairs-path", default="heal/baseline-pool600/pairs.jsonl")
|
| 34 |
+
ap.add_argument("--model", default="CohereLabs/tiny-aya-global")
|
| 35 |
+
ap.add_argument("--adapter-repo", default="HideIron/tiny-aya-global-sql-selfheal")
|
| 36 |
+
ap.add_argument("--epochs", type=float, default=3.0)
|
| 37 |
+
ap.add_argument("--lr", type=float, default=1e-5)
|
| 38 |
+
ap.add_argument("--beta", type=float, default=0.1)
|
| 39 |
+
ap.add_argument("--lora-r", type=int, default=16)
|
| 40 |
+
ap.add_argument("--batch-size", type=int, default=2)
|
| 41 |
+
ap.add_argument("--grad-accum", type=int, default=8)
|
| 42 |
+
ap.add_argument("--max-steps", type=int, default=None, help="smoke-test cap")
|
| 43 |
+
ap.add_argument("--max-prompt-len", type=int, default=1408)
|
| 44 |
+
ap.add_argument("--rpo-alpha", type=float, default=0.0)
|
| 45 |
+
ap.add_argument("--algorithm", default="dpo", choices=["dpo", "bco"])
|
| 46 |
+
ap.add_argument("--replay-run", default=None,
|
| 47 |
+
help="BCO only: student run whose CORRECT answers are added as "
|
| 48 |
+
"positive (label=True) replay rows against forgetting")
|
| 49 |
+
ap.add_argument("--no-push-adapter", action="store_true")
|
| 50 |
+
args = ap.parse_args()
|
| 51 |
+
|
| 52 |
+
from huggingface_hub import HfApi, hf_hub_download
|
| 53 |
+
from transformers import AutoTokenizer
|
| 54 |
+
|
| 55 |
+
from agenttune.decide.closed_loop.contracts import TrainingExample
|
| 56 |
+
from agenttune.decide.closed_loop.retrain_config import RetrainConfig, build_retrainer
|
| 57 |
+
|
| 58 |
+
pp = hf_hub_download(args.pairs_repo, args.pairs_path, repo_type="dataset")
|
| 59 |
+
pairs = [json.loads(l) for l in open(pp, encoding="utf-8")]
|
| 60 |
+
healed = [p for p in pairs if p.get("healed") and p.get("chosen_sql")]
|
| 61 |
+
print(f"[dpo] {len(healed)} verified pairs of {len(pairs)} failures", flush=True)
|
| 62 |
+
|
| 63 |
+
tok = AutoTokenizer.from_pretrained(args.model)
|
| 64 |
+
|
| 65 |
+
def templ(user_content: str) -> str:
|
| 66 |
+
return tok.apply_chat_template(
|
| 67 |
+
[{"role": "user", "content": user_content}],
|
| 68 |
+
add_generation_prompt=True, tokenize=False,
|
| 69 |
+
)
|
| 70 |
+
|
| 71 |
+
examples = [
|
| 72 |
+
TrainingExample(
|
| 73 |
+
trajectory_id=p["qid"],
|
| 74 |
+
original_failure_type=p["status_before"],
|
| 75 |
+
root_cause=p["root_cause"],
|
| 76 |
+
# role "" -> _messages_to_text emits content verbatim (no "user: " prefix)
|
| 77 |
+
prompt=[{"role": "", "content": templ(p["prompt_user_content"])}],
|
| 78 |
+
chosen=[{"role": "", "content": p["chosen_sql"]}],
|
| 79 |
+
rejected=[{"role": "", "content": p["rejected_sql"]}],
|
| 80 |
+
)
|
| 81 |
+
for p in healed
|
| 82 |
+
]
|
| 83 |
+
|
| 84 |
+
if args.replay_run:
|
| 85 |
+
assert args.algorithm == "bco", "success replay needs BCO's independent rows"
|
| 86 |
+
import re as _re
|
| 87 |
+
|
| 88 |
+
def _extract(t): # minimal extract_sql (fences + first statement)
|
| 89 |
+
t = _re.sub(r"<think>.*?</think>", "", t, flags=_re.S)
|
| 90 |
+
m = _re.search(r"```(?:sql)?\s*(.*?)```", t, _re.S | _re.I)
|
| 91 |
+
if m:
|
| 92 |
+
t = m.group(1)
|
| 93 |
+
m = _re.search(r"\b(select|with)\b", t, _re.I)
|
| 94 |
+
return t[m.start():].split(";")[0].strip() if m else t.strip()
|
| 95 |
+
|
| 96 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 97 |
+
from student import PROMPT # the exact student prompt builder
|
| 98 |
+
|
| 99 |
+
split_rows = {}
|
| 100 |
+
sp = os.path.join(args.data or "/data", "splits", "pool.jsonl")
|
| 101 |
+
for s in map(json.loads, open(sp, encoding="utf-8")):
|
| 102 |
+
split_rows[s["qid"]] = s
|
| 103 |
+
rp2 = hf_hub_download(args.pairs_repo, f"runs/{args.replay_run}/results.jsonl",
|
| 104 |
+
repo_type="dataset")
|
| 105 |
+
n0 = len(examples)
|
| 106 |
+
for r in map(json.loads, open(rp2, encoding="utf-8")):
|
| 107 |
+
if r["status"] == "correct" and r["qid"] in split_rows:
|
| 108 |
+
s = split_rows[r["qid"]]
|
| 109 |
+
examples.append(TrainingExample(
|
| 110 |
+
trajectory_id=r["qid"] + "-replay",
|
| 111 |
+
original_failure_type="success_replay", root_cause="none",
|
| 112 |
+
prompt=[{"role": "", "content": templ(
|
| 113 |
+
PROMPT.format(schema=s["schema"], question=s["question"]))}],
|
| 114 |
+
chosen=[{"role": "", "content": _extract(r["raw_output"])}],
|
| 115 |
+
))
|
| 116 |
+
print(f"[dpo] +{len(examples)-n0} success-replay rows (label=True)", flush=True)
|
| 117 |
+
|
| 118 |
+
cfg = RetrainConfig(
|
| 119 |
+
model=args.model,
|
| 120 |
+
algorithm=args.algorithm,
|
| 121 |
+
output_dir=os.path.join(args.out, "adapter"),
|
| 122 |
+
lora_r=args.lora_r,
|
| 123 |
+
lora_alpha=args.lora_r * 2,
|
| 124 |
+
lora_dropout=0.05,
|
| 125 |
+
lora_target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
|
| 126 |
+
"gate_proj", "up_proj", "down_proj"],
|
| 127 |
+
num_train_epochs=args.epochs,
|
| 128 |
+
per_device_train_batch_size=args.batch_size,
|
| 129 |
+
gradient_accumulation_steps=args.grad_accum,
|
| 130 |
+
learning_rate=args.lr,
|
| 131 |
+
beta=args.beta,
|
| 132 |
+
max_steps=args.max_steps,
|
| 133 |
+
extra={
|
| 134 |
+
"bf16": True,
|
| 135 |
+
"logging_steps": 2,
|
| 136 |
+
"save_strategy": "no",
|
| 137 |
+
"report_to": "none",
|
| 138 |
+
"gradient_checkpointing": True,
|
| 139 |
+
},
|
| 140 |
+
)
|
| 141 |
+
if args.algorithm == "dpo":
|
| 142 |
+
# The BCO wrapper instantiates the model itself and rejects model_init_kwargs.
|
| 143 |
+
cfg.extra["model_init_kwargs"] = {"dtype": "bfloat16"}
|
| 144 |
+
|
| 145 |
+
# Cohere vocab is ~256k: full-sequence fp32 logits are ~4 GiB per copy and OOM a
|
| 146 |
+
# 24 GB card. use_logits_to_keep restricts logits to completion tokens (~60 here).
|
| 147 |
+
# If this TRL lacks it, shrink sequence caps hard instead.
|
| 148 |
+
import inspect
|
| 149 |
+
|
| 150 |
+
import trl
|
| 151 |
+
|
| 152 |
+
cfg_keys = set(inspect.signature(trl.DPOConfig.__init__).parameters)
|
| 153 |
+
if "use_logits_to_keep" in cfg_keys:
|
| 154 |
+
cfg.extra.update({"use_logits_to_keep": True,
|
| 155 |
+
"max_prompt_length": 2944, "max_length": 3072})
|
| 156 |
+
print("[dpo] use_logits_to_keep=True (trl", trl.__version__, ")", flush=True)
|
| 157 |
+
else:
|
| 158 |
+
cfg.extra.update({"max_prompt_length": args.max_prompt_len,
|
| 159 |
+
"max_length": args.max_prompt_len + 128})
|
| 160 |
+
print("[dpo] trl lacks use_logits_to_keep -> caps", args.max_prompt_len, flush=True)
|
| 161 |
+
# rpo_alpha adds an NLL term on the chosen completion (DPO+SFT hybrid): the model
|
| 162 |
+
# must LEARN to produce the verified fix, not merely prefer it. v1 (pure DPO,
|
| 163 |
+
# 30 steps) failed to move pool accuracy at all — this is the main v2 fix.
|
| 164 |
+
if args.rpo_alpha and "rpo_alpha" in cfg_keys:
|
| 165 |
+
cfg.extra["rpo_alpha"] = args.rpo_alpha
|
| 166 |
+
print("[dpo] rpo_alpha =", args.rpo_alpha, flush=True)
|
| 167 |
+
elif args.rpo_alpha:
|
| 168 |
+
print("[dpo] WARNING: trl lacks rpo_alpha; pure DPO", flush=True)
|
| 169 |
+
|
| 170 |
+
lens = sorted(len(tok(templ(p["prompt_user_content"]))["input_ids"]) for p in healed)
|
| 171 |
+
print(f"[dpo] prompt tokens: median={lens[len(lens)//2]} p90={lens[int(len(lens)*.9)]} "
|
| 172 |
+
f"max={lens[-1]} | cap={cfg.extra['max_prompt_length']}", flush=True)
|
| 173 |
+
|
| 174 |
+
trainer = build_retrainer(examples, cfg)
|
| 175 |
+
stats = trainer.train()
|
| 176 |
+
print("[dpo] train() returned:", str(stats)[:500], flush=True)
|
| 177 |
+
|
| 178 |
+
# Save the adapter explicitly (find the underlying HF trainer/model).
|
| 179 |
+
os.makedirs(cfg.output_dir, exist_ok=True)
|
| 180 |
+
inner = None
|
| 181 |
+
for attr in ("trainer", "_trainer", "dpo_trainer"):
|
| 182 |
+
inner = getattr(trainer, attr, None)
|
| 183 |
+
if inner is not None and hasattr(inner, "model"):
|
| 184 |
+
break
|
| 185 |
+
model_obj = getattr(inner, "model", None) or getattr(trainer, "model", None)
|
| 186 |
+
if model_obj is None:
|
| 187 |
+
raise RuntimeError(
|
| 188 |
+
f"cannot locate trained model on {type(trainer).__name__}; attrs: "
|
| 189 |
+
+ ", ".join(a for a in dir(trainer) if not a.startswith("__"))[:800]
|
| 190 |
+
)
|
| 191 |
+
model_obj.save_pretrained(cfg.output_dir) # PeftModel -> adapter only
|
| 192 |
+
print("[dpo] adapter saved:", os.listdir(cfg.output_dir), flush=True)
|
| 193 |
+
|
| 194 |
+
with open(os.path.join(args.out, "train_stats.json"), "w", encoding="utf-8") as f:
|
| 195 |
+
json.dump({"n_pairs": len(healed), "stats": str(stats)[:2000],
|
| 196 |
+
"config": {k: v for k, v in vars(args).items()}}, f, indent=2)
|
| 197 |
+
|
| 198 |
+
api = HfApi()
|
| 199 |
+
if not args.no_push_adapter:
|
| 200 |
+
api.create_repo(args.adapter_repo, private=True, exist_ok=True)
|
| 201 |
+
api.upload_folder(repo_id=args.adapter_repo, folder_path=cfg.output_dir,
|
| 202 |
+
commit_message=f"DPO LoRA ({len(healed)} self-heal pairs)")
|
| 203 |
+
print("[dpo] adapter pushed:", args.adapter_repo, flush=True)
|
| 204 |
+
if args.push_repo:
|
| 205 |
+
api.upload_folder(repo_id=args.push_repo, repo_type="dataset",
|
| 206 |
+
folder_path=args.out, path_in_repo=args.push_path or "dpo",
|
| 207 |
+
commit_message="dpo training stats")
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
if __name__ == "__main__":
|
| 211 |
+
main()
|
third_party/agenttune_gate/PROVENANCE.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Vendored verbatim from Lexsi-Labs/AgentTune @ 1dfd054 (origin/main, Nov 12 2026)
|
| 2 |
+
# src/agenttune/decide/closed_loop/deployment_gate.py — LSAL license applies.
|
| 3 |
+
|
third_party/agenttune_gate/deployment_gate.py
ADDED
|
@@ -0,0 +1,447 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Deployment Gate — Path B, Weeks 1–3
|
| 3 |
+
===================================
|
| 4 |
+
|
| 5 |
+
Gates the deployment of a retrained model.
|
| 6 |
+
|
| 7 |
+
- Week 1: ``build_test_set()`` — reconstruct ground-truth test cases from past
|
| 8 |
+
*successful* pipeline runs in the Decide audit log.
|
| 9 |
+
- Week 2: ``score_model()`` / ``compare_models()`` — task-accuracy A/B scoring.
|
| 10 |
+
- Week 3: ``evaluate_decision()`` — the full gate. Blocks a deploy if the new
|
| 11 |
+
model is worse on the task test set OR worse on Path A's trajectory eval
|
| 12 |
+
scores (conservative secondary signal). Wraps the existing deployment bridge
|
| 13 |
+
(``deploy`` / ``rollback``).
|
| 14 |
+
|
| 15 |
+
A "successful run" is a completion entry with ``is_complete == True``,
|
| 16 |
+
``error is None``, and a passing verdict (see ``PASS_VERDICTS``). For each
|
| 17 |
+
such run we recover its per-stage entries (matched by ``pipeline_id``) so the
|
| 18 |
+
test case carries the original input and the stage outputs the model is
|
| 19 |
+
expected to reproduce.
|
| 20 |
+
|
| 21 |
+
The audit log has two entry shapes, both written by ``AuditWriter``:
|
| 22 |
+
1. per-stage : {pipeline_id, stage_id, stage_type, input, output, reward, ...}
|
| 23 |
+
2. completion: {pipeline_id, template_id, verdict, is_complete, error,
|
| 24 |
+
step_count, episode_reward, ...}
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
from __future__ import annotations
|
| 28 |
+
|
| 29 |
+
import json
|
| 30 |
+
import logging
|
| 31 |
+
import os
|
| 32 |
+
import random
|
| 33 |
+
from collections.abc import Callable
|
| 34 |
+
from dataclasses import dataclass, field
|
| 35 |
+
from typing import Any
|
| 36 |
+
|
| 37 |
+
from agenttune.decide.state import PASS_VERDICTS
|
| 38 |
+
|
| 39 |
+
logger = logging.getLogger(__name__)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
@dataclass
|
| 43 |
+
class ABComparison:
|
| 44 |
+
"""Result of A/B scoring an old model vs a new (retrained) model."""
|
| 45 |
+
|
| 46 |
+
old_score: float
|
| 47 |
+
new_score: float
|
| 48 |
+
delta: float # new_score - old_score
|
| 49 |
+
num_cases: int
|
| 50 |
+
new_is_better: bool # new_score > old_score (by >= min_delta)
|
| 51 |
+
recommendation: str # "deploy" | "keep_old" (advisory in Week 2)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
@dataclass
|
| 55 |
+
class GateDecision:
|
| 56 |
+
"""Final gate decision for a retrained model (Week 3).
|
| 57 |
+
|
| 58 |
+
A deploy is allowed only if the new model is NOT worse on task accuracy AND
|
| 59 |
+
NOT worse on trajectory quality (the conservative secondary signal). Either
|
| 60 |
+
regression blocks the deploy.
|
| 61 |
+
"""
|
| 62 |
+
|
| 63 |
+
approved: bool # True → deploy, False → keep old (rollback/no-op)
|
| 64 |
+
reason: str # human-readable explanation
|
| 65 |
+
task_delta: float = 0.0 # new task pass-rate − old
|
| 66 |
+
trajectory_delta: float = 0.0 # new mean trajectory score − old
|
| 67 |
+
task_old: float = 0.0
|
| 68 |
+
task_new: float = 0.0
|
| 69 |
+
trajectory_old: float | None = None
|
| 70 |
+
trajectory_new: float | None = None
|
| 71 |
+
num_cases: int = 0
|
| 72 |
+
details: dict[str, Any] = field(default_factory=dict)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class DeploymentGate:
|
| 76 |
+
"""Builds a ground-truth test set and gates deploy of a retrained model.
|
| 77 |
+
|
| 78 |
+
Week 1: ``build_test_set``. Week 2: ``score_model`` / ``compare_models``.
|
| 79 |
+
Week 3: ``evaluate_decision`` (task + trajectory gate) and ``apply_decision``
|
| 80 |
+
(wrap the deployment bridge).
|
| 81 |
+
"""
|
| 82 |
+
|
| 83 |
+
def __init__(self, seed: int | None = None) -> None:
|
| 84 |
+
# Deterministic sampling when a seed is provided (reproducible test sets).
|
| 85 |
+
self._rng = random.Random(seed)
|
| 86 |
+
|
| 87 |
+
# ------------------------------------------------------------------
|
| 88 |
+
# Test-set builder
|
| 89 |
+
# ------------------------------------------------------------------
|
| 90 |
+
|
| 91 |
+
def build_test_set(
|
| 92 |
+
self,
|
| 93 |
+
audit_path: str,
|
| 94 |
+
min_samples: int = 20,
|
| 95 |
+
max_samples: int = 200,
|
| 96 |
+
) -> list[dict[str, Any]]:
|
| 97 |
+
"""Reconstruct test cases from successful pipeline runs.
|
| 98 |
+
|
| 99 |
+
Args:
|
| 100 |
+
audit_path: Path to the Decide ``audit.jsonl`` file.
|
| 101 |
+
min_samples: If fewer successful runs than this are found, return
|
| 102 |
+
an empty list (not enough signal for a meaningful gate).
|
| 103 |
+
max_samples: Cap on the returned set; randomly sampled if exceeded.
|
| 104 |
+
|
| 105 |
+
Returns:
|
| 106 |
+
A list of test cases, each::
|
| 107 |
+
|
| 108 |
+
{
|
| 109 |
+
"pipeline_id": str,
|
| 110 |
+
"template_id": Optional[str],
|
| 111 |
+
"input_text": Any, # input to the first stage
|
| 112 |
+
"expected_verdict": str,
|
| 113 |
+
"expected_stage_outputs": {stage_id: output, ...},
|
| 114 |
+
"episode_reward": Optional[float],
|
| 115 |
+
}
|
| 116 |
+
"""
|
| 117 |
+
if not os.path.exists(audit_path):
|
| 118 |
+
logger.warning("DeploymentGate.build_test_set: %s not found", audit_path)
|
| 119 |
+
return []
|
| 120 |
+
|
| 121 |
+
# Pass 1: collect per-stage entries grouped by pipeline, and the set of
|
| 122 |
+
# successful completion entries.
|
| 123 |
+
stages_by_pipeline: dict[str, list[dict[str, Any]]] = {}
|
| 124 |
+
completions: dict[str, dict[str, Any]] = {}
|
| 125 |
+
|
| 126 |
+
with open(audit_path, encoding="utf-8") as f:
|
| 127 |
+
for line in f:
|
| 128 |
+
if not line.strip():
|
| 129 |
+
continue
|
| 130 |
+
try:
|
| 131 |
+
record = json.loads(line)
|
| 132 |
+
except json.JSONDecodeError:
|
| 133 |
+
continue
|
| 134 |
+
|
| 135 |
+
pid = record.get("pipeline_id")
|
| 136 |
+
if not pid:
|
| 137 |
+
continue
|
| 138 |
+
|
| 139 |
+
if "stage_type" in record or "stage_id" in record:
|
| 140 |
+
# per-stage entry
|
| 141 |
+
stages_by_pipeline.setdefault(pid, []).append(record)
|
| 142 |
+
elif self._is_successful_completion(record):
|
| 143 |
+
# successful completion entry (last writer wins per pipeline)
|
| 144 |
+
completions[pid] = record
|
| 145 |
+
|
| 146 |
+
# Pass 2: build a test case per successful pipeline.
|
| 147 |
+
test_cases: list[dict[str, Any]] = []
|
| 148 |
+
for pid, completion in completions.items():
|
| 149 |
+
stages = stages_by_pipeline.get(pid, [])
|
| 150 |
+
if not stages:
|
| 151 |
+
# No stage detail to reconstruct an input — skip.
|
| 152 |
+
continue
|
| 153 |
+
|
| 154 |
+
input_text = self._first_stage_input(stages)
|
| 155 |
+
if input_text is None:
|
| 156 |
+
continue
|
| 157 |
+
|
| 158 |
+
expected_outputs = {
|
| 159 |
+
s.get("stage_id"): s.get("output") for s in stages if s.get("stage_id") is not None
|
| 160 |
+
}
|
| 161 |
+
|
| 162 |
+
test_cases.append(
|
| 163 |
+
{
|
| 164 |
+
"pipeline_id": pid,
|
| 165 |
+
"template_id": completion.get("template_id"),
|
| 166 |
+
"input_text": input_text,
|
| 167 |
+
"expected_verdict": completion.get("verdict"),
|
| 168 |
+
"expected_stage_outputs": expected_outputs,
|
| 169 |
+
"episode_reward": completion.get("episode_reward"),
|
| 170 |
+
}
|
| 171 |
+
)
|
| 172 |
+
|
| 173 |
+
if len(test_cases) < min_samples:
|
| 174 |
+
logger.warning(
|
| 175 |
+
"DeploymentGate.build_test_set: only %d successful runs found "
|
| 176 |
+
"(min_samples=%d) — returning empty test set.",
|
| 177 |
+
len(test_cases),
|
| 178 |
+
min_samples,
|
| 179 |
+
)
|
| 180 |
+
return []
|
| 181 |
+
|
| 182 |
+
if len(test_cases) > max_samples:
|
| 183 |
+
test_cases = self._rng.sample(test_cases, max_samples)
|
| 184 |
+
|
| 185 |
+
logger.info(
|
| 186 |
+
"DeploymentGate.build_test_set: built %d test cases from %s",
|
| 187 |
+
len(test_cases),
|
| 188 |
+
audit_path,
|
| 189 |
+
)
|
| 190 |
+
return test_cases
|
| 191 |
+
|
| 192 |
+
# ------------------------------------------------------------------
|
| 193 |
+
# Scoring & A/B comparison
|
| 194 |
+
# ------------------------------------------------------------------
|
| 195 |
+
|
| 196 |
+
def score_model(
|
| 197 |
+
self,
|
| 198 |
+
test_set: list[dict[str, Any]],
|
| 199 |
+
model_runner_fn: Callable[[Any], Any],
|
| 200 |
+
) -> float | None:
|
| 201 |
+
"""Score a model on the test set: fraction of verdicts that match.
|
| 202 |
+
|
| 203 |
+
``model_runner_fn(input_text) -> verdict`` is supplied by the caller
|
| 204 |
+
(it runs the model / pipeline and returns the produced verdict). This
|
| 205 |
+
keeps the gate model-agnostic and testable without a GPU.
|
| 206 |
+
|
| 207 |
+
The returned verdict is compared case-insensitively against each test
|
| 208 |
+
case's ``expected_verdict``. Returns the pass rate in ``[0, 1]``, or
|
| 209 |
+
``None`` if the test set is empty.
|
| 210 |
+
"""
|
| 211 |
+
if not test_set:
|
| 212 |
+
logger.warning("DeploymentGate.score_model: empty test set — nothing to score.")
|
| 213 |
+
return None
|
| 214 |
+
|
| 215 |
+
passed = 0
|
| 216 |
+
scored = 0
|
| 217 |
+
for case in test_set:
|
| 218 |
+
expected = str(case.get("expected_verdict") or "").upper()
|
| 219 |
+
try:
|
| 220 |
+
produced = model_runner_fn(case.get("input_text"))
|
| 221 |
+
except Exception as exc: # noqa: BLE001 — a crash is a failed case
|
| 222 |
+
logger.warning("score_model: runner raised on %s: %s", case.get("pipeline_id"), exc)
|
| 223 |
+
scored += 1
|
| 224 |
+
continue
|
| 225 |
+
scored += 1
|
| 226 |
+
if str(produced or "").upper() == expected:
|
| 227 |
+
passed += 1
|
| 228 |
+
|
| 229 |
+
pass_rate = passed / scored if scored else 0.0
|
| 230 |
+
logger.info("DeploymentGate.score_model: %d/%d passed (%.3f).", passed, scored, pass_rate)
|
| 231 |
+
return round(pass_rate, 4)
|
| 232 |
+
|
| 233 |
+
def compare_models(
|
| 234 |
+
self,
|
| 235 |
+
test_set: list[dict[str, Any]],
|
| 236 |
+
old_model_fn: Callable[[Any], Any],
|
| 237 |
+
new_model_fn: Callable[[Any], Any],
|
| 238 |
+
min_delta: float = 0.0,
|
| 239 |
+
) -> ABComparison:
|
| 240 |
+
"""A/B score old vs new model on the same test set.
|
| 241 |
+
|
| 242 |
+
``new_is_better`` requires the new model to beat the old by at least
|
| 243 |
+
``min_delta`` (default 0.0 → strictly better). The ``recommendation``
|
| 244 |
+
is advisory in Week 2 — actually gating deploy/rollback is Week 3.
|
| 245 |
+
"""
|
| 246 |
+
old_score = self.score_model(test_set, old_model_fn) or 0.0
|
| 247 |
+
new_score = self.score_model(test_set, new_model_fn) or 0.0
|
| 248 |
+
delta = round(new_score - old_score, 4)
|
| 249 |
+
new_is_better = delta >= min_delta and new_score > old_score
|
| 250 |
+
recommendation = "deploy" if new_is_better else "keep_old"
|
| 251 |
+
logger.info(
|
| 252 |
+
"DeploymentGate.compare_models: old=%.3f new=%.3f delta=%+.3f → %s",
|
| 253 |
+
old_score,
|
| 254 |
+
new_score,
|
| 255 |
+
delta,
|
| 256 |
+
recommendation,
|
| 257 |
+
)
|
| 258 |
+
return ABComparison(
|
| 259 |
+
old_score=old_score,
|
| 260 |
+
new_score=new_score,
|
| 261 |
+
delta=delta,
|
| 262 |
+
num_cases=len(test_set),
|
| 263 |
+
new_is_better=new_is_better,
|
| 264 |
+
recommendation=recommendation,
|
| 265 |
+
)
|
| 266 |
+
|
| 267 |
+
# ------------------------------------------------------------------
|
| 268 |
+
# Week 3 — full gate: task accuracy AND trajectory quality
|
| 269 |
+
# ------------------------------------------------------------------
|
| 270 |
+
|
| 271 |
+
@staticmethod
|
| 272 |
+
def _mean_trajectory_score(results: list[Any]) -> float | None:
|
| 273 |
+
"""Mean composite trajectory score over a list of AgenticEvalResult.
|
| 274 |
+
|
| 275 |
+
Incorporates base LLM judge score with the novel task-agnostic metrics
|
| 276 |
+
(TAC, SCSR, IASA, EGS) and applies penalties for redundancy (ARR) and
|
| 277 |
+
loop collapses (LCF).
|
| 278 |
+
"""
|
| 279 |
+
scores = []
|
| 280 |
+
for r in results or []:
|
| 281 |
+
if hasattr(r, "overall_judge_score"):
|
| 282 |
+
base = getattr(r, "overall_judge_score", 0.0)
|
| 283 |
+
|
| 284 |
+
# Positive deterministic & specific LLM signals
|
| 285 |
+
tac = getattr(r, "tac_score", 0.0)
|
| 286 |
+
scsr = getattr(r, "scsr_score", 0.0)
|
| 287 |
+
iasa = getattr(r, "iasa_score", 0.0)
|
| 288 |
+
egs = getattr(r, "egs_score", 0.0)
|
| 289 |
+
|
| 290 |
+
# Average the positive components
|
| 291 |
+
composite = (base + tac + scsr + iasa + egs) / 5.0
|
| 292 |
+
|
| 293 |
+
# Penalties for failure modes
|
| 294 |
+
arr = getattr(r, "arr_score", 0.0)
|
| 295 |
+
lcf = getattr(r, "lcf_score", 0)
|
| 296 |
+
|
| 297 |
+
composite -= arr * 0.2 # Penalty for API redundancy
|
| 298 |
+
if lcf > 0:
|
| 299 |
+
composite -= 0.1 * lcf # Penalty for loop collapses
|
| 300 |
+
|
| 301 |
+
scores.append(max(0.0, min(1.0, composite)))
|
| 302 |
+
|
| 303 |
+
if not scores:
|
| 304 |
+
return None
|
| 305 |
+
return round(sum(scores) / len(scores), 4)
|
| 306 |
+
|
| 307 |
+
def evaluate_decision(
|
| 308 |
+
self,
|
| 309 |
+
test_set: list[dict[str, Any]],
|
| 310 |
+
old_model_fn: Callable[[Any], Any],
|
| 311 |
+
new_model_fn: Callable[[Any], Any],
|
| 312 |
+
old_trajectory_results: list[Any] | None = None,
|
| 313 |
+
new_trajectory_results: list[Any] | None = None,
|
| 314 |
+
task_regression_tol: float = 0.0,
|
| 315 |
+
trajectory_regression_tol: float = 0.05,
|
| 316 |
+
) -> GateDecision:
|
| 317 |
+
"""Decide whether to deploy the new model.
|
| 318 |
+
|
| 319 |
+
Blocks the deploy if EITHER signal regresses:
|
| 320 |
+
- Task accuracy: ``new_task < old_task - task_regression_tol``.
|
| 321 |
+
- Trajectory quality: ``new_traj < old_traj - trajectory_regression_tol``
|
| 322 |
+
(conservative secondary signal; only applied when both trajectory
|
| 323 |
+
scores are supplied — see Path A's ``TrajectoryEvaluator``).
|
| 324 |
+
|
| 325 |
+
``*_trajectory_results`` are lists of ``AgenticEvalResult`` (from
|
| 326 |
+
``TrajectoryEvaluator.evaluate_batch``); pass ``None`` to skip the
|
| 327 |
+
trajectory check (e.g. when the eval suite hasn't run).
|
| 328 |
+
"""
|
| 329 |
+
task_old = self.score_model(test_set, old_model_fn) or 0.0
|
| 330 |
+
task_new = self.score_model(test_set, new_model_fn) or 0.0
|
| 331 |
+
task_delta = round(task_new - task_old, 4)
|
| 332 |
+
|
| 333 |
+
traj_old = self._mean_trajectory_score(old_trajectory_results)
|
| 334 |
+
traj_new = self._mean_trajectory_score(new_trajectory_results)
|
| 335 |
+
traj_delta = (
|
| 336 |
+
round((traj_new or 0.0) - (traj_old or 0.0), 4)
|
| 337 |
+
if traj_old is not None and traj_new is not None
|
| 338 |
+
else 0.0
|
| 339 |
+
)
|
| 340 |
+
|
| 341 |
+
# --- regression checks ---
|
| 342 |
+
task_regressed = task_new < (task_old - task_regression_tol)
|
| 343 |
+
trajectory_regressed = (
|
| 344 |
+
traj_old is not None
|
| 345 |
+
and traj_new is not None
|
| 346 |
+
and traj_new < (traj_old - trajectory_regression_tol)
|
| 347 |
+
)
|
| 348 |
+
|
| 349 |
+
if task_regressed and trajectory_regressed:
|
| 350 |
+
approved, reason = False, "blocked: task AND trajectory regressed"
|
| 351 |
+
elif task_regressed:
|
| 352 |
+
approved, reason = False, "blocked: task accuracy regressed"
|
| 353 |
+
elif trajectory_regressed:
|
| 354 |
+
approved, reason = False, "blocked: trajectory quality regressed"
|
| 355 |
+
else:
|
| 356 |
+
approved, reason = True, "approved: no regression on task or trajectory"
|
| 357 |
+
|
| 358 |
+
decision = GateDecision(
|
| 359 |
+
approved=approved,
|
| 360 |
+
reason=reason,
|
| 361 |
+
task_delta=task_delta,
|
| 362 |
+
trajectory_delta=traj_delta,
|
| 363 |
+
task_old=task_old,
|
| 364 |
+
task_new=task_new,
|
| 365 |
+
trajectory_old=traj_old,
|
| 366 |
+
trajectory_new=traj_new,
|
| 367 |
+
num_cases=len(test_set),
|
| 368 |
+
details={
|
| 369 |
+
"task_regressed": task_regressed,
|
| 370 |
+
"trajectory_regressed": trajectory_regressed,
|
| 371 |
+
"task_regression_tol": task_regression_tol,
|
| 372 |
+
"trajectory_regression_tol": trajectory_regression_tol,
|
| 373 |
+
"trajectory_checked": traj_old is not None and traj_new is not None,
|
| 374 |
+
},
|
| 375 |
+
)
|
| 376 |
+
logger.info(
|
| 377 |
+
"GateDecision: approved=%s (%s) task %.3f→%.3f traj %s→%s",
|
| 378 |
+
approved,
|
| 379 |
+
reason,
|
| 380 |
+
task_old,
|
| 381 |
+
task_new,
|
| 382 |
+
traj_old,
|
| 383 |
+
traj_new,
|
| 384 |
+
)
|
| 385 |
+
return decision
|
| 386 |
+
|
| 387 |
+
def apply_decision(
|
| 388 |
+
self,
|
| 389 |
+
decision: GateDecision,
|
| 390 |
+
config_path: str,
|
| 391 |
+
trained_path: str | None = None,
|
| 392 |
+
deploy_fn: Callable[..., Any] | None = None,
|
| 393 |
+
rollback_fn: Callable[..., Any] | None = None,
|
| 394 |
+
) -> str:
|
| 395 |
+
"""Act on a :class:`GateDecision` via the deployment bridge.
|
| 396 |
+
|
| 397 |
+
On ``approved`` → deploy the new model; otherwise → keep the old model
|
| 398 |
+
(rollback hook, a no-op by default). ``deploy_fn`` / ``rollback_fn`` are
|
| 399 |
+
injectable so this is testable without touching real deployments; when
|
| 400 |
+
omitted they default to the Decide ``model_deployment`` bridge.
|
| 401 |
+
|
| 402 |
+
Returns one of ``"deployed"`` / ``"kept_old"``.
|
| 403 |
+
"""
|
| 404 |
+
if decision.approved:
|
| 405 |
+
if deploy_fn is None:
|
| 406 |
+
from agenttune.decide.model_deployment import (
|
| 407 |
+
deploy_trained_model as deploy_fn, # type: ignore
|
| 408 |
+
)
|
| 409 |
+
if trained_path is None:
|
| 410 |
+
raise ValueError("apply_decision(approved): trained_path is required to deploy.")
|
| 411 |
+
deploy_fn(config_path=config_path, trained_path=trained_path)
|
| 412 |
+
logger.info("apply_decision: DEPLOYED new model (%s).", decision.reason)
|
| 413 |
+
return "deployed"
|
| 414 |
+
|
| 415 |
+
# Not approved → keep old model.
|
| 416 |
+
if rollback_fn is not None:
|
| 417 |
+
rollback_fn(config_path=config_path)
|
| 418 |
+
logger.warning("apply_decision: KEPT OLD model (%s).", decision.reason)
|
| 419 |
+
return "kept_old"
|
| 420 |
+
|
| 421 |
+
# ------------------------------------------------------------------
|
| 422 |
+
# Internal helpers
|
| 423 |
+
# ------------------------------------------------------------------
|
| 424 |
+
|
| 425 |
+
@staticmethod
|
| 426 |
+
def _is_successful_completion(record: dict[str, Any]) -> bool:
|
| 427 |
+
"""A completion entry that finished cleanly with a passing verdict."""
|
| 428 |
+
if "is_complete" not in record and "verdict" not in record:
|
| 429 |
+
return False
|
| 430 |
+
if not record.get("is_complete", False):
|
| 431 |
+
return False
|
| 432 |
+
if record.get("error") is not None:
|
| 433 |
+
return False
|
| 434 |
+
verdict = str(record.get("verdict") or "").upper()
|
| 435 |
+
return verdict in PASS_VERDICTS
|
| 436 |
+
|
| 437 |
+
@staticmethod
|
| 438 |
+
def _first_stage_input(stages: list[dict[str, Any]]) -> Any | None:
|
| 439 |
+
"""Return the input recorded for the earliest stage of a pipeline.
|
| 440 |
+
|
| 441 |
+
Falls back across stages because not every stage records an ``input``.
|
| 442 |
+
"""
|
| 443 |
+
for stage in stages:
|
| 444 |
+
value = stage.get("input")
|
| 445 |
+
if value is not None:
|
| 446 |
+
return value
|
| 447 |
+
return None
|