Download index.html from Mrunal1201/aya-self-healing-sql-article: direct link, hf CLI and curl.
- Browser
- Download file 96.6 kB
-
https://huggingface.co/spaces/Mrunal1201/aya-self-healing-sql-article/resolve/main/index.html
- Command line
-
hf download hf://spaces/Mrunal1201/aya-self-healing-sql-article/index.html
-
curl -L -o index.html https://huggingface.co/spaces/Mrunal1201/aya-self-healing-sql-article/resolve/main/index.html
96.6 kB
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8" /> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> | |
| <title>Self-Healing Multilingual SQL Agents on Cohere Aya | Lexsi Track 3</title> | |
| <style> | |
| :root { | |
| --bg: #f8fafc; | |
| --surface: #ffffff; | |
| --surface-alt: #f1f5f9; | |
| --text: #0f172a; | |
| --muted: #475569; | |
| --border: #e2e8f0; | |
| --primary: #1d4ed8; | |
| --primary-soft: #eff6ff; | |
| --accent: #0f766e; | |
| --accent-soft: #f0fdfa; | |
| --danger: #b91c1c; | |
| --danger-soft: #fef2f2; | |
| --code-bg: #0f172a; | |
| --code-text: #e2e8f0; | |
| --shadow: 0 1px 3px rgba(15, 23, 42, 0.08), 0 8px 24px rgba(15, 23, 42, 0.04); | |
| --radius: 12px; | |
| } | |
| * { box-sizing: border-box; } | |
| html { scroll-behavior: smooth; } | |
| body { | |
| margin: 0; | |
| font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; | |
| background: var(--bg); | |
| color: var(--text); | |
| line-height: 1.65; | |
| font-size: 16px; | |
| } | |
| a { color: var(--primary); text-decoration: none; } | |
| a:hover { text-decoration: underline; } | |
| .hero { | |
| background: linear-gradient(135deg, #0f172a 0%, #1e3a8a 58%, #312e81 100%); | |
| color: #f8fafc; | |
| padding: 2.75rem 1.5rem 2.25rem; | |
| border-bottom: 1px solid rgba(255,255,255,0.12); | |
| } | |
| .hero-inner { max-width: 1220px; margin: 0 auto; } | |
| .badge-row { display: flex; flex-wrap: wrap; gap: 0.5rem; margin-bottom: 1rem; } | |
| .badge { | |
| display: inline-block; | |
| font-size: 0.78rem; | |
| font-weight: 600; | |
| padding: 0.25rem 0.65rem; | |
| border-radius: 999px; | |
| background: rgba(255,255,255,0.14); | |
| color: #e0e7ff; | |
| border: 1px solid rgba(255,255,255,0.2); | |
| } | |
| .hero h1 { | |
| margin: 0 0 0.75rem; | |
| font-size: clamp(1.65rem, 3vw, 2.4rem); | |
| line-height: 1.2; | |
| font-weight: 800; | |
| letter-spacing: -0.02em; | |
| } | |
| .hero p.subtitle { | |
| margin: 0 0 1.25rem; | |
| max-width: 920px; | |
| color: #cbd5e1; | |
| font-size: 1.03rem; | |
| } | |
| .hero-links { display: flex; flex-wrap: wrap; gap: 0.65rem; margin-bottom: 1.5rem; } | |
| .hero-btn { | |
| display: inline-flex; | |
| align-items: center; | |
| gap: 0.4rem; | |
| padding: 0.48rem 0.9rem; | |
| border-radius: 8px; | |
| font-size: 0.88rem; | |
| font-weight: 600; | |
| background: rgba(255,255,255,0.12); | |
| color: #ffffff; | |
| border: 1px solid rgba(255,255,255,0.24); | |
| } | |
| .hero-btn.primary { background: #2563eb; border-color: #60a5fa; } | |
| .hero-btn:hover { background: rgba(255,255,255,0.22); text-decoration: none; } | |
| .kpi-grid { | |
| display: grid; | |
| grid-template-columns: repeat(auto-fit, minmax(210px, 1fr)); | |
| gap: 0.9rem; | |
| } | |
| .kpi-card { | |
| background: rgba(15, 23, 42, 0.55); | |
| border: 1px solid rgba(255, 255, 255, 0.16); | |
| border-radius: 10px; | |
| padding: 0.9rem 1rem; | |
| backdrop-filter: blur(6px); | |
| } | |
| .kpi-label { font-size: 0.76rem; text-transform: uppercase; letter-spacing: 0.05em; color: #94a3b8; } | |
| .kpi-value { font-size: 1.38rem; font-weight: 800; color: #ffffff; margin: 0.2rem 0; } | |
| .kpi-sub { font-size: 0.8rem; color: #bfdbfe; } | |
| .layout { | |
| max-width: 1220px; | |
| margin: 1.75rem auto 3rem; | |
| padding: 0 1.25rem; | |
| display: grid; | |
| grid-template-columns: 260px minmax(0, 1fr); | |
| gap: 1.75rem; | |
| align-items: start; | |
| } | |
| .sidebar { | |
| position: sticky; | |
| top: 1.25rem; | |
| background: var(--surface); | |
| border: 1px solid var(--border); | |
| border-radius: var(--radius); | |
| padding: 1rem; | |
| box-shadow: var(--shadow); | |
| max-height: calc(100vh - 2.5rem); | |
| overflow-y: auto; | |
| } | |
| .sidebar-title { | |
| font-size: 0.78rem; | |
| font-weight: 700; | |
| text-transform: uppercase; | |
| letter-spacing: 0.06em; | |
| color: var(--muted); | |
| margin-bottom: 0.6rem; | |
| } | |
| .toc-link { | |
| display: block; | |
| padding: 0.28rem 0.45rem; | |
| border-radius: 6px; | |
| font-size: 0.85rem; | |
| color: var(--muted); | |
| line-height: 1.35; | |
| } | |
| .toc-link:hover { background: var(--primary-soft); color: var(--primary); text-decoration: none; } | |
| .toc-l3 { padding-left: 1rem; font-size: 0.8rem; } | |
| .main-col { display: flex; flex-direction: column; gap: 1.5rem; min-width: 0; } | |
| .explorer-card, .article-card { | |
| background: var(--surface); | |
| border: 1px solid var(--border); | |
| border-radius: var(--radius); | |
| padding: 1.5rem 1.75rem; | |
| box-shadow: var(--shadow); | |
| } | |
| .explorer-header { | |
| display: flex; | |
| flex-wrap: wrap; | |
| justify-content: space-between; | |
| align-items: center; | |
| gap: 0.75rem; | |
| margin-bottom: 1rem; | |
| border-bottom: 1px solid var(--border); | |
| padding-bottom: 0.85rem; | |
| } | |
| .explorer-header h2 { margin: 0; font-size: 1.22rem; } | |
| .pill-row { display: flex; flex-wrap: wrap; gap: 0.45rem; } | |
| .lang-pill, .sample-pill { | |
| border: 1px solid var(--border); | |
| background: var(--surface-alt); | |
| color: var(--text); | |
| border-radius: 999px; | |
| padding: 0.35rem 0.75rem; | |
| font-size: 0.82rem; | |
| font-weight: 600; | |
| cursor: pointer; | |
| } | |
| .lang-pill.active, .sample-pill.active { | |
| background: var(--primary); | |
| color: #ffffff; | |
| border-color: var(--primary); | |
| } | |
| .trace-meta { | |
| display: flex; | |
| flex-wrap: wrap; | |
| gap: 0.5rem; | |
| margin: 0.75rem 0 1rem; | |
| } | |
| .meta-chip { | |
| font-size: 0.78rem; | |
| padding: 0.2rem 0.55rem; | |
| border-radius: 6px; | |
| background: var(--surface-alt); | |
| border: 1px solid var(--border); | |
| color: var(--muted); | |
| font-weight: 600; | |
| } | |
| .trace-grid { display: grid; grid-template-columns: 1fr; gap: 0.85rem; } | |
| .trace-step { | |
| border: 1px solid var(--border); | |
| border-radius: 10px; | |
| padding: 0.85rem 1rem; | |
| background: #ffffff; | |
| } | |
| .trace-step.error { background: var(--danger-soft); border-color: #fecaca; } | |
| .trace-step.healed { background: var(--accent-soft); border-color: #99f6e4; } | |
| .step-title { | |
| font-size: 0.78rem; | |
| font-weight: 700; | |
| text-transform: uppercase; | |
| letter-spacing: 0.04em; | |
| margin-bottom: 0.4rem; | |
| color: var(--muted); | |
| } | |
| .trace-step.error .step-title { color: var(--danger); } | |
| .trace-step.healed .step-title { color: var(--accent); } | |
| .trace-step pre { | |
| margin: 0.35rem 0 0; | |
| background: var(--code-bg); | |
| color: var(--code-text); | |
| padding: 0.75rem 0.9rem; | |
| border-radius: 8px; | |
| overflow-x: auto; | |
| font-size: 0.82rem; | |
| line-height: 1.45; | |
| white-space: pre-wrap; | |
| word-break: break-word; | |
| } | |
| .article-card h2 { | |
| margin: 2rem 0 0.75rem; | |
| padding-bottom: 0.4rem; | |
| border-bottom: 1px solid var(--border); | |
| font-size: 1.45rem; | |
| color: #0f172a; | |
| } | |
| .article-card h2:first-child { margin-top: 0.25rem; } | |
| .article-card h3 { margin: 1.45rem 0 0.55rem; font-size: 1.12rem; color: #1e293b; } | |
| .article-card p { margin: 0.65rem 0; } | |
| .article-card ul, .article-card ol { padding-left: 1.35rem; margin: 0.6rem 0; } | |
| .article-card li { margin: 0.35rem 0; } | |
| .article-card code { | |
| font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; | |
| font-size: 0.86em; | |
| background: var(--surface-alt); | |
| padding: 0.12rem 0.35rem; | |
| border-radius: 5px; | |
| color: #0f172a; | |
| } | |
| .code-block, .mermaid-card { | |
| margin: 0.9rem 0; | |
| border-radius: 10px; | |
| overflow: hidden; | |
| border: 1px solid #1e293b; | |
| background: var(--code-bg); | |
| } | |
| .code-label { | |
| font-size: 0.72rem; | |
| text-transform: uppercase; | |
| letter-spacing: 0.06em; | |
| padding: 0.35rem 0.85rem; | |
| background: #1e293b; | |
| color: #94a3b8; | |
| font-weight: 700; | |
| } | |
| .code-block pre, .mermaid-card pre { | |
| margin: 0; | |
| padding: 0.9rem 1rem; | |
| overflow-x: auto; | |
| color: var(--code-text); | |
| font-size: 0.84rem; | |
| line-height: 1.5; | |
| } | |
| .code-block pre code, .mermaid-card pre code { | |
| background: transparent; | |
| color: inherit; | |
| padding: 0; | |
| } | |
| blockquote { | |
| margin: 1rem 0; | |
| padding: 0.85rem 1.1rem; | |
| background: var(--primary-soft); | |
| border-left: 4px solid var(--primary); | |
| border-radius: 0 8px 8px 0; | |
| color: #1e293b; | |
| } | |
| blockquote p, blockquote ul { margin: 0.3rem 0; } | |
| .table-wrap { | |
| width: 100%; | |
| overflow-x: auto; | |
| margin: 1rem 0; | |
| border: 1px solid var(--border); | |
| border-radius: 10px; | |
| } | |
| table { width: 100%; border-collapse: collapse; font-size: 0.88rem; } | |
| th { | |
| background: var(--surface-alt); | |
| color: #0f172a; | |
| font-weight: 700; | |
| padding: 0.62rem 0.75rem; | |
| border-bottom: 1px solid var(--border); | |
| white-space: nowrap; | |
| } | |
| td { | |
| padding: 0.55rem 0.75rem; | |
| border-bottom: 1px solid var(--border); | |
| vertical-align: top; | |
| } | |
| tbody tr:last-child td { border-bottom: none; } | |
| tbody tr:nth-child(even) { background: #f8fafc; } | |
| .doc-figure { | |
| margin: 1.25rem 0; | |
| padding: 0.75rem; | |
| background: #ffffff; | |
| border: 1px solid var(--border); | |
| border-radius: 10px; | |
| text-align: center; | |
| } | |
| .doc-figure img { max-width: 100%; height: auto; border-radius: 6px; } | |
| .doc-figure figcaption { margin-top: 0.5rem; font-size: 0.84rem; color: var(--muted); } | |
| hr { border: 0; border-top: 1px solid var(--border); margin: 1.6rem 0; } | |
| @media (max-width: 960px) { | |
| .layout { grid-template-columns: 1fr; } | |
| .sidebar { position: static; max-height: none; } | |
| .explorer-card, .article-card { padding: 1.1rem; } | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <header class="hero"> | |
| <div class="hero-inner"> | |
| <div class="badge-row"> | |
| <span class="badge">Lexsi Labs Track 3: Data & Database Agents</span> | |
| <span class="badge">CuratorKIT 0.1.21 → AgentTune 1.1.0 → AuditKIT 1.1.12</span> | |
| <span class="badge">Cohere Aya 3.35B & 8B · Apache-2.0</span> | |
| </div> | |
| <h1>Self-Healing Multilingual SQL Agents on Cohere Aya</h1> | |
| <p class="subtitle"> | |
| Trajectory distillation, AST-level SQLGlot guardrails, and verified SQLite execution auditing across | |
| English (<code>en</code>), Hindi (<code>hi</code>), German (<code>de</code>), and Kannada (<code>kn</code>) | |
| with 0% held-out domain, table, or near-duplicate leakage. | |
| </p> | |
| <div class="hero-links"> | |
| <a class="hero-btn primary" href="https://github.com/Mru1201/aya-self-healing-sql-agent" target="_blank" rel="noopener noreferrer">GitHub Repository</a> | |
| <a class="hero-btn" href="https://huggingface.co/collections/Mrunal1201/lexsi-track-3-self-healing-sql-aya-models-6af8e7fc0a621b28fefccd18" target="_blank" rel="noopener noreferrer">HF Model Collection</a> | |
| <a class="hero-btn" href="https://huggingface.co/datasets/Mrunal1201/aya-self-healing-sql-dataset" target="_blank" rel="noopener noreferrer">HF Dataset & Audit Logs</a> | |
| <a class="hero-btn" href="https://huggingface.co/Mrunal1201/tiny-aya-sql-heal-lora/discussions/1" target="_blank" rel="noopener noreferrer">Model Discussion #1</a> | |
| <a class="hero-btn" href="#trace-explorer">Interactive Trace Explorer</a> | |
| </div> | |
| <div class="kpi-grid"> | |
| <div class="kpi-card"> | |
| <div class="kpi-label">Aya-Expanse 8B (AuditKIT Verified)</div> | |
| <div class="kpi-value">57.0% → 63.5%</div> | |
| <div class="kpi-sub">+6.5 pts vs Base · 96.0% SQL Validity</div> | |
| </div> | |
| <div class="kpi-card"> | |
| <div class="kpi-label">Tiny-Aya 3.35B Trajectory SFT</div> | |
| <div class="kpi-value">56.5% → 60.0%</div> | |
| <div class="kpi-sub">91.0% SQL Validity vs 86.0% 1-Turn Control</div> | |
| </div> | |
| <div class="kpi-card"> | |
| <div class="kpi-label">Turn-1 OperationalError Reduction</div> | |
| <div class="kpi-value">33 → 18 → 8</div> | |
| <div class="kpi-sub">-45.5% (3.35B Heal) · -75.8% (8B Heal)</div> | |
| </div> | |
| <div class="kpi-card"> | |
| <div class="kpi-label">Aya-8B L4 Inference Cost</div> | |
| <div class="kpi-value">$0.3076 / 1k</div> | |
| <div class="kpi-sub">35.67 tok/s vs 18.95 tok/s (3.35B)</div> | |
| </div> | |
| </div> | |
| </div> | |
| </header> | |
| <div class="layout"> | |
| <aside class="sidebar"> | |
| <div class="sidebar-title">Report Navigation</div> | |
| <a class="toc-link" href="#trace-explorer">Interactive Self-Healing Traces</a> | |
| <a class="toc-link toc-l2" href="#1-why-small-multilingual-database-agents-fail-and-how-we-framed-track-3">1. Why Small Multilingual Database Agents Fail—and How We Framed Track 3</a> | |
| <a class="toc-link toc-l3" href="#the-track-3-threat-model-t3">The Track 3 Threat Model [T3]</a> | |
| <a class="toc-link toc-l3" href="#kit-architecture-curatorkit-agenttune-auditkit-with-sqlglot-sqlite">Kit Architecture: CuratorKIT → AgentTune → AuditKIT with SQLGlot & SQLite</a> | |
| <a class="toc-link toc-l2" href="#2-dataset-curation-multilingual-coverage-zero-leakage-audit">2. Dataset Curation, Multilingual Coverage & Zero-Leakage Audit</a> | |
| <a class="toc-link toc-l3" href="#curatorkit-0-1-21-hygiene-deduplication-funnel-01curationprovenance-json">CuratorKIT 0.1.21 Hygiene & Deduplication Funnel (01_curation_provenance.json)</a> | |
| <a class="toc-link toc-l3" href="#language-episode-type-attack-distributions">Language, Episode Type & Attack Distributions</a> | |
| <a class="toc-link toc-l3" href="#zero-leakage-guarantee-02leakcheckreport-json">Zero-Leakage Guarantee (02_leak_check_report.json)</a> | |
| <a class="toc-link toc-l2" href="#3-main-before-after-results-95-bootstrap-confidence-intervals">3. Main Before → After Results & 95% Bootstrap Confidence Intervals</a> | |
| <a class="toc-link toc-l3" href="#paired-bootstrap-significance-tests-auditkit-bootstrap-pairedbootstrap">Paired Bootstrap Significance Tests (auditkit._bootstrap.paired_bootstrap)</a> | |
| <a class="toc-link toc-l2" href="#4-dual-ablation-studies">4. Dual Ablation Studies</a> | |
| <a class="toc-link toc-l3" href="#ablation-1-supervision-structure-tiny-aya-sql-heal-lora-vs-tiny-aya-sql-control-lora">Ablation 1: Supervision Structure (tiny-aya-sql-heal-lora vs. tiny-aya-sql-control-lora)</a> | |
| <a class="toc-link toc-l3" href="#ablation-2-model-scale-coherelabs-tiny-aya-global-3-35b-vs-coherelabs-aya-expanse-8b-8b">Ablation 2: Model Scale (CohereLabs/tiny-aya-global 3.35B vs. CohereLabs/aya-expanse-8b 8B)</a> | |
| <a class="toc-link toc-l2" href="#5-safety-guardrails-multilingual-retention-honest-failure-forensics">5. Safety Guardrails, Multilingual Retention & Honest Failure Forensics</a> | |
| <a class="toc-link toc-l3" href="#adversarial-sql-injection-probes-n-50">Adversarial SQL Injection Probes (n=50)</a> | |
| <a class="toc-link toc-l3" href="#general-multilingual-reasoning-retention-n-50-across-10-languages">General Multilingual Reasoning Retention (n=50 Across 10 Languages)</a> | |
| <a class="toc-link toc-l3" href="#forensics-of-remaining-failure-modes-the-tiny-aya-366-token-preamble">Forensics of Remaining Failure Modes & the Tiny Aya 366-Token Preamble</a> | |
| <a class="toc-link toc-l2" href="#6-compute-hyperparameters-cost-roi-economics">6. Compute, Hyperparameters & Cost / ROI Economics</a> | |
| <a class="toc-link toc-l3" href="#cost-throughput-per-1-000-multi-turn-queries-05roithroughput-json">Cost & Throughput per 1,000 Multi-Turn Queries (05_roi_throughput.json)</a> | |
| <a class="toc-link toc-l2" href="#7-reproduce-everything-from-scratch">7. Reproduce Everything from Scratch</a> | |
| </aside> | |
| <main class="main-col"> | |
| <section class="explorer-card" id="trace-explorer"> | |
| <div class="explorer-header"> | |
| <div> | |
| <h2>Interactive Multilingual AuditKIT Self-Healing Trace Explorer</h2> | |
| <div style="font-size:0.86rem;color:var(--muted);"> | |
| Inspect held-out unseen-domain episodes from <code>data/curated/test_heldout.jsonl</code> (English, Hindi, German, Kannada). | |
| </div> | |
| </div> | |
| <div class="pill-row" id="lang-pills"></div> | |
| </div> | |
| <div class="pill-row" id="sample-pills" style="margin-bottom:0.75rem;"></div> | |
| <div class="trace-meta" id="trace-meta"></div> | |
| <div class="trace-grid" id="trace-body"></div> | |
| </section> | |
| <article class="article-card"> | |
| <p><strong>TL;DR:</strong> Using <strong><code>CuratorKIT 0.1.21</code></strong>, <strong><code>AgentTune 1.1.0</code> QLoRA Trajectory Distillation</strong>, <strong><code>SQLGlot 30.31.0</code> AST Guardrails</strong>, and <strong><code>AuditKIT 1.1.12</code></strong>, we trained multilingual (<code>en</code>, <code>hi</code>, <code>de</code>, <code>kn</code>) self-healing SQL adapters on <strong><code>CohereLabs/tiny-aya-global</code> (3.35B)</strong> and <strong><code>CohereLabs/aya-expanse-8b</code> (8B)</strong>. On 200 strictly disjoint held-out database tasks (<code>0</code> domain overlap, <code>0</code> table overlap), execution accuracy improved <strong>57.5% → 60.0%</strong> (+2.5 pp final; <strong>56.5% → 60.0%</strong> Turn-1) on 3.35B and <strong>57.5% → 63.5%</strong> (+6.0 pp final; <strong>57.0% → 63.5%</strong> Turn-1) on 8B, while raising executable SQL validity over single-turn SFT (<code>86.0%</code>) to <strong>91.0%</strong> (+5.0 pp) on 3.35B and <strong>96.0%</strong> (+10.0 pp) on 8B—cutting Turn-1 SQLite <code>OperationalError</code> rates by <strong>45.5%</strong> (<code>33 → 18</code>) and <strong>75.8%</strong> (<code>33 → 8</code>), respectively, with <strong>96.0%</strong> multilingual reasoning retention across 10 languages for <strong>USD 0.50 total project R&D compute</strong>.</p> | |
| <blockquote><p><strong>Headline Before → After Summary (<code>AuditKIT v1.1.12</code>, 200 Held-Out Cases, 1,000 Bootstrap Resamples)</strong></p> | |
| <ul><li><strong><code>CohereLabs/tiny-aya-global</code> (3.35B):</strong> Turn-1 Accuracy <strong>56.5% → 60.0%</strong> (+3.5 pp) | Final Accuracy <strong>57.5% → 60.0%</strong> (+2.5 pp, 95% CI <code>[52.5%, 66.01%]</code>) | Executable SQL Validity <strong>86.0% (Control) → 91.0%</strong> (+5.0 pp) | Turn-1 <code>OperationalError</code> Failures <strong>33 → 18</strong> (-45.5%) | Multilingual Retention <strong>96.0%</strong> (0.0 pp loss)</li><li><strong><code>CohereLabs/aya-expanse-8b</code> (8B):</strong> Turn-1 Accuracy <strong>57.0% → 63.5%</strong> (+6.5 pp) | Final Accuracy <strong>57.5% → 63.5%</strong> (+6.0 pp, 95% CI <code>[57.0%, 70.0%]</code>) | Executable SQL Validity <strong>86.0% (Control) → 96.0%</strong> (+10.0 pp) | Turn-1 <code>OperationalError</code> Failures <strong>33 → 8</strong> (-75.8%) | Mean Query Latency <strong>2,307.3 ms</strong> (faster than a <strong>2,891.0 ms</strong> single-call cloud teacher API) | Peak VRAM <strong>13.898 GB</strong></li></ul></blockquote> | |
| <p><strong>Base Models & Method:</strong> <code>CohereLabs/tiny-aya-global</code> (3.35B) & <code>CohereLabs/aya-expanse-8b</code> (8B) via <code>CuratorKIT</code> + <code>AgentTune</code> QLoRA Trajectory Distillation (<code>max_seq_length=1536</code>, <code>completion_only</code> loss masking) + <code>SQLGlot</code> AST Guardrails + <code>AuditKIT</code> Paired Bootstrap Evaluation.</p> | |
| <div class="table-wrap"><table><thead><tr><th style="text-align:left">Resource</th><th style="text-align:left">Link</th></tr></thead><tbody><tr><td style="text-align:left"><strong>Public GitHub Repository</strong></td><td style="text-align:left"><a href="https://github.com/Mru1201/aya-self-healing-sql-agent" target="_blank" rel="noopener noreferrer"><code>Mru1201/aya-self-healing-sql-agent</code></a></td></tr><tr><td style="text-align:left"><strong>Public HF Model Collection</strong></td><td style="text-align:left"><a href="https://huggingface.co/collections/Mrunal1201/lexsi-track-3-self-healing-sql-aya-models-6af8e7fc0a621b28fefccd18" target="_blank" rel="noopener noreferrer">Lexsi Track 3 — Self-Healing SQL Aya Models</a></td></tr><tr><td style="text-align:left"><strong>Public HF Dataset Collection</strong></td><td style="text-align:left"><a href="https://huggingface.co/collections/Mrunal1201/lexsi-track-3-curated-sql-trajectories-and-audit-logs-6af8e7fed8ac99740564a2fb" target="_blank" rel="noopener noreferrer">Lexsi Track 3 — Curated SQL Trajectories & Audit Logs</a></td></tr><tr><td style="text-align:left"><strong>Primary 3.35B Multi-Turn Adapter</strong></td><td style="text-align:left"><a href="https://huggingface.co/Mrunal1201/tiny-aya-sql-heal-lora" target="_blank" rel="noopener noreferrer"><code>Mrunal1201/tiny-aya-sql-heal-lora</code></a></td></tr><tr><td style="text-align:left"><strong>3.35B Single-Turn Ablation Control</strong></td><td style="text-align:left"><a href="https://huggingface.co/Mrunal1201/tiny-aya-sql-control-lora" target="_blank" rel="noopener noreferrer"><code>Mrunal1201/tiny-aya-sql-control-lora</code></a></td></tr><tr><td style="text-align:left"><strong>8B Scale-Ablation Adapter</strong></td><td style="text-align:left"><a href="https://huggingface.co/Mrunal1201/aya-8b-sql-heal-lora" target="_blank" rel="noopener noreferrer"><code>Mrunal1201/aya-8b-sql-heal-lora</code></a></td></tr><tr><td style="text-align:left"><strong>Curated Dataset & Audit Logs</strong></td><td style="text-align:left"><a href="https://huggingface.co/datasets/Mrunal1201/aya-self-healing-sql-dataset" target="_blank" rel="noopener noreferrer"><code>Mrunal1201/aya-self-healing-sql-dataset</code></a></td></tr><tr><td style="text-align:left"><strong>Executed Colab Notebook</strong></td><td style="text-align:left"><a href="https://github.com/Mru1201/aya-self-healing-sql-agent/blob/main/notebooks/train_and_audit_aya_sql.ipynb" target="_blank" rel="noopener noreferrer"><code>notebooks/train_and_audit_aya_sql.ipynb</code></a></td></tr><tr><td style="text-align:left"><strong>HF Static Article Space</strong></td><td style="text-align:left"><a href="https://huggingface.co/spaces/Mrunal1201/aya-self-healing-sql-article" target="_blank" rel="noopener noreferrer">Static Space (<code>Mrunal1201/aya-self-healing-sql-article</code>)</a></td></tr><tr><td style="text-align:left"><strong>Direct Static Space Embed</strong></td><td style="text-align:left"><a href="https://mrunal1201-aya-self-healing-sql-article.static.hf.space" target="_blank" rel="noopener noreferrer">Interactive HTML Report (<code>static.hf.space</code>)</a></td></tr><tr><td style="text-align:left"><strong>HF Community Article (Model Repo)</strong></td><td style="text-align:left"><a href="https://huggingface.co/Mrunal1201/tiny-aya-sql-heal-lora/discussions/1" target="_blank" rel="noopener noreferrer">Model Repo Discussion #1</a></td></tr><tr><td style="text-align:left"><strong>HF Community Article (Dataset Repo)</strong></td><td style="text-align:left"><a href="https://huggingface.co/datasets/Mrunal1201/aya-self-healing-sql-dataset/discussions/1" target="_blank" rel="noopener noreferrer">Dataset Repo Discussion #1</a></td></tr></tbody></table></div> | |
| <hr /> | |
| <h2 id="1-why-small-multilingual-database-agents-fail-and-how-we-framed-track-3">1. Why Small Multilingual Database Agents Fail—and How We Framed Track 3</h2> | |
| <p>Organizations running analytics over proprietary relational databases want two things that rarely coexist: <strong>data sovereignty</strong> (never shipping internal DDL schemas, row samples, and SQLite error traces to external APIs) and <strong>multilingual accessibility</strong> (supporting analysts querying in English, Hindi, German, or Kannada). This directly targets <strong>Lexsi Hackathon Track 3</strong>:</p> | |
| <ul><li><strong>Problem Statement #6 (Data and Database Agents):</strong> Converting natural-language questions across multiple scripts and grammars into strictly schema-grounded SQL.</li><li><strong>Problem Statement #8 (Self-Healing Agents):</strong> Recovering autonomously when generated queries encounter database runtime errors rather than halting or hallucinating new identifiers.</li></ul> | |
| <h3 id="the-track-3-threat-model-t3">The Track 3 Threat Model <code>[T3]</code></h3> | |
| <p>Deploying an open-weights <code>3.35B</code> or <code>8B</code> model in front of a SQL database exposes two distinct risk surfaces:</p> | |
| <ol><li><strong>Schema Hallucination on Unseen Tables:</strong> Models frequently guess plausible-sounding column or table names (<code>amount</code>, <code>gender</code>, <code>users_archive</code>) instead of binding strictly to the supplied <code>CREATE TABLE</code> DDL, causing <code>sqlite3.OperationalError: no such column</code> or <code>no such table</code>.</li><li><strong>Adversarial Prompt Injections & Destructive Writes:</strong> Malicious or compromised user turns inject instructions attempting <code>DROP TABLE</code>, <code>DELETE FROM</code>, <code>UPDATE</code>, <code>ALTER TABLE</code>, stacked multi-statement execution (<code>;</code>), or <code>UNION SELECT</code> credential extraction against <code>sqlite_master</code>.</li></ol> | |
| <h3 id="kit-architecture-curatorkit-agenttune-auditkit-with-sqlglot-sqlite">Kit Architecture: <code>CuratorKIT</code> → <code>AgentTune</code> → <code>AuditKIT</code> with <code>SQLGlot</code> & <code>SQLite</code></h3> | |
| <p>Instead of relying purely on synthetic LLM chat logs or unverified string matching, we chained all three Lexsi toolkits with deterministic <code>SQLGlot 30.31.0</code> AST parsing and in-memory <code>SQLite 3.46.0</code> execution:</p> | |
| <div class="mermaid-card"><div class="code-label">Architecture Flow (Mermaid)</div><pre class="mermaid-source"><code>flowchart LR | |
| Seed["gretelai/synthetic_text_to_sql"] --> Split["Disjoint Split: 80 Train / 20 Test Domains"] | |
| Split --> Mutate["SQLGlot 30.31.0 AST Mutation (Column / Table)"] | |
| Mutate --> SQLiteGen["In-Memory SQLite 3.46.0 -> Real OperationalError"] | |
| SQLiteGen --> Teacher["HF Router Localization: Qwen2.5-72B (en, hi, de, kn)"] | |
| Teacher --> Curator["CuratorKIT 0.1.21: SchemaGate + Exact/MinHash (0.85) + SecretsGate"] | |
| Curator --> Leak["Leak Check Gate: 0 Domain / 0 Table / 0 LSH Overlap"] | |
| Leak --> AgentTune["AgentTune 1.1.0: 4-bit QLoRA (max_seq_length=1536, completion_only)"] | |
| AgentTune --> Agent["Runtime Loop: Aya LoRA -> SQLGlot Read-Only AST Check -> SQLite"] | |
| Agent -->|OperationalError| Repair["Error Feedback -> Thought: Schema Check -> Repaired SQL"] | |
| Repair --> Agent | |
| Agent --> Audit["AuditKIT 1.1.12: Verified-Outcome Oracles & 1,000-Resample CIs"]</code></pre></div> | |
| <figure class="doc-figure"><img src="assets/figures/fig0_architecture_pipeline.png" alt="End-to-End Lexsi Track 3 Architecture Pipeline" loading="lazy" /><figcaption>End-to-End Lexsi Track 3 Architecture Pipeline</figcaption></figure> | |
| <p><em>Figure 0: End-to-end closed-loop Track 3 pipeline uniting <code>CuratorKIT 0.1.21</code> curation & leak checks, <code>AgentTune 1.1.0</code> QLoRA trajectory distillation, <code>SQLGlot 30.31.0</code> read-only AST guardrails over in-memory <code>SQLite 3.46.0</code>, and <code>AuditKIT 1.1.12</code> outcome verification.</em></p> | |
| <p>Here is what an actual <code>self_healing</code> training trajectory looks like after <code>SQLGlot</code> AST mutation, <code>SQLite</code> error capture, and <code>CuratorKIT</code> validation (<code>id: train_0982_en_self_healing</code>):</p> | |
| <div class="code-block"><div class="code-label">text</div><pre><code class="language-text">[SYSTEM] You are a multilingual SQLite data assistant. Translate the user's question into a single | |
| read-only SQLite SELECT query using ONLY the provided schema. ... | |
| Schema (SQLite DDL): | |
| CREATE TABLE warehouse (item_name TEXT, item_type TEXT, weight INTEGER); | |
| INSERT INTO warehouse (item_name, item_type, weight) VALUES ('Carrots', 'organic_produce', 500), ...; | |
| [USER] What is the total weight of organic_produce in the warehouse table? | |
| [ASSISTANT - Turn 1 (SQLGlot AST Mutated)] | |
| ```sql | |
| SELECT SUM(weight_legacy) FROM warehouse WHERE item_type = 'organic_produce' | |
| ``` | |
| [USER - Real SQLite 3.46.0 Feedback] | |
| SQLite Execution Error (sqlite3.OperationalError): no such column: weight_legacy | |
| Please inspect the schema and fix the SQL query. | |
| [ASSISTANT - Turn 2 (Supervised Completion Only)] | |
| Thought: The previous query failed with OperationalError (no such column: weight_legacy). Checking | |
| the schema for table(s) warehouse, This query sums (SUM) the weight column in the warehouse table | |
| where item_type is 'organic_produce'. | |
| ```sql | |
| SELECT SUM(weight) FROM warehouse WHERE item_type = 'organic_produce'; | |
| ```</code></pre></div> | |
| <p>All stack dependencies (<code>curatorkit 0.1.21</code>, <code>auditkit 1.1.12</code>, <code>sqlglot 30.31.0</code>, <code>datasketch 2.0.0</code>, <code>sqlite3 3.46.0</code>, <code>agenttune 1.1.0</code>) and HF Router endpoint connectivity (<code>Qwen/Qwen2.5-72B-Instruct:cheapest</code>, <code>1015.73 ms</code>, zero fallback calls, zero Gemini API dependencies) are logged in <code>audit_logs/00_stack_verified.json</code>.</p> | |
| <hr /> | |
| <h2 id="2-dataset-curation-multilingual-coverage-zero-leakage-audit">2. Dataset Curation, Multilingual Coverage & Zero-Leakage Audit</h2> | |
| <p>We sourced base DDL contexts and gold queries from <code>gretelai/synthetic_text_to_sql</code> (<code>seed=42</code>, curation wall-clock <code>328.99 s</code>). Rather than letting a teacher LLM hallucinate fake SQL error messages, we deterministically mutated column and table AST nodes with <code>SQLGlot</code>, ran the mutated SQL against an in-memory <code>sqlite3</code> database, and recorded the verbatim <code>OperationalError</code>. Multilingual questions (<code>hi</code>, <code>de</code>, <code>kn</code>) were localized via <code>Qwen/Qwen2.5-72B-Instruct:cheapest</code> across <code>109</code> HF Router API calls (<code>59,431</code> total tokens: <code>17,782</code> prompt + <code>41,649</code> completion; total localization cost <strong>USD 0.05054</strong> under a USD 0.60 cap, <code>0</code> retries, <code>0</code> fallbacks).</p> | |
| <h3 id="curatorkit-0-1-21-hygiene-deduplication-funnel-01curationprovenance-json"><code>CuratorKIT 0.1.21</code> Hygiene & Deduplication Funnel (<code>01_curation_provenance.json</code>)</h3> | |
| <p><code>CuratorConfig</code> enforced <code>format="sharegpt"</code>, <code>min_tokens=10</code>, <code>max_tokens=1536</code>, <code>dedup="minhash"</code> (<code>threshold=0.85</code>, <code>num_perm=128</code>), and <code>secrets_gate=True</code> (<code>secrets_hex_limit=4.5</code>, <code>secrets_base64_limit=5.5</code>), filtering out 30 injected synthetic noise candidates in the raw pool:</p> | |
| <div class="table-wrap"><table><thead><tr><th style="text-align:left">Curation Stage</th><th style="text-align:center">Train Input</th><th style="text-align:center">Train Output</th><th style="text-align:center">Train Filtered</th><th style="text-align:center">Test Input</th><th style="text-align:center">Test Output</th><th style="text-align:center">Test Filtered</th></tr></thead><tbody><tr><td style="text-align:left">Raw Candidate Pool</td><td style="text-align:center">—</td><td style="text-align:center"><code>1,313</code></td><td style="text-align:center"><code>0</code></td><td style="text-align:center">—</td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><code>0</code></td></tr><tr><td style="text-align:left"><code>JSONLReader</code></td><td style="text-align:center"><code>1,313</code></td><td style="text-align:center"><code>1,313</code></td><td style="text-align:center"><code>0</code></td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><code>0</code></td></tr><tr><td style="text-align:left"><code>SchemaGate</code></td><td style="text-align:center"><code>1,313</code></td><td style="text-align:center"><code>1,305</code></td><td style="text-align:center"><code>8</code> (<code>below_min_tokens:4</code>)</td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><code>0</code></td></tr><tr><td style="text-align:left"><code>ExactDeduplicator</code></td><td style="text-align:center"><code>1,305</code></td><td style="text-align:center"><code>1,299</code></td><td style="text-align:center"><code>6</code> exact duplicates</td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><code>0</code></td></tr><tr><td style="text-align:left"><code>MinHashDeduplicator</code> (<code>0.85</code>, <code>128</code> perm)</td><td style="text-align:center"><code>1,299</code></td><td style="text-align:center"><code>1,287</code></td><td style="text-align:center"><code>12</code> near-duplicates</td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><code>0</code></td></tr><tr><td style="text-align:left"><code>TextCleaner</code></td><td style="text-align:center"><code>1,287</code></td><td style="text-align:center"><code>1,287</code></td><td style="text-align:center"><code>0</code></td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><code>0</code></td></tr><tr><td style="text-align:left"><code>SecretsGate</code></td><td style="text-align:center"><code>1,287</code></td><td style="text-align:center"><code>1,283</code></td><td style="text-align:center"><code>4</code> (<code>1</code> JWT, <code>2</code> Basic Auth, <code>1</code> Private Key)</td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><code>0</code></td></tr><tr><td style="text-align:left"><code>MessagesExporter</code></td><td style="text-align:center"><code>1,283</code></td><td style="text-align:center"><code>1,283</code></td><td style="text-align:center"><code>0</code></td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><code>0</code></td></tr><tr><td style="text-align:left"><strong>Final Stratified Split</strong></td><td style="text-align:center"><code>1,283</code></td><td style="text-align:center"><strong><code>1,200</code></strong></td><td style="text-align:center">—</td><td style="text-align:center"><code>225</code></td><td style="text-align:center"><strong><code>200</code></strong></td><td style="text-align:center">—</td></tr></tbody></table></div> | |
| <h3 id="language-episode-type-attack-distributions">Language, Episode Type & Attack Distributions</h3> | |
| <div class="table-wrap"><table><thead><tr><th style="text-align:left">Split</th><th style="text-align:center">Episodes</th><th style="text-align:center"><code>self_healing</code></th><th style="text-align:center"><code>single_turn</code></th><th style="text-align:center"><code>en</code></th><th style="text-align:center"><code>de</code></th><th style="text-align:center"><code>hi</code></th><th style="text-align:center"><code>kn</code></th><th style="text-align:center"><code>column_not_found</code></th><th style="text-align:center"><code>table_not_found</code></th><th style="text-align:center">Domains</th></tr></thead><tbody><tr><td style="text-align:left"><strong>Train</strong></td><td style="text-align:center"><strong>1,200</strong></td><td style="text-align:center"><code>780</code> (<code>65%</code>)</td><td style="text-align:center"><code>420</code> (<code>35%</code>)</td><td style="text-align:center"><code>840</code> (<code>70%</code>)</td><td style="text-align:center"><code>120</code> (<code>10%</code>)</td><td style="text-align:center"><code>120</code> (<code>10%</code>)</td><td style="text-align:center"><code>120</code> (<code>10%</code>)</td><td style="text-align:center"><code>504</code></td><td style="text-align:center"><code>276</code></td><td style="text-align:center"><code>80</code></td></tr><tr><td style="text-align:left"><strong>Held-Out Test</strong></td><td style="text-align:center"><strong>200</strong></td><td style="text-align:center"><code>130</code> (<code>65%</code>)</td><td style="text-align:center"><code>70</code> (<code>35%</code>)</td><td style="text-align:center"><code>140</code> (<code>70%</code>)</td><td style="text-align:center"><code>20</code> (<code>10%</code>)</td><td style="text-align:center"><code>20</code> (<code>10%</code>)</td><td style="text-align:center"><code>20</code> (<code>10%</code>)</td><td style="text-align:center"><code>92</code></td><td style="text-align:center"><code>38</code></td><td style="text-align:center"><code>20</code></td></tr><tr><td style="text-align:left"><strong>Adversarial Probes</strong></td><td style="text-align:center"><strong>50</strong></td><td style="text-align:center">—</td><td style="text-align:center">—</td><td style="text-align:center"><code>20</code> (<code>40%</code>)</td><td style="text-align:center"><code>10</code> (<code>20%</code>)</td><td style="text-align:center"><code>10</code> (<code>20%</code>)</td><td style="text-align:center"><code>10</code> (<code>20%</code>)</td><td style="text-align:center">—</td><td style="text-align:center">—</td><td style="text-align:center">—</td></tr></tbody></table></div> | |
| <p>The 50 Track 3 adversarial probes span six distinct exploit vectors: <code>delete_from</code> (<code>9</code>), <code>drop_table</code> (<code>9</code>), <code>alter_table</code> (<code>8</code>), <code>stacked_query</code> (<code>8</code>), <code>union_credential_extraction</code> (<code>8</code>), and <code>update_injection</code> (<code>8</code>).</p> | |
| <h3 id="zero-leakage-guarantee-02leakcheckreport-json">Zero-Leakage Guarantee (<code>02_leak_check_report.json</code>)</h3> | |
| <p>To ensure evaluation measures true generalization to unseen schemas and domains, we enforced strict domain-level partitioning followed by cross-split deduplication checks:</p> | |
| <div class="table-wrap"><table><thead><tr><th style="text-align:left">Contamination Check</th><th style="text-align:center">Train Pool</th><th style="text-align:center">Held-Out Test Pool</th><th style="text-align:center">Overlap Measured</th><th style="text-align:center">Result</th></tr></thead><tbody><tr><td style="text-align:left"><strong>Domain Disjointness</strong></td><td style="text-align:center"><code>80</code> domains</td><td style="text-align:center"><code>20</code> domains</td><td style="text-align:center"><strong><code>0</code></strong> shared domains</td><td style="text-align:center"><strong>Passed</strong></td></tr><tr><td style="text-align:left"><strong>Schema Table Overlap</strong></td><td style="text-align:center"><code>933</code> unique tables</td><td style="text-align:center"><code>185</code> unique tables</td><td style="text-align:center"><strong><code>0</code></strong> shared tables</td><td style="text-align:center"><strong>Passed</strong></td></tr><tr><td style="text-align:left"><strong>Exact Question Match</strong></td><td style="text-align:center"><code>1,200</code> prompts</td><td style="text-align:center"><code>200</code> prompts</td><td style="text-align:center"><strong><code>0</code></strong> localized / <strong><code>0</code></strong> English</td><td style="text-align:center"><strong>Passed</strong></td></tr><tr><td style="text-align:left"><strong>MinHash LSH (<code>threshold=0.85</code>, <code>num_perm=128</code>)</strong></td><td style="text-align:center"><code>1,200</code> episodes</td><td style="text-align:center"><code>200</code> episodes</td><td style="text-align:center"><strong><code>0</code></strong> localized / <strong><code>0</code></strong> English</td><td style="text-align:center"><strong>Passed</strong></td></tr><tr><td style="text-align:left"><strong><code>CuratorKIT</code> Leakage Report</strong></td><td style="text-align:center"><code>groups_in_multiple_splits</code></td><td style="text-align:center">—</td><td style="text-align:center"><strong><code>0</code></strong> groups, <strong><code>0</code></strong> exact, <strong><code>0</code></strong> near-dups</td><td style="text-align:center"><strong>Passed</strong></td></tr></tbody></table></div> | |
| <p>Immutable SHA-256 checksums (<code>04_auditkit_before_after.json</code>): <code>train.jsonl</code> (<code>4e44f530a8a1d705bba6f36fc707e3e5e4791093fdf00c4a16437eae684b6f7d</code>), <code>test.jsonl</code> (<code>26fedb21b23c595ad6e26b75a3848d3970072cbf0fa398a605852b6117e383b5</code>), and <code>adversarial_probes.jsonl</code> (<code>aa41eaafe0e61d61b197aaa5dd682bcac0d873162f9c96664ff2344b01de156b</code>).</p> | |
| <hr /> | |
| <h2 id="3-main-before-after-results-95-bootstrap-confidence-intervals">3. Main Before → After Results & 95% Bootstrap Confidence Intervals</h2> | |
| <p>Every model was evaluated with greedy decoding (<code>max_new_tokens=384</code> for SQL, <code>64</code> for multilingual arithmetic retention; 4-bit NF4 / <code>bfloat16</code> compute locally) under <code>AuditKIT v1.1.12</code> deterministic verified-outcome predicates (<code>FinalStateAssertion(exec_match)</code>, <code>FinalStateAssertion(injection_safe)</code>, and <code>AnswerAssertion(mode='contains')</code>) with <code>1,000</code> bootstrap resamples (<code>95%</code> CI, <code>seed=42</code>, zero unknown/error omissions):</p> | |
| <div class="table-wrap"><table><thead><tr><th style="text-align:left">Model Configuration</th><th style="text-align:center">Turn-1 Exec Accuracy [95% CI]</th><th style="text-align:center">Final Self-Healed Exec Accuracy [95% CI]</th><th style="text-align:center">Executable SQL Validity Rate [95% CI]</th><th style="text-align:center">Self-Healing Recovery (<code>n</code> T1 Errors)</th><th style="text-align:center">Mean Turns</th><th style="text-align:center">Adversarial Safety [95% CI]</th><th style="text-align:center">Multilingual Retention [95% CI]</th><th style="text-align:center">Mean Latency (<code>sql_heal</code>, ms)</th></tr></thead><tbody><tr><td style="text-align:left"><strong><code>tiny-aya-global</code> Base (3.35B)</strong></td><td style="text-align:center">56.5% <code>[49.5%, 63.0%]</code></td><td style="text-align:center">57.5% <code>[50.5%, 64.0%]</code> (<code>115/200</code>)</td><td style="text-align:center">95.5% <code>[92.5%, 98.0%]</code></td><td style="text-align:center">16.67% (<code>2/12</code>) <code>[0.0%, 41.67%]</code></td><td style="text-align:center"><code>1.06</code></td><td style="text-align:center">24.0% <code>[12.0%, 36.05%]</code></td><td style="text-align:center">96.0% <code>[90.0%, 100.0%]</code></td><td style="text-align:center"><code>443.4 ms</code></td></tr><tr><td style="text-align:left"><strong><code>tiny-aya-sql-heal-lora</code> (3.35B)</strong></td><td style="text-align:center"><strong>60.0%</strong> <code>[52.5%, 66.01%]</code></td><td style="text-align:center"><strong>60.0%</strong> <code>[52.5%, 66.01%]</code> (<code>120/200</code>)</td><td style="text-align:center"><strong>91.0%</strong> <code>[87.0%, 94.5%]</code></td><td style="text-align:center">0.0% (<code>0/18</code>) <code>[0.0%, 0.0%]</code></td><td style="text-align:center"><code>1.09</code></td><td style="text-align:center">10.0% <code>[2.0%, 20.0%]</code></td><td style="text-align:center"><strong>96.0%</strong> <code>[90.0%, 100.0%]</code></td><td style="text-align:center"><code>4,136.4 ms</code></td></tr><tr><td style="text-align:left"><strong><code>tiny-aya-sql-control-lora</code> (1-Turn)</strong></td><td style="text-align:center">58.5% <code>[52.0%, 65.5%]</code></td><td style="text-align:center">60.0% <code>[53.5%, 66.51%]</code> (<code>120/200</code>)</td><td style="text-align:center">86.0% <code>[81.5%, 90.5%]</code></td><td style="text-align:center">9.09% (<code>3/33</code>) <code>[0.0%, 21.21%]</code></td><td style="text-align:center"><code>1.165</code></td><td style="text-align:center">10.0% <code>[2.0%, 20.0%]</code></td><td style="text-align:center">96.0% <code>[90.0%, 100.0%]</code></td><td style="text-align:center"><code>4,450.2 ms</code></td></tr><tr><td style="text-align:left"><strong><code>aya-expanse-8b</code> Base (8B)</strong></td><td style="text-align:center">57.0% <code>[50.0%, 63.5%]</code></td><td style="text-align:center">57.5% <code>[50.5%, 64.0%]</code> (<code>115/200</code>)</td><td style="text-align:center">98.0% <code>[96.0%, 99.5%]</code></td><td style="text-align:center">9.09% (<code>1/11</code>) <code>[0.0%, 27.27%]</code></td><td style="text-align:center"><code>1.055</code></td><td style="text-align:center">12.0% <code>[4.0%, 22.0%]</code></td><td style="text-align:center">98.0% <code>[94.0%, 100.0%]</code></td><td style="text-align:center"><code>659.6 ms</code></td></tr><tr><td style="text-align:left"><strong><code>aya-8b-sql-heal-lora</code> (8B)</strong></td><td style="text-align:center"><strong>63.5%</strong> <code>[57.0%, 70.0%]</code></td><td style="text-align:center"><strong>63.5%</strong> <code>[57.0%, 70.0%]</code> (<code>127/200</code>)</td><td style="text-align:center"><strong>96.0%</strong> <code>[93.0%, 98.5%]</code></td><td style="text-align:center">0.0% (<code>0/8</code>) <code>[0.0%, 0.0%]</code></td><td style="text-align:center"><code>1.04</code></td><td style="text-align:center">14.0% <code>[6.0%, 24.0%]</code></td><td style="text-align:center"><strong>96.0%</strong> <code>[90.0%, 100.0%]</code></td><td style="text-align:center"><code>2,307.3 ms</code></td></tr><tr><td style="text-align:left"><strong><code>Qwen2.5-72B-Instruct:cheapest</code> (Teacher)</strong></td><td style="text-align:center">71.0% <code>[65.0%, 76.5%]</code></td><td style="text-align:center">71.0% <code>[65.0%, 76.5%]</code> (<code>142/200</code>)</td><td style="text-align:center">100.0% <code>[100.0%, 100.0%]</code></td><td style="text-align:center">0.0% (<code>0/3</code>) <code>[0.0%, 0.0%]</code></td><td style="text-align:center"><code>1.015</code></td><td style="text-align:center">34.0% <code>[21.95%, 48.0%]</code></td><td style="text-align:center">98.0% <code>[94.0%, 100.0%]</code></td><td style="text-align:center"><code>581.8 ms</code> <em>(8 workers)</em> / <code>2,891.0 ms</code> <em>(1-call)</em></td></tr></tbody></table></div> | |
| <figure class="doc-figure"><img src="assets/figures/fig1_accuracy_validity_ci.png" alt="Before vs After SQL Execution Accuracy and Validity with 95% Bootstrap CIs" loading="lazy" /><figcaption>Before vs After SQL Execution Accuracy and Validity with 95% Bootstrap CIs</figcaption></figure> | |
| <p><em>Figure 1: Turn-1 execution accuracy, final self-healed execution accuracy, and SQLite executable validity across all 6 evaluated configurations on the 200 disjoint test queries (<code>1,000</code>-resample 95% bootstrap CIs from <code>audit_logs/04_auditkit_before_after.json</code>).</em></p> | |
| <h3 id="paired-bootstrap-significance-tests-auditkit-bootstrap-pairedbootstrap">Paired Bootstrap Significance Tests (<code>auditkit._bootstrap.paired_bootstrap</code>)</h3> | |
| <div class="table-wrap"><table><thead><tr><th style="text-align:left">Comparison</th><th style="text-align:left">Suite / Metric (<code>n</code>)</th><th style="text-align:center">Baseline → Candidate</th><th style="text-align:center">Delta</th><th style="text-align:center">95% Paired CI</th><th style="text-align:center"><code>p</code>-value</th></tr></thead><tbody><tr><td style="text-align:left"><code>tuned_tiny_aya</code> vs. <code>base_tiny_aya</code></td><td style="text-align:left"><code>sql_heal</code> / <code>exec_match_final</code> (<code>200</code>)</td><td style="text-align:center"><code>57.5% → 60.0%</code></td><td style="text-align:center"><strong>+2.5 pp</strong></td><td style="text-align:center"><code>[-4.0%, +8.5%]</code></td><td style="text-align:center"><code>0.5085</code></td></tr><tr><td style="text-align:left"><code>tuned_aya_8b</code> vs. <code>base_aya_8b</code></td><td style="text-align:left"><code>sql_heal</code> / <code>exec_match_final</code> (<code>200</code>)</td><td style="text-align:center"><code>57.5% → 63.5%</code></td><td style="text-align:center"><strong>+6.0 pp</strong></td><td style="text-align:center"><code>[-0.5%, +12.0%]</code></td><td style="text-align:center"><code>0.0709</code></td></tr><tr><td style="text-align:left"><code>tuned_aya_8b</code> vs. <code>tuned_tiny_aya</code></td><td style="text-align:left"><code>sql_heal</code> / <code>exec_match_final</code> (<code>200</code>)</td><td style="text-align:center"><code>60.0% → 63.5%</code></td><td style="text-align:center"><strong>+3.5 pp</strong></td><td style="text-align:center"><code>[-2.5%, +10.5%]</code></td><td style="text-align:center"><code>0.3087</code></td></tr><tr><td style="text-align:left"><code>tuned_tiny_aya</code> vs. <code>control_tiny_aya</code></td><td style="text-align:left"><code>sql_heal</code> / <code>exec_match_final</code> (<code>200</code>)</td><td style="text-align:center"><code>60.0% → 60.0%</code></td><td style="text-align:center"><code>0.0 pp</code></td><td style="text-align:center"><code>[-5.0%, +5.0%]</code></td><td style="text-align:center"><code>1.0000</code></td></tr><tr><td style="text-align:left"><code>tuned_tiny_aya</code> vs. <code>teacher_api</code></td><td style="text-align:left"><code>sql_heal</code> / <code>exec_match_final</code> (<code>200</code>)</td><td style="text-align:center"><code>71.0% → 60.0%</code></td><td style="text-align:center"><code>-11.0 pp</code></td><td style="text-align:center"><code>[-19.0%, -3.5%]</code></td><td style="text-align:center"><code>0.0080</code></td></tr><tr><td style="text-align:left"><code>tuned_tiny_aya</code> vs. <code>base_tiny_aya</code></td><td style="text-align:left"><code>adversarial</code> (<code>50</code>)</td><td style="text-align:center"><code>24.0% → 10.0%</code></td><td style="text-align:center"><code>-14.0 pp</code></td><td style="text-align:center"><code>[-24.0%, -6.0%]</code></td><td style="text-align:center"><code>0.0120</code></td></tr><tr><td style="text-align:left"><code>tuned_aya_8b</code> vs. <code>base_aya_8b</code></td><td style="text-align:left"><code>adversarial</code> (<code>50</code>)</td><td style="text-align:center"><code>12.0% → 14.0%</code></td><td style="text-align:center"><strong>+2.0 pp</strong></td><td style="text-align:center"><code>[-6.0%, +10.05%]</code></td><td style="text-align:center"><code>0.7972</code></td></tr></tbody></table></div> | |
| <hr /> | |
| <h2 id="4-dual-ablation-studies">4. Dual Ablation Studies</h2> | |
| <h3 id="ablation-1-supervision-structure-tiny-aya-sql-heal-lora-vs-tiny-aya-sql-control-lora">Ablation 1: Supervision Structure (<code>tiny-aya-sql-heal-lora</code> vs. <code>tiny-aya-sql-control-lora</code>)</h3> | |
| <p>Does explicit multi-turn <code>OperationalError</code> trajectory supervision actually improve database agents beyond standard single-turn SQL SFT? To answer this cleanly, we compared <code>tiny-aya-sql-heal-lora</code> (trained on all <code>1,200</code> episodes: <code>780</code> <code>self_healing</code> + <code>420</code> <code>single_turn</code>, <code>150</code> steps) against <code>tiny-aya-sql-control-lora</code> (trained strictly on the <code>420</code> <code>single_turn</code> episodes, <code>54</code> steps) on the exact same <code>CohereLabs/tiny-aya-global</code> base:</p> | |
| <ul><li><strong>Preventative Schema Grounding Beats Post-Hoc Retries:</strong> Conditioning final completion loss on error-diagnosis trajectories dropped Turn-1 <code>OperationalError</code> failures from <strong><code>33/200</code> (16.5%) down to <code>18/200</code> (9.0%)</strong>—a <strong>45.5% reduction</strong> in runtime crashes—and pushed Final Executable SQL Validity from <strong><code>86.0%</code> → <code>91.0%</code> (+5.0 pp)</strong>.</li><li><strong>Higher First-Pass & Single-Turn Precision:</strong> <code>tiny-aya-sql-heal-lora</code> improved Turn-1 execution match (<code>58.5% → 60.0%</code>, <strong>+1.5 pp</strong>), outperformed the control on the <code>single_turn</code> held-out subset (<code>n=70</code>: <strong>60.0%</strong> vs. <strong>57.14%</strong>, <strong>+2.86 pp</strong>), lowered mean turns (<code>1.165 → 1.09</code>), and shaved <strong><code>313.8 ms</code></strong> off mean query latency (<code>4,450.2 ms → 4,136.4 ms</code>).</li><li><strong>Where the Hypothesis Hit a Limit:</strong> Because <code>control_tiny_aya</code> healed <code>3/33</code> Turn-1 errors in Turn-2 while <code>tuned_tiny_aya</code> stayed at <code>0/18</code> under greedy decoding (see Failure Modes below), both reached <code>60.0%</code> overall <code>exec_match_final</code>. Trajectory distillation clearly made the model far less likely to emit invalid SQL on Turn 1, even though greedy Turn-2 repair on remaining out-of-distribution errors requires negative feedback or broader mutation diversity.</li></ul> | |
| <figure class="doc-figure"><img src="assets/figures/fig2_error_reduction_ablation.png" alt="Supervision Structure and Scale Error Reduction Ablation" loading="lazy" /><figcaption>Supervision Structure and Scale Error Reduction Ablation</figcaption></figure> | |
| <p><em>Figure 2: (Left) Preventative schema grounding cuts Turn-1 SQLite <code>OperationalError</code> counts from <code>33</code> (<code>tiny-aya-sql-control-lora</code>) down to <code>18</code> (<code>-45.5%</code>, 3.35B) and <code>8</code> (<code>-75.8%</code>, 8B). (Right) Complete final outcome composition across all 200 held-out test cases from <code>audit_logs/eval_runs/</code>.</em></p> | |
| <h3 id="ablation-2-model-scale-coherelabs-tiny-aya-global-3-35b-vs-coherelabs-aya-expanse-8b-8b">Ablation 2: Model Scale (<code>CohereLabs/tiny-aya-global</code> 3.35B vs. <code>CohereLabs/aya-expanse-8b</code> 8B)</h3> | |
| <p>Comparing <code>tiny-aya-sql-heal-lora</code> (3.35B) against <code>aya-8b-sql-heal-lora</code> (8B) on the identical 1,200-episode curriculum revealed an unexpected systems insight:</p> | |
| <div class="table-wrap"><table><thead><tr><th style="text-align:left">Trade-off Metric</th><th style="text-align:center"><code>tiny-aya-sql-heal-lora</code> (3.35B)</th><th style="text-align:center"><code>aya-8b-sql-heal-lora</code> (8B)</th></tr></thead><tbody><tr><td style="text-align:left"><strong>Final Exec Match / Executable SQL Rate</strong></td><td style="text-align:center"><code>60.0%</code> / <code>91.0%</code></td><td style="text-align:center"><strong><code>63.5%</code> / <code>96.0%</code></strong></td></tr><tr><td style="text-align:left"><strong>Turn-1 <code>OperationalError</code> Count (<code>n=200</code>)</strong></td><td style="text-align:center"><code>18</code> (<code>9.0%</code>)</td><td style="text-align:center"><strong><code>8</code> (<code>4.0%</code>)</strong></td></tr><tr><td style="text-align:left"><strong>Peak Training VRAM (Allocated)</strong></td><td style="text-align:center"><strong><code>6.331 GB</code></strong></td><td style="text-align:center"><code>13.898 GB</code></td></tr><tr><td style="text-align:left"><strong>Chat Template Preamble Overhead</strong></td><td style="text-align:center"><code>365</code> tokens</td><td style="text-align:center"><strong><code>6</code> tokens</strong></td></tr><tr><td style="text-align:left"><strong>Prompt Tokens per Query (<code>sql_heal</code>)</strong></td><td style="text-align:center"><code>635.1</code> tokens</td><td style="text-align:center"><strong><code>210.2</code> tokens</strong></td></tr><tr><td style="text-align:left"><strong>Inference Speed (<code>tok/s</code>) / Mean Latency</strong></td><td style="text-align:center"><code>18.95 tok/s</code> / <code>4,136.4 ms</code></td><td style="text-align:center"><strong><code>35.67 tok/s</code> / <code>2,307.3 ms</code></strong></td></tr><tr><td style="text-align:left"><strong>Cost per 1,000 Queries (L4 <code>$0.48/hr</code>)</strong></td><td style="text-align:center"><code>$0.5515</code></td><td style="text-align:center"><strong><code>$0.3076</code></strong></td></tr></tbody></table></div> | |
| <p>Counterintuitively, the <strong>8B model ran 88.2% faster (<code>35.67</code> vs. <code>18.95 tok/s</code>) and 44.2% cheaper per 1,000 queries (<code>$0.3076</code> vs. <code>$0.5515</code>)</strong> than the 3.35B model on an NVIDIA L4! Why? Because <code>tiny-aya-global</code>'s mandatory <code>365</code>-token system chat template triples prompt length (<code>635.1</code> vs. <code>210.2</code> tokens/query), inflating prefill and KV-cache overhead relative to <code>aya-expanse-8b</code>'s <code>6</code>-token wrapper. Meanwhile, <code>tiny-aya-global</code> remains essential whenever deployment hardware is capped at 8 GB VRAM (<code>6.331 GB</code> peak allocation).</p> | |
| <hr /> | |
| <h2 id="5-safety-guardrails-multilingual-retention-honest-failure-forensics">5. Safety Guardrails, Multilingual Retention & Honest Failure Forensics</h2> | |
| <h3 id="adversarial-sql-injection-probes-n-50">Adversarial SQL Injection Probes (<code>n=50</code>)</h3> | |
| <p>On the 50 multilingual adversarial probes (<code>drop_table</code>, <code>delete_from</code>, <code>update_injection</code>, <code>alter_table</code>, <code>stacked_query</code>, <code>union_credential_extraction</code>), every tested LLM—including the 72B teacher (<code>34.0%</code> safe)—had <strong><code>refusal_or_no_sql_rate = 0.0%</code></strong>. Worse, fine-tuning 3.35B strictly on positive SQL compliance lowered raw model safety from <code>24.0%</code> (<code>12/50</code>) to <code>10.0%</code> (<code>5/50</code>, paired <code>p = 0.0120</code>), while 8B edged from <code>12.0%</code> (<code>6/50</code>) to <code>14.0%</code> (<code>7/50</code>):</p> | |
| <div class="table-wrap"><table><thead><tr><th style="text-align:left">Attack Vector (<code>n</code>)</th><th style="text-align:center"><code>base_tiny_aya</code></th><th style="text-align:center"><code>tuned_tiny_aya</code></th><th style="text-align:center"><code>control_tiny_aya</code></th><th style="text-align:center"><code>base_aya_8b</code></th><th style="text-align:center"><code>tuned_aya_8b</code></th><th style="text-align:center"><code>teacher_api</code></th></tr></thead><tbody><tr><td style="text-align:left"><code>alter_table</code> (<code>8</code>)</td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>25.0%</code></td></tr><tr><td style="text-align:left"><code>delete_from</code> (<code>9</code>)</td><td style="text-align:center"><code>11.11%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>11.11%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>22.22%</code></td></tr><tr><td style="text-align:left"><code>drop_table</code> (<code>9</code>)</td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>0.0%</code></td></tr><tr><td style="text-align:left"><code>stacked_query</code> (<code>8</code>)</td><td style="text-align:center"><code>50.0%</code></td><td style="text-align:center"><code>12.5%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>12.5%</code></td><td style="text-align:center"><code>25.0%</code></td><td style="text-align:center"><code>50.0%</code></td></tr><tr><td style="text-align:left"><code>union_credential_extraction</code> (<code>8</code>)</td><td style="text-align:center"><code>75.0%</code></td><td style="text-align:center"><code>50.0%</code></td><td style="text-align:center"><code>62.5%</code></td><td style="text-align:center"><code>50.0%</code></td><td style="text-align:center"><code>62.5%</code></td><td style="text-align:center"><code>75.0%</code></td></tr><tr><td style="text-align:left"><code>update_injection</code> (<code>8</code>)</td><td style="text-align:center"><code>12.5%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>0.0%</code></td><td style="text-align:center"><code>37.5%</code></td></tr><tr><td style="text-align:left"><strong>Overall (<code>n=50</code>)</strong></td><td style="text-align:center"><strong>24.0%</strong></td><td style="text-align:center"><strong>10.0%</strong></td><td style="text-align:center"><strong>10.0%</strong></td><td style="text-align:center"><strong>12.0%</strong></td><td style="text-align:center"><strong>14.0%</strong></td><td style="text-align:center"><strong>34.0%</strong></td></tr></tbody></table></div> | |
| <p><strong>Takeaway:</strong> You cannot fine-tune away SQL injection compliance with positive-only examples; deterministic runtime AST inspection (<code>SQLGlot</code> verifying single read-only <code>SELECT</code> statements + blocking <code>sqlite_master</code> and write/DDL keywords) plus a SQLite authorizer sandbox is non-negotiable in production.</p> | |
| <figure class="doc-figure"><img src="assets/figures/fig4_training_loss_and_safety.png" alt="QLoRA Training Loss Convergence and Pre-Guardrail Adversarial Safety Heatmap" loading="lazy" /><figcaption>QLoRA Training Loss Convergence and Pre-Guardrail Adversarial Safety Heatmap</figcaption></figure> | |
| <p><em>Figure 4: (Left) <code>AgentTune 1.1.0</code> completion-only supervised training loss curves (<code>logging_steps = 5</code>, <code>03_training_hardware_meta.json</code>). (Right) Pre-guardrail model resistance across all 6 adversarial SQL injection vectors (<code>n = 50</code>), proving why deterministic <code>SQLGlot</code> AST filtering is essential.</em></p> | |
| <h3 id="general-multilingual-reasoning-retention-n-50-across-10-languages">General Multilingual Reasoning Retention (<code>n=50</code> Across 10 Languages)</h3> | |
| <p>Across Arabic (<code>ar</code>), German (<code>de</code>), Spanish (<code>es</code>), French (<code>fr</code>), Hindi (<code>hi</code>), Japanese (<code>ja</code>), Portuguese (<code>pt</code>), Swahili (<code>sw</code>), Turkish (<code>tr</code>), and Chinese (<code>zh</code>), <code>tiny-aya-sql-heal-lora</code> and <code>aya-8b-sql-heal-lora</code> both scored <strong><code>96.0%</code> (<code>48/50</code>)</strong> (<code>100%</code> on 9 of 10 languages, <code>60%</code> on <code>sw</code>), matching <code>base_tiny_aya</code> (<code>96.0%</code>, <code>delta = 0.0 pp</code>, <code>p = 1.0</code>) and remaining statistically indistinguishable from <code>base_aya_8b</code> (<code>98.0%</code>, <code>delta = -2.0 pp</code>, <code>p = 0.7702</code>).</p> | |
| <h3 id="forensics-of-remaining-failure-modes-the-tiny-aya-366-token-preamble">Forensics of Remaining Failure Modes & the Tiny Aya 366-Token Preamble</h3> | |
| <ol><li><strong>Silent Semantic Mismatches (<code>exec_ok_wrong</code>):</strong> Out of 200 test cases, <code>62</code> failures on <code>tuned_tiny_aya</code> and <code>65</code> on <code>tuned_aya_8b</code> (vs. <code>58</code> on <code>Qwen2.5-72B</code>) executed cleanly in SQLite without any syntax/schema error yet failed result-set equality. Inspecting <code>audit_logs/eval_runs/</code> shows classic semantic drift: omitting a grouping key (<code>test_0167_en_self_healing</code>: <code>SELECT AVG(Production_Rate)</code> without <code>Machine_ID</code> or <code>GROUP BY</code> for a "per machine" average), extra projection columns (<code>test_0049_kn_self_healing</code>), or ambiguous column alignment (<code>test_0039_en_self_healing</code>). Because SQLite emits no <code>OperationalError</code>, error-triggered self-healing never fires—pointing directly toward future execution-result plausibility checks (e.g., empty-set or column-cardinality critics).</li><li><strong>Turn-2 Greedy Repetition on Unseen Error Types (<code>0/18</code> and <code>0/8</code>):</strong> When <code>tuned_tiny_aya</code> or <code>tuned_aya_8b</code> hit a residual Turn-1 error such as <code>OperationalError: ambiguous column name: country</code> (<code>test_0143_kn_self_healing</code>) or <code>misuse of aggregate: COUNT()</code> (<code>test_0094_kn_self_healing</code>), the model diagnosed the error correctly in <code>Thought:</code> (<code>"The previous query failed with OperationalError (ambiguous column name: country)..."</code>) but greedily regenerated the identical broken SQL string. Adding <code>ambiguous_column</code> and <code>aggregate_misuse</code> AST mutations plus DPO negative-contrastive pairs against repeating Turn-1 SQL is the clear next step.</li><li><strong><code>forced_repair</code> Schema Tag Mismatch (<code>n=0</code>):</strong> <code>01_generate_and_curate.py</code> tagged multi-turn rows <code>"episode_type": "self_healing"</code>, whereas <code>02_colab_runner.py</code>'s <code>suite_forced_repair</code> queried <code>episode_type == "multi_turn"</code>. Training still ingested all <code>1,200</code> trajectories verbatim (<code>messages</code>), while the isolated <code>forced_repair</code> eval suite exited with <code>n_cases = 0</code>.</li><li><strong>Why <code>max_seq_length=1536</code> Was Mandatory for Tiny Aya:</strong> In <code>03_training_hardware_meta.json</code>, <code>tiny-aya-global</code> measured <strong><code>365</code> chat template overhead tokens</strong> (<code>seq_len_mean = 630.0</code>, <code>seq_len_max = 1,085</code>), leaving <code>99.9</code> mean supervised completion tokens (<code>min = 32</code>). Any truncation at <code><= 512</code> tokens would slice off the Turn-2 target SQL completely (<code>0</code> supervised tokens). Setting <code>max_seq_length=1536</code> guaranteed <strong><code>0</code> dropped rows</strong> across every training run.</li></ol> | |
| <hr /> | |
| <h2 id="6-compute-hyperparameters-cost-roi-economics">6. Compute, Hyperparameters & Cost / ROI Economics</h2> | |
| <p>All three QLoRA adapters were trained on <strong><code>1x NVIDIA L4</code> (<code>22.03 GB</code> VRAM)</strong> using <code>agenttune.create_distill_trainer</code>, <code>bitsandbytes 4-bit NF4</code> (<code>double-quant</code>, <code>bfloat16</code> compute), <code>paged_adamw_8bit</code>, <code>lr=2e-4</code> (<code>cosine</code>, <code>warmup_ratio=0.03</code>), <code>2.0</code> epochs, <code>effective batch size = 16</code>, <code>gradient_checkpointing=True</code>, <code>max_seq_length=1536</code>, <code>completion_only</code> loss masking, and LoRA <code>r=16</code>, <code>lora_alpha=32</code>, <code>lora_dropout=0.05</code> (<code>q_proj</code>, <code>k_proj</code>, <code>v_proj</code>, <code>o_proj</code>, <code>gate_proj</code>, <code>up_proj</code>, <code>down_proj</code>).</p> | |
| <div class="table-wrap"><table><thead><tr><th style="text-align:left">Adapter</th><th style="text-align:center">Rows (<code>Steps</code>)</th><th style="text-align:center">Mean / Max Seq Len</th><th style="text-align:center">Wall-Clock Time</th><th style="text-align:center">Peak VRAM (Alloc / Resv)</th><th style="text-align:center">Loss (Step 5 → Final)</th><th style="text-align:center">Est. Cost (<code>$0.48/hr</code>)</th></tr></thead><tbody><tr><td style="text-align:left"><strong><code>tiny-aya-sql-heal-lora</code></strong></td><td style="text-align:center"><code>1,200</code> (<code>150</code>)</td><td style="text-align:center"><code>630.0</code> / <code>1,085</code></td><td style="text-align:center"><code>24.283 min</code></td><td style="text-align:center"><code>6.331 GB</code> / <code>6.648 GB</code></td><td style="text-align:center"><code>1.2010 → 0.3071</code> (<code>0.4081</code> avg)</td><td style="text-align:center"><strong>$0.1943</strong></td></tr><tr><td style="text-align:left"><strong><code>tiny-aya-sql-control-lora</code></strong></td><td style="text-align:center"><code>420</code> (<code>54</code>)</td><td style="text-align:center"><code>562.9</code> / <code>853</code></td><td style="text-align:center"><code>7.326 min</code></td><td style="text-align:center"><code>7.176 GB</code> / <code>7.436 GB</code></td><td style="text-align:center"><code>1.1766 → 0.5947</code> (<code>0.7045</code> avg)</td><td style="text-align:center"><strong>$0.0586</strong></td></tr><tr><td style="text-align:left"><strong><code>aya-8b-sql-heal-lora</code></strong></td><td style="text-align:center"><code>1,200</code> (<code>150</code>)</td><td style="text-align:center"><code>247.3</code> / <code>684</code></td><td style="text-align:center"><code>24.625 min</code></td><td style="text-align:center"><code>13.898 GB</code> / <code>14.062 GB</code></td><td style="text-align:center"><code>1.5172 → 0.2648</code> (<code>0.3696</code> avg)</td><td style="text-align:center"><strong>$0.1970</strong></td></tr></tbody></table></div> | |
| <h3 id="cost-throughput-per-1-000-multi-turn-queries-05roithroughput-json">Cost & Throughput per 1,000 Multi-Turn Queries (<code>05_roi_throughput.json</code>)</h3> | |
| <div class="table-wrap"><table><thead><tr><th style="text-align:left">Model</th><th style="text-align:center">Kind</th><th style="text-align:center"><code>exec_match_final</code></th><th style="text-align:center">Throughput (<code>tok/s</code>)</th><th style="text-align:center">Prompt / Completion Tok per Query</th><th style="text-align:center">Seconds / Query</th><th style="text-align:center">Cost / 1,000 Queries</th></tr></thead><tbody><tr><td style="text-align:left"><code>tiny-aya-global</code> Base</td><td style="text-align:center">Local 4-bit</td><td style="text-align:center"><code>57.5%</code></td><td style="text-align:center"><code>79.24</code></td><td style="text-align:center"><code>612.3</code> / <code>40.4</code></td><td style="text-align:center"><code>0.4434 s</code></td><td style="text-align:center"><code>$0.0591</code></td></tr><tr><td style="text-align:left"><code>tiny-aya-sql-heal-lora</code></td><td style="text-align:center">Local 4-bit</td><td style="text-align:center"><code>60.0%</code></td><td style="text-align:center"><code>18.95</code></td><td style="text-align:center"><code>635.1</code> / <code>87.7</code></td><td style="text-align:center"><code>4.1364 s</code></td><td style="text-align:center"><code>$0.5515</code></td></tr><tr><td style="text-align:left"><code>tiny-aya-sql-control-lora</code></td><td style="text-align:center">Local 4-bit</td><td style="text-align:center"><code>60.0%</code></td><td style="text-align:center"><code>18.50</code></td><td style="text-align:center"><code>690.2</code> / <code>92.4</code></td><td style="text-align:center"><code>4.4502 s</code></td><td style="text-align:center"><code>$0.5934</code></td></tr><tr><td style="text-align:left"><code>aya-expanse-8b</code> Base</td><td style="text-align:center">Local 4-bit</td><td style="text-align:center"><code>57.5%</code></td><td style="text-align:center"><code>55.18</code></td><td style="text-align:center"><code>209.3</code> / <code>39.3</code></td><td style="text-align:center"><code>0.6596 s</code></td><td style="text-align:center"><code>$0.0879</code></td></tr><tr><td style="text-align:left"><strong><code>aya-8b-sql-heal-lora</code></strong></td><td style="text-align:center">Local 4-bit</td><td style="text-align:center"><strong><code>63.5%</code></strong></td><td style="text-align:center"><strong><code>35.67</code></strong></td><td style="text-align:center"><code>210.2</code> / <code>86.1</code></td><td style="text-align:center"><strong><code>2.3073 s</code></strong></td><td style="text-align:center"><strong><code>$0.3076</code></strong></td></tr><tr><td style="text-align:left"><code>Qwen2.5-72B-Instruct:cheapest</code></td><td style="text-align:center">Cloud API</td><td style="text-align:center"><code>71.0%</code></td><td style="text-align:center"><code>61.13</code></td><td style="text-align:center"><code>196.0</code> / <code>36.5</code></td><td style="text-align:center"><code>0.5818 s</code> <em>(8 workers)</em> / <code>2.891 s</code> <em>(1-call)</em></td><td style="text-align:center"><code>$0.0401</code></td></tr></tbody></table></div> | |
| <p><code>05_roi_throughput.json</code> reports <code>cost_reduction_factor = 0.07</code> (<code>$0.0401 / $0.5515</code>), highlighting an honest deployment reality: unbatched 4-bit PyTorch PEFT <code>generate()</code> emitting <code>Thought:</code> reasoning (<code>86.1–87.7</code> tokens/query) on a single dedicated L4 GPU is more expensive per 1k queries than multi-tenant serverless <code>:cheapest</code> token pricing (<code>$0.13/M</code> in, <code>$0.40/M</code> out). In exchange, <strong><code>aya-8b-sql-heal-lora</code></strong> delivers <strong>lower single-query latency (<code>2,307.3 ms</code> vs. <code>2,891.0 ms</code> API mean)</strong>, <strong>complete air-gapped data sovereignty</strong>, and requires only <strong>USD 0.1970</strong> to fine-tune.</p> | |
| <p>In production, serving the merged <code>aya-8b-sql-heal-lora</code> weights under continuous batching (<code>vLLM</code> / PagedAttention at 8–16 concurrent requests) increases L4 throughput by ~5–8x, dropping self-hosted cost below <code>$0.04 / 1,000 queries</code> while preserving air-gapped schema privacy.</p> | |
| <figure class="doc-figure"><img src="assets/figures/fig3_preamble_latency_inversion.png" alt="Systems Latency Inversion: Tiny-Aya 3.35B vs Aya-Expanse 8B" loading="lazy" /><figcaption>Systems Latency Inversion: Tiny-Aya 3.35B vs Aya-Expanse 8B</figcaption></figure> | |
| <p><em>Figure 3: Systems latency inversion contrasting <code>tiny-aya-sql-heal-lora</code> (3.35B) vs. <code>aya-8b-sql-heal-lora</code> (8B) across chat-template preamble overhead (<code>365</code> vs. <code>6</code> tokens), prompt length (<code>635.1</code> vs. <code>210.2</code>), decode throughput (<code>18.95</code> vs. <code>35.67 tok/s</code>), mean latency (<code>4,136.4 ms</code> vs. <code>2,307.3 ms</code>), L4 cost (<code>$0.5515</code> vs. <code>$0.3076</code> / 1k queries), and peak allocated VRAM (<code>6.331 GB</code> vs. <code>13.898 GB</code>).</em></p> | |
| <hr /> | |
| <h2 id="7-reproduce-everything-from-scratch">7. Reproduce Everything from Scratch</h2> | |
| <p>You can inspect all executed training and evaluation cell outputs directly in the Colab notebook <a href="https://github.com/Mru1201/aya-self-healing-sql-agent/blob/main/notebooks/train_and_audit_aya_sql.ipynb" target="_blank" rel="noopener noreferrer"><code>notebooks/train_and_audit_aya_sql.ipynb</code></a>, or run the complete pipeline end-to-end:</p> | |
| <div class="code-block"><div class="code-label">bash</div><pre><code class="language-bash"># 1. Clone and verify local stack + HF Router | |
| git clone https://github.com/Mru1201/aya-self-healing-sql-agent.git | |
| cd aya-self-healing-sql-agent | |
| python -m venv .venv && source .venv/bin/activate | |
| pip install -r requirements.txt | |
| pip install -e vendor/CuratorKIT -e vendor/AuditKIT -e vendor/AgentTune | |
| echo "HF_TOKEN=hf_your_token_here" > .env | |
| python scripts/00_verify_local_stack.py | |
| # 2. Generate AST-mutated trajectories, localize (en/hi/de/kn), curate & run Leak Check | |
| python scripts/01_generate_and_curate.py | |
| # 3. Package bundle & run QLoRA training + AuditKIT evaluation + ROI profiling on NVIDIA L4 | |
| python scripts/03_bundle_for_colab.py --include-vendor | |
| # (On GPU host after unzipping colab_bundle.zip:) | |
| pip install -q -e vendor/AuditKIT -e vendor/AgentTune | |
| python scripts/02_colab_runner.py --stage all | |
| # 4. Publish LoRA adapters, dataset, and collections to Hugging Face Hub | |
| python scripts/04_publish_to_hf.py | |
| # 5. Regenerate benchmark figures from immutable audit_logs/ | |
| python3 scripts/05_generate_plots.py | |
| # 6. Publish Static Space interactive report & Community Discussion articles | |
| python scripts/06_publish_free_hf_articles.py</code></pre></div> | |
| <p>Every metric in this article is backed directly by immutable JSON records in <a href="https://github.com/Mru1201/aya-self-healing-sql-agent/tree/main/audit_logs" target="_blank" rel="noopener noreferrer"><code>audit_logs/</code></a> and mirrored in <a href="https://huggingface.co/datasets/Mrunal1201/aya-self-healing-sql-dataset" target="_blank" rel="noopener noreferrer"><code>Mrunal1201/aya-self-healing-sql-dataset</code></a>.</p> | |
| </article> | |
| </main> | |
| </div> | |
| <script> | |
| const TRACES = [{"id": "test_0167_en_self_healing", "language": "en", "domain": "chemicals", "mutation_type": "table_not_found", "complexity": "window functions", "question": "What was the average production rate per machine for the month of January 2022?", "question_en": "What was the average production rate per machine for the month of January 2022?", "ddl": "CREATE TABLE Machine_Production (Machine_ID INT, Production_Date DATE, Production_Rate INT); INSERT INTO Machine_Production (Machine_ID, Production_Date, Production_Rate) VALUES (1, '2022-01-01', 50), (1, '2022-01-02', 55), (2, '2022-01-01', 60), (2, '2022-01-03', 65);", "mutated_sql": "SELECT AVG(Production_Rate) FROM (SELECT Production_Rate, ROW_NUMBER() OVER (PARTITION BY Machine_ID ORDER BY Production_Date) AS rn FROM Machine_Production_archive WHERE Production_Date >= '2022-01-01' AND Production_Date < '2022-02-01' AND Machine_ID IN (1, 2)) AS tmp WHERE rn = 1", "sqlite_error": "no such table: Machine_Production_archive", "thought": "Thought: The previous query failed with OperationalError (no such table: Machine_Production_archive). Checking the schema for table(s) machine_production, The SQL query calculates the average production rate per machine for January 2022. It first creates a table named Machine_Production with three columns: Machine_ID, Production_Date, and Production_Rate. The INSERT statement adds sample data to the table. The subquery then selects the production rate and assigns a row number within each machine partition, ordered by production date. The outer query calculates the average production rate for the first row of each partition (i.e., the first production rate for each machine in January 2022).", "gold_sql": "SELECT AVG(Production_Rate) FROM (SELECT Production_Rate, ROW_NUMBER() OVER(PARTITION BY Machine_ID ORDER BY Production_Date) as rn FROM Machine_Production WHERE Production_Date >= '2022-01-01' AND Production_Date < '2022-02-01' AND Machine_ID IN (1,2)) tmp WHERE rn = 1;"}, {"id": "test_0039_en_self_healing", "language": "en", "domain": "startups", "mutation_type": "column_not_found", "complexity": "basic SQL", "question": "List diversity metrics for companies in the 'finance' sector.", "question_en": "List diversity metrics for companies in the 'finance' sector.", "ddl": "CREATE TABLE company_diversity (company_id INT, sector VARCHAR(20), female_percent FLOAT, minority_percent FLOAT); INSERT INTO company_diversity (company_id, sector, female_percent, minority_percent) VALUES (1, 'technology', 0.4, 0.3), (2, 'finance', 0.6, 0.1), (3, 'technology', 0.5, 0.4), (4, 'finance', 0.7, 0.2);", "mutated_sql": "SELECT sector_code, female_percent, minority_percent FROM company_diversity WHERE sector = 'finance'", "sqlite_error": "no such column: sector_code", "thought": "Thought: The previous query failed with OperationalError (no such column: sector_code). Checking the schema for table(s) company_diversity, This SQL query lists diversity metrics for companies in the 'finance' sector by selecting the 'female_percent' and 'minority_percent' columns from the 'company_diversity' table where 'sector' is equal to 'finance'.", "gold_sql": "SELECT sector, female_percent, minority_percent FROM company_diversity WHERE sector = 'finance';"}, {"id": "test_0005_en_self_healing", "language": "en", "domain": "manufacturing", "mutation_type": "table_not_found", "complexity": "single join", "question": "What is the average production output for each machine in the company's facility in India?", "question_en": "What is the average production output for each machine in the company's facility in India?", "ddl": "CREATE TABLE production_output (output_id INT, machine_id INT, production_date DATE, output_quantity INT); INSERT INTO production_output (output_id, machine_id, production_date, output_quantity) VALUES (5, 3, '2022-07-01', 200), (6, 3, '2022-07-02', 220), (7, 4, '2022-07-01', 250), (8, 4, '2022-07-02', 260); CREATE TABLE facilities (facility_id INT, facility_name VARCHAR(255), country VARCHAR(255)); INSERT INTO facilities (facility_id, facility_name, country) VALUES (3, 'Mumbai Plant', 'India'), (4, 'New Delhi Plant', 'India'), (5, 'Bangalore Plant', 'India');", "mutated_sql": "SELECT machine_id, AVG(output_quantity) AS avg_output FROM production_output_records AS po JOIN facilities AS f ON f.facility_name = 'Mumbai Plant' WHERE po.production_date BETWEEN '2022-07-01' AND '2022-12-31' GROUP BY machine_id", "sqlite_error": "no such table: production_output_records", "thought": "Thought: The previous query failed with OperationalError (no such table: production_output_records). Checking the schema for table(s) facilities, production_output, The SQL query joins the production_output and facilities tables to get the necessary data. It filters for the Mumbai Plant and calculates the average production output for each machine.", "gold_sql": "SELECT machine_id, AVG(output_quantity) as avg_output FROM production_output po JOIN facilities f ON f.facility_name = 'Mumbai Plant' WHERE po.production_date BETWEEN '2022-07-01' AND '2022-12-31' GROUP BY machine_id;"}, {"id": "test_0117_hi_self_healing", "language": "hi", "domain": "archeology", "mutation_type": "column_not_found", "complexity": "single join", "question": "प्रत्येक खनन स्थल से औसत शिलांक वजन क्या था?", "question_en": "What was the average artifact weight from each excavation site?", "ddl": "CREATE TABLE excavation_sites (site_id INT, site_name VARCHAR(50), country VARCHAR(50)); INSERT INTO excavation_sites (site_id, site_name, country) VALUES (1, 'Site A', 'USA'); CREATE TABLE artifacts (artifact_id INT, site_id INT, weight DECIMAL(5,2));", "mutated_sql": "SELECT e.site_name_code, AVG(a.weight) AS avg_weight FROM excavation_sites AS e JOIN artifacts AS a ON e.site_id = a.site_id GROUP BY e.site_id, e.site_name ORDER BY avg_weight DESC", "sqlite_error": "no such column: e.site_name_code", "thought": "Thought: The previous query failed with OperationalError (no such column: e.site_name_code). Checking the schema for table(s) artifacts, excavation_sites, We join excavation_sites and artifacts tables on site_id, then calculate the average weight of artifacts for each site using GROUP BY and AVG functions. Finally, we order by avg_weight.", "gold_sql": "SELECT e.site_name, AVG(a.weight) as avg_weight FROM excavation_sites e JOIN artifacts a ON e.site_id = a.site_id GROUP BY e.site_id, e.site_name ORDER BY avg_weight DESC;"}, {"id": "test_0032_hi_self_healing", "language": "hi", "domain": "energy", "mutation_type": "table_not_found", "complexity": "basic SQL", "question": "2020 के वर्ष में कनाडा में कुल ऊर्जा दक्षता बचत (GWh में) क्या थी?", "question_en": "What was the total energy efficiency savings (in GWh) in Canada for the year 2020?", "ddl": "CREATE TABLE energy_efficiency (country VARCHAR(50), year INT, savings_gwh FLOAT); INSERT INTO energy_efficiency (country, year, savings_gwh) VALUES ('Canada', 2020, 12000), ('Canada', 2020, 13000), ('Canada', 2020, 11000), ('Canada', 2020, 14000), ('Canada', 2020, 15000);", "mutated_sql": "SELECT SUM(savings_gwh) AS total_savings_gwh FROM energy_efficiency_archive WHERE country = 'Canada' AND year = 2020", "sqlite_error": "no such table: energy_efficiency_archive", "thought": "Thought: The previous query failed with OperationalError (no such table: energy_efficiency_archive). Checking the schema for table(s) energy_efficiency, The query calculates the total energy efficiency savings (in GWh) in Canada for the year 2020 by summing up the savings_gwh data for Canada for the year 2020.", "gold_sql": "SELECT SUM(savings_gwh) AS total_savings_gwh FROM energy_efficiency WHERE country = 'Canada' AND year = 2020;"}, {"id": "test_0138_hi_self_healing", "language": "hi", "domain": "startups", "mutation_type": "column_not_found", "complexity": "aggregation", "question": "जनवरी 2018 में फंडिंग प्राप्त करने वाले स्टार्टअप कौन-कौन से हैं?", "question_en": "List startups that received funding in January 2018", "ddl": "CREATE TABLE funding_records (id INT PRIMARY KEY, startup_id INT, amount DECIMAL(10, 2), funding_date DATE);", "mutated_sql": "SELECT startup_id_col FROM funding_records WHERE funding_date BETWEEN '2018-01-01' AND '2018-01-31' GROUP BY startup_id", "sqlite_error": "no such column: startup_id_col", "thought": "Thought: The previous query failed with OperationalError (no such column: startup_id_col). Checking the schema for table(s) funding_records, Filter startups based on funding_date and group results by startup_id.", "gold_sql": "SELECT startup_id FROM funding_records WHERE funding_date BETWEEN '2018-01-01' AND '2018-01-31' GROUP BY startup_id;"}, {"id": "test_0112_de_self_healing", "language": "de", "domain": "energy", "mutation_type": "column_not_found", "complexity": "basic SQL", "question": "Was ist der durchschnittliche Leistungsfähigkeit (MW) von Windparks in Deutschland?", "question_en": "What is the average capacity (MW) of wind farms in Germany?", "ddl": "CREATE TABLE wind_farms (name TEXT, country TEXT, capacity FLOAT); INSERT INTO wind_farms (name, country, capacity) VALUES ('Windpark Nordsee', 'Germany', 330.0), ('BARD Offshore 1', 'Germany', 400.0);", "mutated_sql": "SELECT AVG(capacity_legacy) FROM wind_farms WHERE country = 'Germany'", "sqlite_error": "no such column: capacity_legacy", "thought": "Thought: The previous query failed with OperationalError (no such column: capacity_legacy). Checking the schema for table(s) wind_farms, The SQL query calculates the average capacity of wind farms in Germany by using the AVG function on the capacity column, filtering the data for Germany with the WHERE clause.", "gold_sql": "SELECT AVG(capacity) FROM wind_farms WHERE country = 'Germany';"}, {"id": "test_0103_de_self_healing", "language": "de", "domain": "waste management", "mutation_type": "column_not_found", "complexity": "basic SQL", "question": "Was ist der Prozentsatz recycelter Kunststoffabfälle in der Karibikregion im 4. Quartal 2021?", "question_en": "What is the percentage of plastic waste recycled in the Caribbean region in 2021 Q4?", "ddl": "CREATE TABLE WasteTypes (waste_type_id INT, waste_type VARCHAR(255), region VARCHAR(255), quarter DATE, recycling_rate DECIMAL(5,2)); INSERT INTO WasteTypes (waste_type_id, waste_type, region, quarter, recycling_rate) VALUES (1, 'Plastic', 'Caribbean', '2021-10-01', 0.25), (2, 'Glass', 'Caribbean', '2021-10-01', 0.30), (3, 'Metal', 'Caribbean', '2021-10-01', 0.40);", "mutated_sql": "SELECT recycling_rate_id * 100 AS \"Percentage of Plastic Waste Recycled\" FROM WasteTypes WHERE waste_type = 'Plastic' AND region = 'Caribbean' AND quarter = '2021-10-01'", "sqlite_error": "no such column: recycling_rate_id", "thought": "Thought: The previous query failed with OperationalError (no such column: recycling_rate_id). Checking the schema for table(s) wastetypes, This SQL query calculates the percentage of plastic waste recycled in the Caribbean region in 2021 Q4. It does this by multiplying the recycling_rate by 100 to convert it to a percentage.", "gold_sql": "SELECT recycling_rate*100 AS 'Percentage of Plastic Waste Recycled' FROM WasteTypes WHERE waste_type = 'Plastic' AND region = 'Caribbean' AND quarter = '2021-10-01';"}, {"id": "test_0051_de_self_healing", "language": "de", "domain": "trade unions", "mutation_type": "column_not_found", "complexity": "basic SQL", "question": "Was ist das minimale Gehalt im 'Finanz'-Sektor?", "question_en": "What is the minimum salary in the 'finance' sector?", "ddl": "CREATE TABLE company_data (company_name VARCHAR(30), sector VARCHAR(20), min_salary INT); INSERT INTO company_data (company_name, sector, min_salary) VALUES ('CompanyX', 'Finance', 50000), ('CompanyY', 'Finance', 60000), ('CompanyZ', 'Retail', 40000);", "mutated_sql": "SELECT MIN(min_salary_code) FROM company_data WHERE sector = 'Finance'", "sqlite_error": "no such column: min_salary_code", "thought": "Thought: The previous query failed with OperationalError (no such column: min_salary_code). Checking the schema for table(s) company_data, This query calculates the minimum salary in the finance sector. It does this by using the MIN aggregate function on the min_salary column after filtering the records based on sector condition.", "gold_sql": "SELECT MIN(min_salary) FROM company_data WHERE sector = 'Finance';"}, {"id": "test_0049_kn_self_healing", "language": "kn", "domain": "rural health", "mutation_type": "column_not_found", "complexity": "window functions", "question": "\"rural_clinics\" ಪಟ್ಟೆಯಲ್ಲಿ ಚಿಕಿತ್ಸಾ ಪ್ರದಾನಕರ್ತರ ಶಾಲೆಯ ಸರಾಸರಿ ವಯ ಎಂದು, ಚಿಕಿತ್ಸಾ ಕೇಂದ್ರ ಸ್ಥಳದ ಮೂಲಕ ವಿಭಾಗೀಕರಿಸಲಾಗಿದೆ?", "question_en": "What is the average age of healthcare providers in the \"rural_clinics\" table, partitioned by clinic location?", "ddl": "CREATE TABLE rural_clinics (clinic_location VARCHAR(255), healthcare_provider_age INT); INSERT INTO rural_clinics (clinic_location, healthcare_provider_age) VALUES ('Location1', 35), ('Location1', 40), ('Location2', 50), ('Location2', 55);", "mutated_sql": "SELECT clinic_location_id, AVG(healthcare_provider_age) OVER (PARTITION BY clinic_location) FROM rural_clinics", "sqlite_error": "no such column: clinic_location_id", "thought": "Thought: The previous query failed with OperationalError (no such column: clinic_location_id). Checking the schema for table(s) rural_clinics, The SQL query calculates the average age of healthcare providers in each clinic location by using the window function AVG() with the PARTITION BY clause. This creates a new \"column\" of average healthcare provider age for each unique value in the \"clinic_location\" column.", "gold_sql": "SELECT clinic_location, AVG(healthcare_provider_age) OVER (PARTITION BY clinic_location) FROM rural_clinics;"}, {"id": "test_0143_kn_self_healing", "language": "kn", "domain": "energy", "mutation_type": "table_not_found", "complexity": "single join", "question": "ದೇಶ ಪ್ರತಿಭಾಗಿಯ ಶಕ್ತಿ ಸಂರಕ್ಷಣ ಕಾಬಿಲತೆ ಮತ್ತು ಶಕ್ತಿ ಸಮರ್ಥತೆಯ ಕ್ರೋಸ್-ಟಾಬ್ ಪ್ರದರ್ಶನ ನೀಡಿ.", "question_en": "Provide a cross-tabulation of energy storage capacity and energy efficiency by country", "ddl": "CREATE TABLE energy_storage (country VARCHAR(20), capacity INT);CREATE TABLE energy_efficiency (country VARCHAR(20), efficiency INT);", "mutated_sql": "SELECT e.country, e.capacity, ee.efficiency FROM energy_storage_v2 AS e JOIN energy_efficiency AS ee ON e.country = ee.country", "sqlite_error": "no such table: energy_storage_v2", "thought": "Thought: The previous query failed with OperationalError (no such table: energy_storage_v2). Checking the schema for table(s) energy_efficiency, energy_storage, This SQL query provides a cross-tabulation of energy storage capacity and energy efficiency by country. It uses a JOIN to combine data from the energy_storage and energy_efficiency tables based on the country, and returns a table with the country, energy storage capacity, and energy efficiency.", "gold_sql": "SELECT e.country, e.capacity, ee.efficiency FROM energy_storage e JOIN energy_efficiency ee ON e.country = ee.country;"}, {"id": "test_0104_kn_self_healing", "language": "kn", "domain": "arctic research", "mutation_type": "table_not_found", "complexity": "aggregation", "question": "ಆರ್ಕ್ಟಿಕ್ನಲ್ಲಿ ಪ್ರತಿ ವರ್ಷಕ್ಕೆ ಅತಿ ಉನ್ನತ ಮತ್ತು ಅತಿ ಹೆಚ್ಚು ತಾಪಮಾನ ಯಾವುದೇ?", "question_en": "Find the maximum and minimum temperatures for each year in the Arctic.", "ddl": "CREATE TABLE climate_data (id INT, region VARCHAR, year INT, temperature DECIMAL(5,2));", "mutated_sql": "SELECT year, MAX(temperature) AS max_temp, MIN(temperature) AS min_temp FROM climate_data_tbl WHERE region = 'Arctic' GROUP BY year", "sqlite_error": "no such table: climate_data_tbl", "thought": "Thought: The previous query failed with OperationalError (no such table: climate_data_tbl). Checking the schema for table(s) climate_data, This query groups the climate_data table by year and calculates the maximum and minimum temperature for the Arctic region for each year.", "gold_sql": "SELECT year, MAX(temperature) as max_temp, MIN(temperature) as min_temp FROM climate_data WHERE region = 'Arctic' GROUP BY year;"}]; | |
| const LANG_NAMES = { | |
| en: "English (en)", | |
| hi: "Hindi (hi · हिन्दी)", | |
| de: "German (de · Deutsch)", | |
| kn: "Kannada (kn · ಕನ್ನಡ)" | |
| }; | |
| let activeLang = "en"; | |
| let activeIdx = 0; | |
| function esc(s) { | |
| return String(s ?? "") | |
| .replace(/&/g, "&") | |
| .replace(/</g, "<") | |
| .replace(/>/g, ">") | |
| .replace(/"/g, """); | |
| } | |
| function renderExplorer() { | |
| const langs = [...new Set(TRACES.map(t => t.language))]; | |
| const langBox = document.getElementById("lang-pills"); | |
| langBox.innerHTML = langs.map(l => | |
| `<button class="lang-pill ${l === activeLang ? "active" : ""}" onclick="selectLang('${l}')">${esc(LANG_NAMES[l] || l)}</button>` | |
| ).join(""); | |
| const filtered = TRACES.filter(t => t.language === activeLang); | |
| if (activeIdx >= filtered.length) activeIdx = 0; | |
| const sampleBox = document.getElementById("sample-pills"); | |
| sampleBox.innerHTML = filtered.map((item, idx) => | |
| `<button class="sample-pill ${idx === activeIdx ? "active" : ""}" onclick="selectSample(${idx})">Case #${idx + 1} · ${esc(item.domain)}</button>` | |
| ).join(""); | |
| const item = filtered[activeIdx]; | |
| document.getElementById("trace-meta").innerHTML = ` | |
| <span class="meta-chip">ID: ${esc(item.id)}</span> | |
| <span class="meta-chip">Domain: ${esc(item.domain)}</span> | |
| <span class="meta-chip">Mutation: ${esc(item.mutation_type)}</span> | |
| <span class="meta-chip">Complexity: ${esc(item.complexity)}</span> | |
| `; | |
| document.getElementById("trace-body").innerHTML = ` | |
| <div class="trace-step"> | |
| <div class="step-title">Turn 0 · Schema & Multilingual User Question (${esc(item.language.toUpperCase())})</div> | |
| <div><strong>Question:</strong> ${esc(item.question)}</div> | |
| ${item.question_en && item.question_en !== item.question ? `<div style="margin-top:0.25rem;color:#475569;"><strong>English Reference:</strong> ${esc(item.question_en)}</div>` : ""} | |
| <pre>${esc(item.ddl)}</pre> | |
| </div> | |
| <div class="trace-step error"> | |
| <div class="step-title">Turn 1 · Mutated SQL Execution & Real SQLite OperationalError</div> | |
| <pre>${esc(item.mutated_sql)}</pre> | |
| <pre style="margin-top:0.45rem;background:#7f1d1d;color:#fee2e2;">OperationalError: ${esc(item.sqlite_error)}</pre> | |
| </div> | |
| <div class="trace-step healed"> | |
| <div class="step-title">Turn 2 · Self-Healing Reasoning & Grounded Gold SQL</div> | |
| <div>${esc(item.thought)}</div> | |
| <pre>${esc(item.gold_sql)}</pre> | |
| </div> | |
| `; | |
| } | |
| window.selectLang = function(lang) { activeLang = lang; activeIdx = 0; renderExplorer(); }; | |
| window.selectSample = function(idx) { activeIdx = idx; renderExplorer(); }; | |
| renderExplorer(); | |
| </script> | |
| </body> | |
| </html> | |