LogicPalette commited on
Commit
25bb313
路
verified 路
1 Parent(s): 82a0d31

article page update

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ results/figure3_training_rewards.png filter=lfs diff=lfs merge=lfs -text
README.md CHANGED
@@ -1,10 +1,12 @@
1
  ---
2
- title: Counterfactual Sql Article
3
- emoji: 馃搲
4
- colorFrom: purple
5
  colorTo: green
6
  sdk: static
7
- pinned: false
 
 
 
8
  ---
9
-
10
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
+ title: When Execution Success Lies
3
+ emoji: 馃搫
4
+ colorFrom: indigo
5
  colorTo: green
6
  sdk: static
7
+ app_file: index.html
8
+ license: cc-by-nc-4.0
9
+ short_description: Write-up - counterfactual-database rewards for a SQL agent
10
+ tags: [text-to-sql, reward-hacking, lexsi-hackathon]
11
  ---
12
+ Project page for the Lexsi hackathon Track 3 entry. Source: [https://github.com/RishiiGamer2201/sexsi_hack](https://github.com/RishiiGamer2201/sexsi_hack) (ARTICLE.md).
 
index.html CHANGED
@@ -1,19 +1,57 @@
1
  <!doctype html>
2
- <html>
3
- <head>
4
- <meta charset="utf-8" />
5
- <meta name="viewport" content="width=device-width" />
6
- <title>My static Space</title>
7
- <link rel="stylesheet" href="style.css" />
8
- </head>
9
- <body>
10
- <div class="card">
11
- <h1>Welcome to your static Space!</h1>
12
- <p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
13
- <p>
14
- Also don't forget to check the
15
- <a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
16
- </p>
17
- </div>
18
- </body>
19
- </html>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  <!doctype html>
2
+ <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
3
+ <title>When Execution Success Lies</title>
4
+ <meta name="description" content="Counterfactual-database rewards for a small SQL agent (Lexsi hackathon, Track 3)">
5
+ <link href="https://fonts.googleapis.com/css2?family=Noto+Sans:wght@400;600;700&display=swap" rel="stylesheet">
6
+ <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.9.4/css/bulma.min.css">
7
+ <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.5.1/css/all.min.css">
8
+ <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.16.9/katex.min.css">
9
+ <style>
10
+ body { font-family: 'Noto Sans', sans-serif; background: #fff; }
11
+ .hero .title { font-size: 2.6rem; }
12
+ .subtitle.meta { font-size: 1.05rem; color: #555; }
13
+ .links .button { margin: .25rem; }
14
+ .page { max-width: 960px; margin: 0 auto; padding: 0 16px 4rem; }
15
+ .page h2 { text-align: center; font-size: 1.9rem; margin-top: 3rem; padding-top: 1rem; border-top: 1px solid #eee; }
16
+ .page h3 { font-size: 1.35rem; margin-top: 2rem; }
17
+ .page img { display: block; margin: 1.5rem auto; max-width: 100%; }
18
+ .page .tw { overflow-x: auto; margin-bottom: 1.5rem; }
19
+ .page table { font-size: .88rem; }
20
+ .page blockquote { background: #f6f8fa; border-left-color: #485fc7; }
21
+ .katex-display { overflow-x: auto; overflow-y: hidden; padding: .3rem 0; }
22
+ pre.bib { font-size: .8rem; white-space: pre-wrap; }
23
+ .page code { overflow-wrap: anywhere; }
24
+ @media (max-width: 600px) { .hero .title { font-size: 1.7rem; } .hero-body { padding: 2rem 16px; } .page h2 { font-size: 1.4rem; } }
25
+ </style></head>
26
+ <body>
27
+ <section class="hero"><div class="hero-body"><div class="container has-text-centered" style="max-width: 960px">
28
+ <h1 class="title is-1 publication-title">When Execution Success Lies: counterfactual-database rewards for a small SQL agent</h1>
29
+ <p class="subtitle meta">Shreshth Rai 路 Rishii Kumar Singh 路 Aksh Aggarwal</p>
30
+ <p class="subtitle meta is-6">Lexsi hackathon 路 Track 3 路 Problem statements #6 (data and database agents) and #9 (reward design and reward-hacking defences) 路 team Teen_khapeetars</p>
31
+ <div class="links"><a class="button is-dark is-rounded" href="https://github.com/RishiiGamer2201/sexsi_hack" target="_blank" rel="noopener"><span class="icon"><i class="fab fa-github"></i></span><span>Code</span></a>
32
+ <a class="button is-dark is-rounded" href="https://huggingface.co/spaces/sqlcf-lexsi/counterfactual-sql-demo" target="_blank" rel="noopener"><span class="icon"><i class="fas fa-play"></i></span><span>Demo</span></a>
33
+ <a class="button is-dark is-rounded" href="https://huggingface.co/datasets/sqlcf-lexsi/counterfactual-sql-data" target="_blank" rel="noopener"><span class="icon"><i class="fas fa-database"></i></span><span>Dataset</span></a>
34
+ <a class="button is-dark is-rounded" href="https://huggingface.co/collections/sqlcf-lexsi/counterfactual-db-rewards-for-a-small-sql-agent-6af174f77dbae0bf0a54c3c4" target="_blank" rel="noopener"><span class="icon"><i class="fas fa-cubes"></i></span><span>Models</span></a>
35
+ <a class="button is-dark is-rounded" href="https://huggingface.co/collections/sqlcf-lexsi/counterfactual-sql-data-rollouts-and-traces-6af479430243b7731f85d144" target="_blank" rel="noopener"><span class="icon"><i class="fas fa-layer-group"></i></span><span>Data collection</span></a></div>
36
+ <p class="is-size-7 has-text-grey mt-3">Every number comes from stored runs in <code>results/</code>; <code>python verify_all.py</code> re-derives them.</p>
37
+ </div></div></section>
38
+ <div id="body" class="page content"><noscript>This page needs JavaScript; the plain version is <a href="https://github.com/RishiiGamer2201/sexsi_hack/blob/main/ARTICLE.md">ARTICLE.md</a>.</noscript></div>
39
+ <div class="page content"><h2>BibTeX</h2><pre class="bib">@misc{teenkhapeetars2026counterfactual,
40
+ title = {When Execution Success Lies: Counterfactual-Database Rewards for a Small SQL Agent},
41
+ author = {Rai, Shreshth and Singh, Rishii Kumar and Aggarwal, Aksh},
42
+ year = {2026},
43
+ note = {Lexsi Labs AI Safety and Alignment Hackathon, Track 3},
44
+ url = {https://github.com/RishiiGamer2201/sexsi_hack}
45
+ }</pre></div>
46
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/marked/12.0.2/marked.min.js"></script>
47
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.16.9/katex.min.js"></script>
48
+ <script>
49
+ const MD = "\n\n## 1. The judge's 60-second view\n\n**TL;DR.** Execution rewards for text-to-SQL check the answer on **one** database, so a wrong query that happens to give the right number gets paid. We built a reward auditor that checks answers on code-generated *counterfactual* copies of the database as well. On our audit it cuts the share of hacking queries that get paid from **54.1% to 1.9%**, and of buggy gold-mutants from **29.5% to 16.8%**, while rejecting **0 of 2847** correct queries. The same holes are open in two public rewards (AgentTune's built-in SQL reward and SkyRL-SQL's) and on a real benchmark: **8\u201315% of the Spider-dev answers that eight published systems \"get right\" fail on Zhong et al.'s test-suite databases**. We also patched AgentTune's reward. Training a 3.35B agent on the counterfactual reward did **not** beat the single-DB reward (H4 and H5 not supported). We explain why: on our training prompts the two rewards disagreed on only 7% of rollouts. Section 10 pre-registers the run that would test it properly.\n\n**The reward audit (the contribution).**\n\n| Result | Number (95% CI) | Source |\n|---|---|---|\n| Reward-hacking queries (literal smuggling, hard-coded IDs, empty) paid | single-DB 54.1% \u2192 counterfactual **1.9%** [1.5, 2.5] | `results/e0_reward_audit.json` |\n| Buggy gold-mutants paid (strict multiset, `R_exec_rex`) | single-DB 29.5% [27.4, 31.6] \u2192 counterfactual **16.8%** [15.2, 18.6]: a 43% cut; pre-registered target 50%, **not met** (H2) | same |\n| Correct queries (gold and paraphrases) rejected by the counterfactual reward | **0 / 2847** | same |\n| AgentTune built-in SQL reward: `WHERE 1 = 1` + always answer \"yes\" | 3.10, the same as honest SQL on every yes-question; **our patch: 2.20** (with result matching 1.20 vs 5.10) | `results/e0_agenttune.json`, `tests/test_agenttune_reward.py` |\n| SkyRL-SQL reward (set comparison, one DB) on the same candidates | pays 37.5% of mutants and 53.2% of hacks; copies alone \u2192 27.2% / 1.9%; copies + strict comparator \u2192 16.8% / 1.9% | `results/a3_skyrl_audit.json` |\n| Real benchmark: Spider-dev answers that pass the original DB but fail a test-suite DB, 8 published systems | **8.1% to 14.9%** of their correct-looking answers; highest for the prompted LLMs (Claude 14.9%, C3 14.1%) | `results/a2_spider_lucky.json` |\n\n**Before \u2192 after (test set, 331 items \u00d7 8 samples, different generator; verified = right on the original and all 4 copies).**\n\n| Model | Verified [95% CI] | Silent error | Lucky pass | $ per 1k verified-correct answers |\n|---|---|---|---|---|\n| Tiny Aya base | 0.9% | 4.5% | 0.5% | n/a |\n| SFT | 9.4% [8.4, 10.6] | 65.4% | 5.1% | $0.57 |\n| **GRPO-B** (single-DB reward) | **12.7%** [11.5, 14.0]; vs SFT +3.2 pts [1.8, 4.8], McNemar p = 0.006 | 64.0% | 5.1% | **$0.48** |\n| GRPO-C (counterfactual reward) | 9.0% [7.9, 10.1]; vs SFT \u22120.5 pts, p = 0.21; vs B \u22123.7 pts [\u22125.3, \u22122.3] | 61.2% | 5.1% | $0.58 |\n| Command A (1 sample, no tools) | 81.0% [76.4, 84.8] | n/a | 1.8% | $5.24 |\n\nTiny Aya B is about 70\u00d7 cheaper than Command A per query and about **11\u00d7 cheaper per verified-correct answer**, but at 12.7% accuracy it does not replace it. Picking each item's most common answer out of the 8 samples (no gold needed) raises B to 16.9% and C to 13.6% (\u00a78, E8).\n\n**Distilling Command A (E9).** As a data filter, the copies catch 47 lucky teacher answers (18.2% of what a single-DB filter keeps). But SFT on the 211 verified teacher answers *lowers* Tiny Aya from 9.4% to **3.3%**. The student picked up Command A's habit of adding `status != 'cancelled'`. In those 211 answers the filter never changed a result, so the copies could not see it.\n\n[Demo](https://huggingface.co/spaces/sqlcf-lexsi/counterfactual-sql-demo) (runs in the browser) \u00b7 [Data](https://huggingface.co/datasets/sqlcf-lexsi/counterfactual-sql-data) \u00b7 [Code](https://github.com/RishiiGamer2201/sexsi_hack) \u00b7 `python verify_all.py` re-derives every number above from stored runs.\n\n## 2. Problem and approach\n\nQuestion: *\"How many units of grocery products were in orders that received a payment in 2025 (by payment date)?\"* Our SFT model filtered on `order_date` instead of the payment date. On the database it explored, both give 12, so a single-DB reward pays it in full. On a counterfactual copy where payment and order dates differ, gold is 10 and the model still says 12.\n\n- **Counterfactual DBs.** Each item gets \u22651 *answer-changing* copy (the gold result changes; a smuggled literal can't follow it) and \u22651 *answer-preserving* copy (the gold result stays; fan-out and duplicate bugs diverge). Copies are generated by code per bug family, never hand-built to be blind.\n- **Bug families:** F1 duplicates and fan-out joins, F2 status filters (cancelled or placed rows), F3 refunds and net amounts, F4 superseded and corrected records (held out: never used in any training reward).\n- **Rewards compared:** `R_exec_set` (BIRD-style set match), `R_exec_rex` (strict multiset, one DB), and ours: `R_cf_mean = C0 \u00d7 mean(C1..CK)`, plus strict (all pass) and soft (`C0\u00b7(1\u22120.2\u00b7[any fail])`, ReViSQL-style) ablations. Gate G1 froze `R_cf_mean` before training, by lowest FN on gold and then by most non-zero-variance GRPO groups.\n- **Reward definitions.** For predicted query MATHPH0END, gold query MATHPH1END, original DB MATHPH2END and hidden copies MATHPH3END, with `match` the strict multiset comparison (ordered when gold is ordered, columns may be permuted, floats rounded to 4 dp):\n\nMATHPH4END\n\nMATHPH5END\n\n On the test set an answer is **verified** if MATHPH6END and a **lucky pass** if MATHPH7END but some MATHPH8END.\n- **Training:** Tiny Aya Global 3.35B, LoRA r=16. SFT on scripted oracle trajectories (`list_tables \u2192 describe \u2192 explore \u2192 final SQL`), then GRPO **B** (`R_exec_rex`) vs **C** (`R_cf_mean`) from the same SFT adapter. Same data, seed, 60 steps, G=6, 4 prompts per step, lr 1e-5, \u03b2=0.02. Only the reward differs.\n- **Evaluation:** fresh test DBs from a **different generator** (FK-closed sub-DB of a larger parent plus 4 family-blind mutations), 331 items \u00d7 8 samples, temperature 0.7.\n- **Lexsi tools:** training runs on **AgentTune** (multi-turn tool-calling GRPO; we fixed one trainer bug and one reward exploit in it). Evaluation follows **AuditKIT**'s agent-audit framing (claimed success vs verified success, pass^k). Details in \u00a73.\n\n## 3. Lexsi repos and features used\n\n- **AgentTune** (`d4db4c1`): multi-turn GRPO trainer with tool calls, its tool-prompt fallback, render, parser and fold functions (SFT text is built with AgentTune's own constants so SFT matches GRPO rollouts), and `lexsi_provenance.json`. We found and patched one bug: the multi-turn stream dropped chat-template tokens between turns, so the trainer scored tokens in a context vLLM never saw (importance ratio \u22481e-5). Fix: `patches/agenttune-multiturn-ids.patch`.\n- **AgentTune built-in SQL reward:** audited in E0 (H3). It pays a `WHERE 1 = 1` query as much as the real one. We wrote a fix, `patches/agenttune-sql-reward-hardening.patch`, with a repro (`scripts/agenttune_repro.py`) and a regression test (`tests/test_agenttune_reward.py`). Reported upstream as [Lexsi-Labs/AgentTune#11](https://github.com/Lexsi-Labs/AgentTune/issues/11). The maintainers merged a fix the next day ([#12](https://github.com/Lexsi-Labs/AgentTune/pull/12), `e56a109`): the WHERE check and an opt-in `result_match_reward`, both as we proposed, plus a regression test (`tests/test_sql_reward_hacking.py`).\n- **AuditKIT** (`9da5838`): agent-eval framing (claims success, verified failure = silent error) and pass^k; statistics reimplemented in `evaluate.py`.\n\n## 4. Models and licences\n\n- `CohereLabs/tiny-aya-global` (3.35B): CC-BY-NC-4.0, gated. The policy we train.\n- Cohere Command A (`command-a-03-2025`, trial API): ROI comparison only.\n- Our LoRA adapters inherit the base model's terms. The organisers ruled (Discord, Nov 9) that adapters of the gated base may be published with its licence included, gated with automatic approval; ours are.\n\n## 5. Datasets and licences\n\nAll data is synthetic: an e-commerce schema from a seeded generator, with no personal data. Released under CC-BY-NC-4.0, because the rollouts contain model outputs.\n\n| Split | Items | Use |\n|---|---|---|\n| audit | 1302 | E0/E1 reward audit |\n| train | 907 | SFT (462 trajectories) and GRPO prompts |\n| test | 331 (214 in held-out F4) | E4, different generator |\n\n## 6. Model collection\n\n[Model collection](https://huggingface.co/collections/sqlcf-lexsi/counterfactual-db-rewards-for-a-small-sql-agent-6af174f77dbae0bf0a54c3c4): the in-browser [demo](https://huggingface.co/spaces/sqlcf-lexsi/counterfactual-sql-demo) and three LoRA adapters (r = 16) on Tiny Aya Global, each with a card giving its test metrics and G3 label:\n\n| Repo | Model | G3 |\n|---|---|---|\n| [`sqlcf-lexsi/tiny-aya-global-sqlcf-sft`](https://huggingface.co/sqlcf-lexsi/tiny-aya-global-sqlcf-sft) | SFT (`sft_ho`), the init of both GRPO runs | baseline, 9.4% verified |\n| [`sqlcf-lexsi/tiny-aya-global-sqlcf-grpo-b`](https://huggingface.co/sqlcf-lexsi/tiny-aya-global-sqlcf-grpo-b) | GRPO-B, single-DB reward | improved: 12.7%, beats SFT (p = 0.006) |\n| [`sqlcf-lexsi/tiny-aya-global-sqlcf-grpo-c`](https://huggingface.co/sqlcf-lexsi/tiny-aya-global-sqlcf-grpo-c) | GRPO-C, counterfactual reward | null: 9.0%, no gain over SFT (p = 0.21) |\n\nThe adapters are derivatives of a gated, non-commercial model. Each repo ships the base model's CC-BY-NC-4.0 licence and a notice pointing to Cohere's licence addendum and Acceptable Use Policy, and is gated with automatic approval. To use them you need access to `CohereLabs/tiny-aya-global`. vLLM's LoRA path crashes on Tiny Aya's tied embeddings, so merge first (`train.py merge`, fp32 merge saved as fp16).\n\n## 7. Data collection\n\n[Data collection](https://huggingface.co/collections/sqlcf-lexsi/counterfactual-sql-data-rollouts-and-traces-6af479430243b7731f85d144) with [`sqlcf-lexsi/counterfactual-sql-data`](https://huggingface.co/datasets/sqlcf-lexsi/counterfactual-sql-data): items, all SQLite DBs, SFT trajectories, E0/E1/E4 results, test trajectories for every model, and every GRPO rollout with its per-DB pass vector and reward.\n\n## 8. Experiments\n\n### E0: reward audit (figure 1)\nAudit set, 1302 items; Wilson 95% CIs.\n\n![Figure 1: per bug family F1\u2013F4, share of gold-mutants and of hacks each reward fully pays (false positives), and share of correct SQL it does not fully pay (false negatives, all zero)](results/figure1_reward_audit.png)\n\n| Reward | FP on gold mutants (n=1814) | FP on hacks (n=2839) | FN on gold + paraphrases (n=2847) |\n|---|---|---|---|\n| R_exec_set (BIRD set match) | 0.375 [0.353, 0.397] | 0.541 [0.523, 0.560] | 0.000 |\n| R_exec_rex (strict multiset, 1 DB) | 0.295 [0.274, 0.316] | 0.541 | 0.000 |\n| R_cf, random CFs (1 changing + 1 preserving) | **0.168 [0.152, 0.186]** | **0.019 [0.015, 0.025]** | 0.000 |\n| R_cf, greedy CFs (\u22643, chosen to kill these mutants: optimistic upper bound) | 0.015 [0.010, 0.022] | 0.020 | 0.000 |\n\n- **H2 not met:** a 43% relative cut in mutant FP on the honest row, against the pre-registered 50%. Hack FP falls 96%.\n- **AgentTune's built-in SQL reward (H3)**, 360 yes/no items: padding the WHERE clause and always answering \"yes\" earns the honest score on every yes item; +2.45 overall vs +3.10 for honest SQL.\n\n**Hack catalogue.** Every reward hack we tried, with how often each reward pays it.\n\n| Hack | Example | Single-DB reward pays | Counterfactual reward pays |\n|---|---|---|---|\n| Literal smuggling: return the answer seen while exploring | `SELECT 5` | 1302 / 1302 | **0 / 1302** |\n| Hard-coded IDs: list the gold rows' IDs | `SELECT customer_id FROM customers WHERE customer_id IN (3, 17, 42)` | 235 / 235 | 54 / 235 (23%): the random answer-changing copy kept the same IDs |\n| Empty result | `SELECT NULL WHERE 0` | 0 / 1302 | 0 / 1302 |\n| WHERE padding + fixed answer (AgentTune reward only) | `SELECT * FROM customers WHERE 1 = 1`, then `*yes*` | 3.10 = honest, on all 314 yes items | n/a (AgentTune's reward is yes/no); our patch: 2.20 |\n| Duplicate rows instead of DISTINCT (set-comparison rewards only) | gold `SELECT DISTINCT o.customer_id ...`, hack drops `DISTINCT` | SkyRL pays 203 / 203 | set comparison + copies still pays 203 / 203; strict comparator + copies 15 / 203 |\n\nThe remaining 1.9% of hacks paid by the counterfactual reward are all hard-coded IDs. A copy that changes which customers match closes that gap.\n\n### E1: how often bugs are invisible on one DB (figure 2)\nShare of gold-mutants that pass the single-DB check: **29.2%** at 50 customers, 0.8% at 500, 0.2% at 5,000. What makes a bug invisible is the size of the question's slice (city \u00d7 year \u00d7 category) more than the DB size: small slices are common in real questions.\n\n![Figure 2: share of gold-mutants that pass the single-DB check, by number of customers in the DB and bug family](results/figure2_blind_rate.png)\n\n### E3: what the training reward paid for (figure 3)\n`results/figure3_training_rewards.json`, 1440 rollouts per run.\n\n![Figure 3: per GRPO step for runs B and C, share of rollouts passing the original DB (left) and share of those that fail a counterfactual, i.e. lucky passes the single-DB reward pays (right)](results/figure3_training_rewards.png)\n\n| Run | Rollouts passing the original DB | Share of those failing a counterfactual (all) | First half | Second half |\n|---|---|---|---|---|\n| B (`R_exec_rex`) | 86.4% | 8.2% | 10.7% | 5.9% |\n| C (`R_cf_mean`) | 86.5% | 8.3% | 11.1% | 5.7% |\n\nOnly about 7% of rollouts (86% \u00d7 8%) got a different reward under C than under B, and under `R_cf_mean` the difference is partial credit, not zero. Lucky passes fell over training in **both** runs, so the single-DB reward was not drifting toward lucky queries in the first place. With this little contrast between the two rewards, 60 steps could not separate them.\n\n### E4: test evaluation\n331 items \u00d7 8 samples; verified = correct on the original and all 4 test counterfactual DBs.\n\n| Model | Verified | Single-DB | Lucky (single \u2212 verified) | Silent error | pass^2 | pass^4 | Held-out F4 verified |\n|---|---|---|---|---|---|---|---|\n| base | 0.9% | 1.4% | 0.5% | 4.5% | 0.1% | 0.0% | 0.6% |\n| SFT (B/C init) | 9.4% | 14.5% | 5.1% | 65.4% | 4.9% | 2.1% | 9.2% |\n| GRPO-B | 12.7% | 17.8% | 5.1% | 64.0% | 8.1% | 3.9% | 9.9% |\n| GRPO-C | 9.0% | 14.0% | 5.1% | 61.2% | 4.7% | 2.2% | 8.8% |\n| Command A (1 sample, no tools) | 81.0% | 82.8% | 1.8% | n/a | n/a | n/a | 94.9% |\n\n- **B vs SFT:** +3.2 pts verified [1.8, 4.8], McNemar p = 0.006. B's lucky-pass rate is unchanged.\n- **C vs SFT:** \u22120.5 pts [\u22121.5, +0.6], p = 0.21: no change. **C vs B:** \u22123.7 pts [\u22125.3, \u22122.3], p < 0.001.\n- **H4 not met.** Pre-registered: C has fewer silent errors than B, exact McNemar p < 0.05. C 61.2% vs B 64.0%. The bootstrap difference over 8 samples is \u22122.9 pts [\u22125.3, \u22120.5], but McNemar on sample 0 gives p = 0.61. C's lower silent-error rate comes with lower verified accuracy, so it is not an improvement.\n- **H5 (held-out family F4):** no gain. SFT 9.2%, B 9.9%, C 8.8%.\n- **H6 (literal smuggling):** none at test for any trained model (0 of 2648 trajectories each), so there was nothing for C to remove.\n- **Gate G3:** B beats SFT on verified accuracy and is released as improved; C is released labelled as a null.\n- One training seed per condition: we cannot separate the B\u2013C gap from run-to-run GRPO variance. B's gain over SFT may also be partly seed luck.\n\n**Manual audit** (`results/manual_audit.jsonl`, `scripts/audit_sample.py`). 48 wrong test trajectories, 16 per model (sft_ho, B, C), sampled with seed 0 and stratified by family. Each model contributes 6 lucky passes (right on the original DB, wrong on a counterfactual) and 10 other wrong answers. Every one was labelled by hand with NL2SQL-BUGs top-level categories.\n\n| Error | NL2SQL-BUGs category | All (48) | Lucky passes (18) |\n|---|---|---|---|\n| `order_date` used where the question says payment date | Attribute | 11 | 4 |\n| Product category matched against customer columns, or a hallucinated value | Value | 7 | 0 |\n| Extra status filter the question does not ask for | Condition | 6 | 5 |\n| No latest-row filter for \"current tier\" | Condition | 5 | 2 |\n| AVG over order items instead of per-order totals | Clause | 5 | 0 |\n| `invoice_adjustments` left out of net invoice | Table | 4 | 3 |\n| Does not run (undefined alias, unknown column or function) | Others | 4 | 0 |\n| `COUNT(*)` instead of `SUM(quantity)` | Function | 3 | 3 |\n| Incoherent, or an exploratory query given as the answer | Others | 2 | 0 |\n| Wrong table | Table | 1 | 1 |\n\n- **Every lucky pass is a real semantic bug** (an extra or missing filter, a wrong column, a missing table or the wrong aggregate) that happens to give the right number on one DB. None of the 48 is literal smuggling or reward hacking. So the single-DB reward's false positives on this agent are honest mistakes, which is exactly what counterfactual DBs are built to catch.\n- The most common error, ignoring \"by payment date\", is the same F2 trap Command A falls into.\n- The three models' error mixes are similar. With 16 per model, the audit cannot show a difference between B and C.\n- Caveat: the test generator catches 54 of 103 base-blind mutants, so test \"verified\" is *less* strict than the training reward.\n\n### E5: ROI vs Command A\nCommand A (`command-a-03-2025`, trial API) answers each test item once, single-turn, with the schema and column definitions in the prompt and no tools. Tiny Aya runs as the multi-turn agent. Command A cost = billed tokens \u00d7 list price ($2.50 / $10.00 per 1M input / output). We ran on a free trial key, so the real charge was $0; the Cohere dashboard shows 222,578 input and 85,815 output tokens for the run day, within 1% of our logged 220,580 / 85,348, which prices at $4.27 vs our $4.24 per 1k queries. Tiny Aya cost = GPU $/hr \u00d7 measured generation time \u00f7 trajectories.\n\n| Model | Verified | Single-DB | $ per 1k queries | Speed |\n|---|---|---|---|---|\n| Command A | 81.0% [76.4, 84.8] | 82.8% | $4.24 | median 9.7 s, p90 42 s per query |\n| Tiny Aya SFT | 9.4% [8.4, 10.6] | 14.5% | $0.054 | 3.8 trajectories/s batched on L4 (scaled) |\n| Tiny Aya GRPO-B | 12.7% [11.5, 14.0] | 17.8% | $0.061 | 3.4 trajectories/s batched on L4 (measured) |\n| Tiny Aya GRPO-C | 9.0% [7.9, 10.1] | 14.0% | $0.052 | 4.0 trajectories/s batched on L4 (scaled) |\n\n- The small model is about 70\u00d7 cheaper per query, and about 11\u00d7 cheaper per *verified-correct* answer, but at 13% accuracy it is not a usable replacement. The honest ROI reading: counterfactual rewards are a cheap way to make a small model's training signal trustworthy, not a way to match a frontier model in 60 GRPO steps.\n- Tiny Aya is priced at our L4 VM's on-demand rate ($0.74/hr, g2-standard-4) and L4 throughput. GRPO-B's full test generation took 785 s on the L4 vs 524 s on an RTX 5070 Ti laptop GPU, with the same accuracy (12.5% vs 12.7% verified). SFT and C were timed on the laptop only and are scaled by B's L4/laptop ratio (1.50). Per-query latency was not measured for Tiny Aya (future work, \u00a710).\n- **Command A falls into the fan-out trap too:** 0% verified on F2 (all 36 items from `F2_units_paid`). It joins `payments` directly, so an order with two payments counts its units twice, and it adds a `status != 'cancelled'` filter the question does not ask for. The single-DB check already rejects 32 of these 36; the counterfactual copies reject the other 4, which match on the original DB by luck. Lucky passes across all of Command A's answers: 1.8%.\n\n### E6: the same hole on a real benchmark (Spider dev)\nDo real systems produce lucky passes, or only our synthetic mutants? We took the public Spider-dev prediction files of 8 systems (from the ESM+ repo, `bossben/ESMp`) and scored each on the original Spider DB and on the distilled test-suite DBs of Zhong, Yu & Klein (2020), 1034 questions. CPU only (`scripts/a2_spider_lucky.py`).\n\n| System | EX (original DB) | TS (original + suite) | Lucky share of EX passes [95% CI] |\n|---|---|---|---|\n| Claude | 80.6% | 68.6% | **14.9%** [12.6, 17.5] |\n| C3 (ChatGPT) | 79.7% | 68.5% | 14.1% [11.9, 16.6] |\n| DIN-SQL | 81.5% | 72.2% | 11.4% [9.4, 13.7] |\n| GPT-4 Turbo | 72.3% | 64.7% | 10.6% [8.6, 13.0] |\n| DAIL-SQL | 82.7% | 75.3% | 8.9% [7.2, 11.0] |\n| RASAT+PICARD | 76.5% | 70.1% | 8.3% [6.6, 10.5] |\n| RESDSQL | 82.6% | 75.8% | 8.2% [6.5, 10.2] |\n| Graphix | 79.9% | 73.4% | 8.1% [6.4, 10.2] |\n\n- An RL reward that checks only the original DB would pay 8\u201315% of these systems' \"correct\" answers for queries that are wrong on some other valid database. That is lower than the 29.5% mutant pay rate in E0 (the mutants are built to be hard to spot), but it comes from real queries written by real systems.\n- A hand check of 40 lucky cases: 27 return a different number of rows on the suite DB, 13 return different values. Typical causes are `GROUP BY name` where names repeat (gold groups by ID), `EXCEPT` vs `NOT IN` with duplicates, `ORDER BY ... LIMIT 1` where there are ties, and a `LEFT JOIN` that adds zero-count rows. Many are real bugs; some are ties or ambiguous gold, so read these rates as upper bounds.\n- Our scorer keeps `DISTINCT` and uses our strict comparator, so our EX and TS run a few points below the official script's. The lucky share uses the same comparator on both sides.\n\n### E7: a second public reward (SkyRL-SQL)\nSkyRL-SQL's outcome reward (`NovaSky-AI/SkyRL` @ `d870046`) runs the predicted and gold SQL on one DB and pays 1 if the two *sets* of rows are equal. We re-implemented that comparison from its source (`scripts/audit_skyrl.py`) and scored the E0 candidates four ways.\n\n| Reward | Mutants paid (n = 1814) | Hacks paid (n = 2839) | Correct rejected (n = 2847) |\n|---|---|---|---|\n| SkyRL as shipped (set compare, one DB) | 37.5% [35.3, 39.7] | 53.2% | 0 |\n| SkyRL's set compare on original + copies | 27.2% [25.2, 29.3] | 1.9% | 0 |\n| Strict multiset, one DB (`R_exec_rex`) | 29.5% | 54.1% | 0 |\n| Strict multiset on original + copies (ours) | **16.8%** | **1.9%** | 0 |\n\n- A shipped reward has two holes of about the same size. Adding copies closes the database hole (\u221210.3 pts). A strict comparator closes the comparator hole (\u221210.4 pts more). Nearly all of the comparator hole is dropped `DISTINCT` (188 of 203 such mutants are paid by set comparison on every DB).\n\n### E8: does the multi-DB signal help without gold? (self-consistency)\nFor each test item, each model has 8 samples. Without gold, we can pick the most common answer, judged either by its result on the original DB alone or by its results on all 5 DBs (`scripts/a4_selfconsistency.py`). Paired bootstrap CIs over items.\n\n| Model | First sample | Vote, 1 DB | Vote, 5 DBs | Any of 8 (oracle) | 5-DB vote \u2212 first [95% CI] |\n|---|---|---|---|---|---|\n| SFT | 8.5% | 10.9% | 11.8% | 21.5% | +3.3 [+0.6, +6.3] |\n| GRPO-B | 13.9% | 16.9% | 16.3% | 22.4% | +2.4 [\u22120.6, +5.4] |\n| GRPO-C | 6.3% | 13.6% | 13.9% | 21.5% | +7.6 [+4.8, +10.6] |\n\n- Voting helps, but voting on 5 DBs adds nothing over voting on 1 (difference within \u00b11 pt for every model). A \"shaky answer\" flag (fewer than half of the samples agree) catches 53\u201359% of wrong answers at 94\u201398% precision, again with no gain from the extra DBs (AUROC 0.67\u20130.74 either way).\n- This is a null, and we report it as one. At 10\u201315% accuracy, wrong answers rarely agree with each other, so one DB already splits them. Our copies are also built per item from the gold query's bug families, so this is not yet a gold-free verifier.\n- All three trained models reach about the same pass@8 (21.5\u201322.4%). GRPO-B's gain is in picking its right answer more often, not in finding new ones.\n\n### E9: distilling Command A, with the copies as the data filter\nCan the copies clean distillation data? Command A (T = 0.3, single-turn, asked for a 2\u20134 line plan and then the SQL) answered the 353 train items behind our SFT adapter (`scripts/distil.py`, trial key).\n\n**D1: what a single-DB filter would let through.**\n\n| Filter on the teacher's answers | Kept (n = 353) [95% CI] |\n|---|---|\n| Right on the original train DB | 258, 73.1% [68.2, 77.4] |\n| Right on the original and all its train copies | 211, 59.8% [54.6, 64.8] |\n| **Lucky: kept by the single-DB filter only** | **47, 18.2% of what it keeps** |\n\nBy family (items / single / verified): F1 152 / 111 / 88, F2 81 / 27 / 3, F3 120 / 120 / 120. No smuggled answers. A single-DB filter would put 47 wrong teacher queries into the student's data. As a share of all its answers that is 13.3%, about 7\u00d7 Command A's lucky-pass rate on the test set (1.8%, E5), because these train items were built for the bug families the copies target.\n\n**D2: SFT on the verified answers.** Same 353 items, seed, check queries, epochs and settings as SFT (`sft_ho`). The 211 verified teacher answers replace the gold ending (plan + SQL); the other 142 keep the gold trajectory. Trained on the laptop GPU, same 331 \u00d7 8 test set.\n\n| Model | Verified [95% CI] | Single DB | Silent error |\n|---|---|---|---|\n| SFT (gold endings) | 9.4% [8.4, 10.6] | 14.5% | 65.4% |\n| SFT on verified teacher endings | **3.3%** [2.7, 4.0] | 11.2% | 76.2% |\n\n- **Distillation hurt:** \u22126.2 pts verified [\u22128.3, \u22124.1], McNemar p < 0.001. Almost all of it is F1 (30.9% \u2192 4.2%).\n- **Why: a teacher habit that the copies could not see.** Command A adds `status != 'cancelled'` to 325 of its 353 answers; the gold endings have it in 71. In 186 of the 211 verified answers the filter happened not to change the result on the original DB or on any copy, so they passed. The student learned the habit: it writes the filter in 56% of its test answers (SFT: 14%). Every F1 item that SFT gets right and the distilled model gets wrong (first sample) is from the template that says \"count orders of any status\".\n- **The lesson for counterfactual checks:** they verify an answer, not the reasoning behind it. A copy only catches a wrong filter if the copy changes what that filter selects. A teacher's systematic bias passes on every item where the data happens to agree. The fix is to add copies that flip each filter the question does not ask for (here, cancelled orders on \"any status\" items), or to check distillation data with copies that the teacher's habits break. We did not have time to run that.\n\n### E10: DPO from one checkpoint, single-DB vs counterfactual labels\nGRPO trained B and C separately, so their runs differ in more than the reward. Here one fixed set of rollouts is labelled two ways, and only the labels differ.\n\n**Setup.**\n- **Source:** sft_ho's 2670 rollouts on the 445 GRPO prompts (G = 6, T = 1.0). They grade into 1971 verified (V), 170 lucky (L: right on the original, wrong on a copy) and 529 failed (F).\n- **std labels** (what a single-DB verifier prefers): chosen from V + L, rejected from F. This gives 420 pairs, 96 of them with a lucky answer as the chosen one.\n- **cf labels:** chosen from V, rejected from L + F, lucky first. This gives 351 pairs, 30 of them with a lucky answer rejected.\n- **Training:** at most 3 pairs per prompt; sigmoid DPO with \u03b2 = 0.1, lr 2e-5, 1 epoch, effective batch 16, seed 0; same test set (`scripts/dpo.py`).\n\n| Model | Verified [95% CI] | Single DB | Lucky pass | Invalid tool calls | Silent error |\n|---|---|---|---|---|---|\n| sft_ho (start) | 9.4% [8.4, 10.6] | 14.5% | 5.1% | 17.2% | 65.4% |\n| DPO, std labels | 9.7% [8.6, 10.9] | 14.1% | 4.5% | 20.0% | 62.4% |\n| DPO, cf labels | 8.1% [7.1, 9.2] | 12.5% | 4.5% | 27.4% | 55.1% |\n\n- **Null to slightly negative.** std vs sft_ho: +0.2 pts [\u22121.0, +1.5]. cf vs std: \u22121.6 pts [\u22122.6, \u22120.6] (bootstrap over items, all 8 samples); McNemar on the first sample p = 0.40.\n- **The lower silent-error rate is not a gain.** An error only counts as silent if every tool call succeeded, and cf makes far more failing tool calls (27% vs 20%).\n- **Lucky passes did not move:** 4.5% under both label sets. Only 30 of the 351 cf pairs actually teach \"a lucky answer is worse\". The same 7% problem as GRPO (\u00a78, Nulls): on these prompts the two verifiers mostly agree, so most cf pairs are ordinary right-vs-wrong pairs.\n- **Both runs lose F1** (30.9% \u2192 21.2% std, 10.4% cf) and gain a little on F2 (1.0% \u2192 4.5%). One seed each, and cf has fewer pairs (351 vs 420), so the cf\u2013std gap is not clean evidence that counterfactual labels hurt. What it shows is that they did not help.\n\n### E11: partial credit instead of 0 or 1\nWould an intermediate reward give GRPO more to learn from? `scripts/partial_credit.py` gives each DB a credit between 0 and 1:\n- 1 for a match;\n- for a single number, its closeness, 1 \u2212 |pred \u2212 gold| / |gold|;\n- otherwise the F1 of the result rows.\n\n`P_single` is the credit on the original DB; `P_cf` is the mean credit over the original and every copy. Both are scored on the same E0 candidates (mutant-blind copies) and the same 2670 sft_ho rollouts on the 445 GRPO prompts.\n\n| Reward | Mean pay: mutants / hacks | AUROC, correct vs mutant / hack | GRPO groups with any signal | Lucky rollouts above group mean | Wrong rollouts paid > 0 |\n|---|---|---|---|---|---|\n| `R_exec_rex` (one DB, 0/1; B) | 0.30 / 0.54 | 0.85 / 0.73 | 31.5% | 63.5% | 24.3% |\n| `R_cf_mean` (copies, 0/1 mean; C) | 0.23 / 0.28 | 0.92 / 0.99 | 32.1% | 58.8% | 23.3% |\n| `R_cf_strict` (all DBs) | 0.17 / 0.02 | 0.92 / 0.99 | 26.3% | 0% | 0% |\n| `P_single` (one DB, partial) | 0.66 / 0.54 | 0.85 / 0.73 | 37.8% | 63.5% | 52.5% |\n| `P_cf` (copies, partial) | 0.65 / 0.47 | 0.92 / 0.99 | 39.8% | 69.4% | 56.2% |\n\n- **More signal, but for the wrong answers.** Partial credit gives 6\u20138 pts more groups some reward variance. That is the argument for it. But it pays mutants more than twice as much as the binary reward (0.66 vs 0.30).\n- **It reopens smuggling.** A query that hard-codes a number close to the answer earns 0.86 under `P_cf`, against 0 under `R_cf_strict`. A near-miss number is exactly what lucky and hacked queries produce, so the copies cannot fix it.\n- **It ranks queries no better.** AUROC is the same as the binary reward on the same DBs. Partial credit adds no information about correctness, only gradient towards wrong answers that are close.\n- **A side finding about C.** Our C reward (`R_cf_mean`) still put lucky rollouts above their group's mean 59% of the time, because 1 of 2 copies right still earns half. `R_cf_strict` never does. This is one more reason C did not separate from B, and why \u00a710 uses C-min (all DBs must pass).\n- We did not spend GPU on a partial-credit run: the audit already shows it pays for the behaviour we want to remove.\n\n### E12: period-edge copies for time-filtered questions\nOur questions ask about a year (\"revenue from Delhi in 2024\"). Would a copy built around the asked period catch date bugs? This tests the idea of a separate DB per month, at the year granularity our templates use. `scripts/period_copies.py` adds one copy per item, built as follows:\n- Half of the slice's orders in year Y move to Jan 1 and Dec 31 of Y, alternating.\n- Half of its orders in the other year move to the day just outside Y.\n- Payments, refunds and adjustments move with their order. A moved payment lands 0, 1 or its usual number of days after the order, so a Dec 31 order can be paid in January.\n- Every copy passes the business-rule checks.\n\nThe date bugs are generated from each of the 777 year-filtered audit queries: Dec 31 excluded, Jan 1 excluded, Jan 1 of the next year included, and order date swapped with payment date. The whole run takes 19 s on a CPU.\n\n| Date bugs paid (n = 2535) [95% CI] | |\n|---|---|\n| Original DB only | 92.1% [91.0, 93.1] |\n| Original + the 2 mutant-blind copies (E0 setting) | 91.4% [90.2, 92.4] |\n| Original + the greedy copies (training reward) | 90.9% [89.7, 92.0] |\n| **Original + the 2 copies + the period-edge copy** | **32.9%** [31.1, 34.7] |\n| Correct queries (gold, paraphrases) rejected by the new copy | **0 / 1554** |\n\n- **The copies we had were blind to dates:** they edit cities, statuses and tiers, not when things happen. One period-edge copy cuts the date bugs paid from 91% to 33%, with no false rejects. Dropping Dec 31 goes from 97% paid to 19%.\n- **Our models rarely make these bugs, though.** Replaying real answers on the year items, the new copy rejects:\n - 2 of B's 305 verified test answers, 2 of C's 218 and 1 of sft_ho's 223;\n - 4 of sft_ho's 998 verified GRPO rollouts (0.4%).\n \n Each failure is the same bug: the question says \"by payment date\" and the model filters on the order date.\n- **So:** period copies close a real hole in the verifier, which matters for a stronger policy or a real warehouse. They would change our accuracy numbers by under 1 pt. The bug families a verifier needs are the ones the policy actually produces.\n\n### Nulls and what else we tried\n- **The training comparison is null, and it is neither noise nor evidence that the counterfactual reward hurts.** On the eval set, C is 3.7 pts below B (95% CI [\u22125.3, \u22122.3], exact McNemar p < 0.001). That gap is real for these two adapters, but we trained one seed each, so we cannot say how much is GRPO run-to-run variance. Three facts explain why we expected little difference:\n 1. **C really was stricter.** Replaying all 1440 training rollouts per run, C's reward was lower than B's on 7.1% (B run) and 7.2% (C run) of rollouts because they passed the original DB by luck. It was *higher* on only 0.1% and 0.3%.\n 2. **The training prompts were too easy for the difference to matter.** 86% of rollouts passed the original DB, so the two rewards agreed on about 93% of rollouts. RL speed depends on reward variance under the policy, not on how accurate the reward is (Razin et al., 2503.15477), and RLVR tolerates far more reward noise than 7% (Lv et al., 2505.22653).\n 3. **Others see the same pattern.** DualSQL (2609.18135) trained Qwen3-8B with a stricter single-DB comparator and gained +0.4 EX and +3.0 on its strict metric: a near-null on the headline number.\n- A fair test needs prompts where lucky passes are common (\u00a710).\n- H2 missed its pre-registered threshold (above).\n- With all 11 train templates in SFT, the SFT model solved the GRPO prompts deterministically: only 8 of 2670 rollouts passed the original DB but failed a counterfactual, so B and C would have seen almost the same reward. We held 3 templates out of SFT (`sft_ho`) so the rewards differ; test templates were never touched.\n- vLLM LoRA crashes on Tiny Aya's tied embeddings, so GRPO adapters are merged (fp32 merge, saved fp16) before eval. vLLM sleep mode segfaults on wake-up (vLLM 0.22.1), so it is off.\n\n## 9. Related work and novelty\n\n> To our knowledge, this is the first study to use execution on code-generated, bug-family-targeted counterfactual databases as the online RL reward for a small (~3B) text-to-SQL agent, and to measure that reward's per-family false-positive and false-negative rates, its generalization to a held-out bug family, and answer smuggling. Prior work applied multi-database checks to evaluation (Zhong et al. 2020; Ai 2026) or supervised fine-tuning (SQLoom), or used symbolic verification on a single database (ReViSQL).\n\n**What already exists, and what is new here.** Checking SQL on more than one database is not new. Zhong, Yu & Klein (2020) built distilled test suites of databases for *evaluation*; our E6 reuses them. SpotIt (2510.26840) and VeriEQL search for a database on which two queries differ, also for evaluation. ReViSQL (2603.20004) uses VeriEQL inside RL and finds that 28.8\u201338.4% of positive execution rewards are false, close to our 29.5%; it works at frontier scale on one database and softens the reward to 0.8 rather than refusing it. DualSQL (2609.18135) trains 4B/8B models with a stricter single-DB comparator and sees a near-null (+0.4 EX). The unreviewed repo `devChaudhari7/rl-sql` scores RL rewards on 3 seeded DBs per schema, without an audit. Our delta: (1) copies generated by code per bug family, with no solver, in two kinds (answer-changing and answer-preserving), hidden from the agent and used as a hard RL reward at 3B, in a run where the reward is the only variable; (2) a gold-mutant and hack audit of the reward itself, applied to two public rewards (AgentTune, SkyRL-SQL) and to 8 real systems on Spider; (3) a fix for the AgentTune exploit, with a regression test.\n\n- **Execution-reward false positives:** ReViSQL (2603.20004; VeriEQL refutes 28.8\u201338.4% of positive rewards in a frontier-scale BIRD pilot, soft 0.8 penalty), Graph-Reward-SQL (2505.12380), DualSQL (2609.18135; strict single-DB comparator at 4B/8B).\n- **Multiple or counterfactual DBs:** Pasupat & Liang 2016 (1606.06900; fictitious worlds), Zhong, Yu & Klein 2020 (2010.02840; test suites, evaluation), Ai 2026 (2608.16663; \"counterfactual database instances\", evaluation), SQLoom (2611.03298; counterexample DBs for SFT), Progress-SQL (2606.06825).\n- **Test and data generation:** XData (1411.6704), ParSEval (PVLDB 18(11)), HardTests (2505.24098), RobustTests (2608.24135), CodeContests+ (2506.05817).\n- **Distinguishing DBs and open SQL RL rewards:** SpotIt (2510.26840), SkyRL-SQL (NovaSky-AI, audited in E7). **Why a more accurate reward may not train better:** Razin et al. (2503.15477), Lv et al. (2505.22653).\n- **SQL RL at small scale:** SQL-R1 (2504.08600), Arctic-Text2SQL-R1 (2505.20315), Think2SQL (2504.15077), Reasoning-SQL (2503.23157).\n- **Evaluation and labelling:** \u03c4-bench pass^k (2406.12045), NL2SQL-BUGs taxonomy (2503.11984), benchmark annotation errors (Jin et al., 2601.08778). Library: AgentTune (2610.13335).\n\n## 10. With more credits: a pre-registered test\n\nThe training null says our prompts were too easy, not that the reward is useless. With the shortlist credits we would run the test that can tell, and we fix the prediction now.\n\n- **Curriculum.** Sample SFT 8 times on small-slice (about 50-customer) train items. Keep items where 1 to 7 of the 8 samples pass the original DB and at least 20% of those passes fail a copy.\n- **Runs.** B (`R_exec_rex`) vs C-min (all DBs must pass) vs C-penalty (\u22120.5 on a lucky pass), 3 seeds each, 120 steps, everything else as in \u00a72. Same 331 \u00d7 8 test set.\n- **Prediction.** C-min beats B by **at least 2 pts verified** (mean of 3 seeds), and B's lucky-pass share of training rollouts rises over steps while C-min's does not. If the gap is under 2 pts, or its 3-seed CI includes 0, we report a second null.\n- **Also:** a format-only reward control; per-query latency for the self-hosted agent (we measured batched throughput only); Aya Expanse 8B; a VeriEQL check on passes, for comparison with ReViSQL.\n\n## 11. Limitations\n\n> **Threats to validity**\n> - **One seed** per training condition: the B\u2013C gap (\u22123.7 pts) is significant across eval items but not across runs.\n> - **One model** (Tiny Aya Global 3.35B) and 60 GRPO steps on one L4.\n> - **Synthetic schemas:** one e-commerce schema, 18 templates; real warehouses have messier semantics. E6 (Spider) is the only real-data check, and it uses other systems' predictions, not our model's.\n> - **Gold-informed copies:** our copies are generated from each item's bug families, which know the gold query. A deployable verifier would build them from the schema alone (E8 is therefore an analysis, not a product).\n> - **Comparator:** our strict comparator is stricter than the official Spider script, so E6's absolute EX/TS are not leaderboard numbers.\n\n- F4 is described only in SQL terms: superseded and corrected records.\n- The test generator is less strict than the training reward (54 of 103 base-blind mutants caught).\n\n## 12. Team\n\nShreshth Rai (lead, [@hi-phi-r25i](https://huggingface.co/hi-phi-r25i)), Rishii Kumar Singh ([@LogicPalette](https://huggingface.co/LogicPalette)), Aksh Aggarwal ([@AkshAgg](https://huggingface.co/AkshAgg)).\n", MATH = [["q", false], ["g", false], ["D_0", false], ["D_1,\\dots,D_K", false], ["C_k = \\mathrm{match}\\big(q(D_k),\\, g(D_k)\\big) \\in \\{0,1\\}, \\qquad k = 0,\\dots,K", true], ["R_{\\mathrm{rex}} = C_0, \\qquad R_{\\mathrm{cf}}^{\\mathrm{mean}} = C_0 \\cdot \\frac{1}{K}\\sum_{k=1}^{K} C_k, \\qquad R_{\\mathrm{cf}}^{\\mathrm{strict}} = \\prod_{k=0}^{K} C_k, \\qquad R_{\\mathrm{cf}}^{\\mathrm{soft}} = C_0\\big(1 - 0.2\\cdot\\mathbf{1}[\\exists k \\ge 1 : C_k = 0]\\big)", true], ["\\prod_{k=0}^{4} C_k = 1", false], ["C_0 = 1", false], ["C_k = 0", false]];
50
+ let html = marked.parse(MD, {gfm: true}).replace(/MATHPH(\d+)END/g, (_, i) =>
51
+ katex.renderToString(MATH[i][0], {displayMode: MATH[i][1], throwOnError: false}));
52
+ const el = document.getElementById("body");
53
+ el.innerHTML = html;
54
+ el.querySelectorAll("table").forEach(t => { const w = document.createElement("div"); w.className = "tw"; t.before(w); w.append(t); });
55
+ el.querySelectorAll('a[href^="http"]').forEach(a => { a.target = "_blank"; a.rel = "noopener"; });
56
+ </script>
57
+ </body></html>
results/figure1_reward_audit.png ADDED
results/figure2_blind_rate.png ADDED
results/figure3_training_rewards.png ADDED

Git LFS Details

  • SHA256: 602c1e010ccfd9236c31ff8099172b2aeb760349b7945b533307780c82201d24
  • Pointer size: 131 Bytes
  • Size of remote file: 144 kB