LogicPalette's picture
article page update
fff80f5 verified
Raw History Blame Contribute Delete
77.9 kB
<!doctype html>
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
<title>When Execution Success Lies</title>
<meta name="description" content="Counterfactual-database rewards for a small SQL agent (Lexsi hackathon, Track 3)">
<link href="https://fonts.googleapis.com/css2?family=Tinos:ital,wght@0,400;0,700;1,400;1,700&display=swap" rel="stylesheet">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.5.1/css/all.min.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.16.9/katex.min.css">
<style>
/* paper look: one narrow serif column, justified, booktabs tables. Tinos is metric-compatible with Times New Roman. */
html { background: #fff; }
body { margin: 0; background: #fff; color: #111; font: 18px/1.5 "Times New Roman", Tinos, Times, serif; }
.page { max-width: 740px; margin: 0 auto; padding: 0 16px; }
header.page { text-align: center; padding-top: 3rem; }
header h1 { font-size: 1.75rem; line-height: 1.25; margin: 0 0 1rem; }
.authors { font-size: 1.1rem; margin: 0; }
.venue { font-style: italic; color: #333; margin: .3rem 0 1rem; }
.btn { display: inline-block; margin: .2rem; padding: .15rem .6rem; border: 1px solid #555; border-radius: 3px;
color: #111; font-size: .85rem; text-decoration: none; }
.btn:hover { background: #f0f0f0; }
.note { font-size: .8rem; color: #555; margin-top: .6rem; }
#body { text-align: justify; hyphens: auto; padding-bottom: 2rem; }
#body p, #body li { text-align: justify; }
h2 { font-size: 1.25rem; margin: 2.2rem 0 .6rem; }
h3 { font-size: 1.05rem; margin: 1.5rem 0 .4rem; }
.abstract { margin: 1.5rem 2.5rem 2rem; font-size: .95rem; }
.abstract h2 { text-align: center; font-size: 1rem; font-variant: small-caps; letter-spacing: .05em; margin: 0 0 .5rem; }
a { color: #1a4d8f; }
ul, ol { padding-left: 1.6rem; }
li { margin: .15rem 0; }
figure { margin: 1.5rem 0; text-align: center; }
figure img { max-width: 100%; }
figcaption { font-size: .85rem; text-align: justify; margin-top: .4rem; }
.tw { overflow-x: auto; margin: .6rem 0 1.4rem; }
table { border-collapse: collapse; margin: 0 auto; font-size: .85rem; border-top: 2px solid #000; border-bottom: 2px solid #000; }
thead th { border-bottom: 1px solid #000; }
th, td { padding: .25rem .55rem; text-align: left; vertical-align: top; }
code { font: .85em/1.3 ui-monospace, Menlo, Consolas, monospace; background: #f4f4f4; padding: 0 .15em; overflow-wrap: anywhere; }
pre { background: #f7f7f7; padding: .7rem; overflow-x: auto; }
pre code { background: none; }
.katex { font-size: 1.05em; }
.katex-display { overflow-x: auto; overflow-y: hidden; padding: .3rem 0; }
pre.bib { font-size: .78rem; white-space: pre-wrap; text-align: left; }
@media (max-width: 600px) {
body { font-size: 17px; } header h1 { font-size: 1.4rem; }
#body p, #body li { text-align: left; } .abstract { margin: 1rem 0 1.5rem; }
}
</style></head>
<body>
<header class="page">
<h1>When Execution Success Lies: counterfactual-database rewards for a small SQL agent</h1>
<p class="authors">Shreshth Rai, Rishii Kumar Singh, Aksh Aggarwal (team Teen_khapeetars)</p>
<p class="venue">Lexsi hackathon, Track 3 · problem statements #6 (data and database agents) and #9 (reward design and reward-hacking defences)</p>
<div class="links"><a class="btn" href="https://github.com/RishiiGamer2201/sexsi_hack" target="_blank" rel="noopener"><i class="fab fa-github"></i> Code</a>
<a class="btn" href="https://huggingface.co/spaces/sqlcf-lexsi/counterfactual-sql-demo" target="_blank" rel="noopener"><i class="fas fa-play"></i> Demo</a>
<a class="btn" href="https://huggingface.co/datasets/sqlcf-lexsi/counterfactual-sql-data" target="_blank" rel="noopener"><i class="fas fa-database"></i> Dataset</a>
<a class="btn" href="https://huggingface.co/collections/sqlcf-lexsi/counterfactual-db-rewards-for-a-small-sql-agent-6af174f77dbae0bf0a54c3c4" target="_blank" rel="noopener"><i class="fas fa-cubes"></i> Models</a>
<a class="btn" href="https://huggingface.co/collections/sqlcf-lexsi/counterfactual-sql-data-rollouts-and-traces-6af479430243b7731f85d144" target="_blank" rel="noopener"><i class="fas fa-layer-group"></i> Data collection</a></div>
</header>
<main id="body" class="page"><noscript>This page needs JavaScript; the plain version is <a href="https://github.com/RishiiGamer2201/sexsi_hack/blob/main/ARTICLE.md">ARTICLE.md</a>.</noscript></main>
<div class="page"><h2>BibTeX</h2><pre class="bib">@misc{teenkhapeetars2026counterfactual,
title = {When Execution Success Lies: Counterfactual-Database Rewards for a Small SQL Agent},
author = {Rai, Shreshth and Singh, Rishii Kumar and Aggarwal, Aksh},
year = {2026},
note = {Lexsi Labs AI Safety and Alignment Hackathon, Track 3},
url = {https://github.com/RishiiGamer2201/sexsi_hack}
}</pre></div>
<script src="https://cdnjs.cloudflare.com/ajax/libs/marked/12.0.2/marked.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/KaTeX/0.16.9/katex.min.js"></script>
<script>
const MD = "\n\n**Claim.** Grading an SQL agent's answer on hidden, code-generated copies of the database as well as the original cuts the buggy queries a reward pays from 29.5% to 16.8% and the reward hacks it pays from 54.1% to 1.9%, without rejecting any correct query.\n\n**Tracks and problem statements.**\n- Track 3 (agentic and tool-use): problem statement #6, data and database agents.\n- Track 3: problem statement #9, reward design and reward-hacking defences.\n\n## Abstract\n\nText-to-SQL agents trained with reinforcement learning usually get rewarded when their answer matches the gold answer on one database. A wrong query can match by luck, and then the reward teaches the luck. Our defence is to run the agent's final query on a few extra copies of the database as well, and to pay only if it is right on all of them. The agent never sees these copies, and code builds each one to expose a particular kind of bug.\n\nWe first checked the reward itself, without training anything. On a single database, almost a third of realistic buggy queries get paid. With the copies, about one in six do, and queries that try to game the reward almost never get through. No correct query was rejected. Two public SQL rewards, AgentTune's and SkyRL-SQL's, have the same holes, and we patched AgentTune's. The problem is not specific to our data: on Spider, between 8 and 15 percent of the answers that eight published systems get right fail on Zhong et al.'s test-suite databases.\n\nWe then trained Tiny Aya Global (3.35B) with GRPO, once with each reward. On our own test set the stricter reward did not help. Logging every rollout shows why: the two rewards disagreed on only about 7 percent of them, because our training questions were too easy for lucky answers to come up. The real-world benchmark tells a different story. On Spider, a public set of real databases that we did not build, the model trained with the copies beats the model trained on one database by about two points of test-suite accuracy, and it makes fewer lucky passes. A second pair of runs, trained only on hard questions where lucky answers are common, repeats that two-point gap on Spider; on our own test set it is again a null. Self-hosted, our model costs far less per query than Cohere Command A, but it is much less accurate and does not replace it.\n\n## At a glance\n\n**Base model and method.** Tiny Aya Global (3.35B) with a rank-16 LoRA, SFT on scripted tool-use trajectories, then GRPO with AgentTune, rewarded either on the original database only (B) or on the original plus hidden counterfactual copies (C).\n\n**Headline before \u2192 after.**\n\n| What | Before | After |\n|---|---|---|\n| Buggy queries paid by the reward (n = 1814) | 29.5% (one database) | **16.8%** (with copies) |\n| Reward hacks paid (n = 2839) | 54.1% (one database) | **1.9%** (with copies) |\n| Correct queries rejected (n = 2847) | 0 | **0** |\n| Tiny Aya verified accuracy, our test set | 0.9% (base) \u2192 9.4% (SFT) | **12.7%** (GRPO-B), 9.0% (GRPO-C) |\n| Spider test-suite accuracy, B \u2192 C (two independent pairs of runs) | 27.9%, 27.6% | **29.9%, 29.7%** (+1.9 and +2.1 points) |\n\n## 1. Introduction\n\nReinforcement learning for text-to-SQL usually pays the agent when its query returns the gold result on the database it was given. The check is cheap and hard to fool outright, but it has a blind spot. A query can be wrong and still return the right rows on one particular database, because the data happens not to contain the case that would expose the bug. The reward then pays for the bug, and training pushes the policy towards it.\n\nHere is a case from our own model. Asked *\"How many units of grocery products were in orders that received a payment in 2025 (by payment date)?\"*, it filtered on `order_date` instead of the payment date. On the database it explored, both give 12, so a single-database reward pays it in full. On a copy where payment and order dates differ, the gold answer is 10 and the model still says 12.\n\nOur defence is to grade the final query on a few hidden copies of the database as well as the original. Code generates the copies, one kind of bug at a time, and the agent never sees them. Some copies change the gold answer, so a query that hard-codes a number it saw while exploring stops matching. Others keep the gold answer but add rows on which duplicate and join bugs give a different result.\n\nWe make four contributions:\n\n1. **An audit of the reward itself.** We generate realistic buggy versions of each gold query, plus a set of reward hacks, and measure how often each reward pays them. The copies cut both, and reject no correct query (Section 4).\n2. **The same audit on rewards and systems that are not ours.** AgentTune's built-in SQL reward and SkyRL-SQL's reward have the same holes, and so do the Spider predictions of eight published systems. We patched AgentTune's reward, and its maintainers merged a fix (Section 4).\n3. **A controlled training comparison.** Two GRPO runs of Tiny Aya Global (3.35B) differ only in the reward. On our own test set the stricter reward did not help, and we show why. On Spider, real databases we did not build, the model trained with the copies beat the single-database model on test-suite accuracy and made fewer lucky passes (Section 5). A second pair of runs on hard questions repeated the Spider gap (Section 8).\n4. **A cost comparison.** We compare a small self-hosted agent with Cohere Command A, per query and per verified answer (Section 6).\n\n**Why it matters.** SQL agents are being put in front of business data: analysts ask in plain language and act on the number they get back. A query that fails is visible; a query that runs and returns a plausible but wrong number is not, and it ends up in a report. If the training reward pays for such queries whenever the data happens to hide the bug, RL makes exactly the failure that is hardest to notice more likely.\n\nThe reward audit is our main result. We report the training results as they came out, including the ones that did not go our way.\n\n## 2. Related work\n\nChecking SQL on more than one database is not new. Zhong, Yu & Klein (2020) built distilled test suites of databases to *evaluate* text-to-SQL on Spider (Yu et al., 2018), and we reuse them in Sections 4.4 and 5.4. SpotIt (Klopfenstein et al., 2025) and VeriEQL (He et al., 2024) search for a database on which two queries differ, also for evaluation. ReViSQL (Zhu et al., 2026) uses VeriEQL inside RL and finds that 28.8\u201338.4% of positive execution rewards are false, close to our own estimate. It works at frontier scale on one database and softens the reward to 0.8 rather than refusing it. DualSQL (Chen et al., 2026) trains 4B and 8B models; its ablation with a stricter single-database comparator, run on the 8B model, sees almost no change on execution accuracy. Two concurrent projects are closest to ours. Mirage (SiddharthaRaja, 2026), an unpublished repository, trains a 0.5B coder model with GRPO against a single database or against perturbed copies picked to reject near-miss queries. It reports that one database accepts 33.2% of near-miss SQL and that the copies did not improve Spider accuracy (\u22121.2 points). RewardShield (shro-2002, 2026), another entry to this hackathon, uses Tiny Aya and AgentTune with mutant-targeted databases as a verifier for filtering SFT data, and also finds no downstream gain. A study of leaky unit tests as a code-RL reward (Zhang, 2026c) and SynSQL (Habibollah & Rafiei, 2026), which re-grades text-to-SQL systems on synthesized databases, point the same way: one check pays for bugs, and fixing the check does not by itself train a better model.\n\nWhat we add:\n- copies generated by code for each bug family, with no solver, in two kinds (answer-changing and answer-preserving), hidden from the agent and used as a hard RL reward for a 3B agent, in a run where the reward is the only variable;\n- an audit of the reward against buggy gold queries and hacks, applied to two public rewards and to eight real systems on Spider;\n- a fix for the AgentTune exploit, with a regression test.\n\nEarlier and concurrent work checks SQL on extra databases mainly for evaluation (Zhong et al., 2020; Ai, 2026; SynSQL) or for cleaning SFT data (SQLoom; Fu et al., 2026; RewardShield), uses a solver inside the RL reward on one database (ReViSQL), or uses generic perturbed copies as a GRPO reward for a 0.5B model (Mirage). What is new here is the copies themselves, generated per bug family in both an answer-changing and an answer-preserving kind; the per-family measurement of how often that reward pays buggy queries, hacks and correct queries; and its test as an RL reward for a 3B tool-using agent, alongside two public rewards and eight published systems. Like Mirage and Zhang (2026c), we find that the stricter reward did not train a better model on our own test set.\n\nOther related lines:\n- **Execution-reward false positives:** ReViSQL (Zhu et al., 2026), Graph-Reward-SQL (Weng et al., 2025), DualSQL (Chen et al., 2026).\n- **Multiple or counterfactual databases:** fictitious worlds (Pasupat & Liang, 2016); test suites (Zhong et al., 2020); counterfactual database instances for evaluation (Ai, 2026); counterexample databases for SFT (SQLoom; Fu et al., 2026); synthesized databases for evaluation (SynSQL); perturbed copies as a GRPO reward (Mirage).\n- **Test and data generation:** XData (Chandra et al., 2014), ParSEval (2025), HardTests (He et al., 2025), RobustTests (Zhang et al., 2026b), CodeContests+ (Wang et al., 2025).\n- **Why a more accurate reward may not train better:** Razin et al. (2025), Lv et al. (2025) and Zhang (2026c).\n- **SQL RL at small scale:** Progress-SQL (Zhang et al., 2026a; test suites used for evaluation only), SQL-R1 (Ma et al., 2025), Arctic-Text2SQL-R1 (Yao et al., 2025), Think2SQL (Papicchio et al., 2025), Reasoning-SQL (Pourreza et al., 2025).\n- **Evaluation and labelling:** pass^k from \u03c4-bench (Yao et al., 2024), the NL2SQL-BUGs error taxonomy (Liu et al., 2025), benchmark annotation errors (Jin et al., 2026), BIRD (Li et al., 2023).\n\n## 3. Method\n\n### 3.1 Data and counterfactual copies\n\nWe use two kinds of data. Training, the reward audit and our own test set are synthetic: an e-commerce schema (customers, orders, order items, payments, refunds, invoices) from a seeded generator, with no personal data. Questions come from templates. Each template belongs to one of four bug families:\n\n- **F1** duplicates and fan-out joins;\n- **F2** status filters (cancelled or placed rows);\n- **F3** refunds and net amounts;\n- **F4** superseded and corrected records. F4 is held out: no training reward ever uses it.\n\nEach item gets at least one *answer-changing* copy, where the gold result changes so that a smuggled literal cannot follow it. It also gets at least one *answer-preserving* copy, where the gold result stays the same but fan-out and duplicate bugs give a different answer. Code generates the copies per bug family; we never build them by hand.\n\nThe second kind is real: Spider (Yu et al., 2018), 20 databases and 1034 dev questions written by people, with the test-suite databases of Zhong et al. (2020). We did not build or change any of it. We use it twice: to score eight published systems (Section 4.4) and to test our own trained models (Section 5.4).\n\n| Split | Items | Use |\n|---|---|---|\n| audit | 1302 | reward audit (Section 4) |\n| train | 907 | SFT (353 trajectories, 3 templates held out) and GRPO prompts (445) |\n| test | 331 (214 in held-out F4) | evaluation, from a different generator |\n\nTest items by family: F1 36, F2 36, F3 45, F4 214.\n\n| Source | Kind | Size | Licence | Use |\n|---|---|---|---|---|\n| Our generator (`sqlcf`) | synthetic e-commerce schema, template questions, SQLite databases and copies | 2540 items | CC-BY-NC-4.0 (released) | audit, training, test |\n| Spider dev (Yu et al., 2018) + test-suite databases (Zhong et al., 2020) | real databases, human-written questions | 1034 questions, 20 databases | CC BY-SA 4.0 (not redistributed) | out-of-distribution test only |\n\n**Languages.** All questions are in English; the agent writes SQLite. Tiny Aya Global is multilingual, but we did not test other languages.\n\n**Leak check.** No test question matches a train question, exactly, up to numbers, or as a near duplicate (Jaccard \u2265 0.8 on word 3-grams). No test gold query matches a train gold query exactly. 36 test gold queries are near duplicates of a train one: they come from a test template that counts distinct customers where a train template counts distinct products, with different wording. The audit split shares templates with both and is never trained on.\n\n### 3.2 Rewards\n\nFor a predicted query MATHPH0END, gold query MATHPH1END, original database MATHPH2END and hidden copies MATHPH3END, let `match` be a strict multiset comparison of result rows. It is ordered when gold is ordered, columns may be permuted, and floats are rounded to 4 decimal places.\n\nMATHPH4END\n\nMATHPH5END\n\nWe compare these with `R_exec_set`, the BIRD-style set match on one database (Li et al., 2023). MATHPH6END follows ReViSQL's softened penalty. We froze the training reward (MATHPH7END) before training at gate G1. The gate chose by the fewest correct queries rejected, then by the most GRPO groups whose rewards were not all equal.\n\n### 3.3 Agent and training\n\nThe policy is Tiny Aya Global (3.35B; Salamanca et al., 2026) with a LoRA adapter of rank 16 (Hu et al., 2021). The agent explores the database through a `run_sql` tool for up to 5 steps, then gives a final query.\n\n**SFT.** First, SFT on scripted oracle trajectories (list tables, describe, explore, final SQL). We built the SFT text with AgentTune's own constants, so that it matches what the model sees in GRPO rollouts. We held 3 of the 11 train templates out of SFT, because with all 11 the SFT model solved the GRPO prompts every time and the two rewards would have agreed almost everywhere (Appendix C).\n\n**GRPO.** From the same SFT adapter we trained two GRPO runs (Shao et al., 2024) with AgentTune (Lyngkhoi et al., 2026):\n- **B** uses the single-database reward MATHPH8END;\n- **C** uses the counterfactual reward MATHPH9END.\n\nBoth use the same data and seed, 60 steps, 6 rollouts per prompt, 4 prompts per step, learning rate 1e-5 and KL weight 0.02. Only the reward differs. Training ran on one L4 GPU and took about 3.5 hours per run.\n\n**Hyperparameters.** Everything not listed is the TRL default as wrapped by AgentTune (`d4db4c1`).\n\n| Setting | SFT | GRPO (B, C) |\n|---|---|---|\n| Base weights | `CohereLabs/tiny-aya-global`, bf16 | same, continuing the SFT adapter |\n| LoRA | r 16, alpha 32, dropout 0.05, on q, k, v, o, gate, up and down projections | same adapter |\n| Learning rate | 1e-4 | 1e-5 |\n| Batch | effective 16 (1 per device \u00d7 16 accumulation steps) | 6 rollouts \u00d7 4 prompts = 24 sequences per optimizer step |\n| Length | max 4096 tokens; loss on assistant turns only | max 2048 completion tokens (768 in Section 8) |\n| Schedule | 2 epochs over 353 trajectories | 60 steps |\n| Other | gradient checkpointing | KL weight \u03b2 0.02, rollout temperature 1.0, vLLM colocated (45% of GPU memory), no importance-sampling correction |\n| Seed | 0 | 0 |\n\n**Environment.** The agent has one tool, `run_sql(database, sql_command)`. It runs a single read-only SQLite query on the item's original database, which the question names by id; the copies are reachable only by the reward. It returns up to 20 rows and 1500 characters, or `ERROR: <message>` for SQL that fails. Tiny Aya's chat template ignores tool schemas, so AgentTune appends its fallback tool prompt to the system turn, the model writes tool calls as a JSON list, and tool results come back folded into user turns. The episode ends when the model replies with a sentence and a final query in an `sql` block. The full system prompt is in Appendix D, and a sample trajectory is in Section 5.2.\n\n**AgentTune fix.** While doing this we found and patched a bug in AgentTune's multi-turn stream. It dropped chat-template tokens between turns, so the trainer scored tokens in a context vLLM never saw.\n\n### 3.4 Evaluation\n\nThe test set comes from a **different generator**: a foreign-key-closed sub-database of a larger parent, plus 4 mutated copies that know nothing about the bug families. We sample each of the 331 items 8 times at temperature 0.7. We use these terms throughout:\n- an answer is **verified** if it matches on the original and all 4 copies;\n- it is a **lucky pass** if it matches on the original but fails a copy;\n- it is a **silent error** if the agent claims success, all its tool calls succeeded, and the answer is wrong. This follows the claimed-versus-verified framing of AuditKIT (Lexsi Labs, 2026).\n\nWe also report pass^k (all of k samples verified).\n\n**Statistics.** Intervals are Wilson 95% intervals. Comparisons between models use a paired bootstrap over items and an exact McNemar test.\n\n**Pre-registered hypotheses:**\n- H1: the single-database reward pays buggy gold-mutants at a material rate, and the rate depends on database size (descriptive; Section 4.2).\n- H2: the copies cut the share of buggy queries paid by at least half.\n- H3: AgentTune's SQL reward can be exploited.\n- H4: C makes fewer silent errors than B.\n- H5: C does better than B on the held-out family F4.\n- H6: C removes literal smuggling at test time.\n- H7: on hard prompts where lucky passes are common, C-min beats B by at least 2 points verified (Section 8).\n\n### 3.5 Hackathon kits and how they connect\n\n- **AgentTune** (`d4db4c1`, Lyngkhoi et al., 2026). We used `create_agentic_trainer(\"grpo\", backend=\"trl\")` for multi-turn GRPO with tool calls, its tool-prompt fallback for models whose chat template has no tool support, and its render, parse and fold functions to build SFT text identical to what rollouts produce. We fixed a bug in its multi-turn token stream (Section 3.3) and audited and patched its built-in SQL reward (Section 4.3; merged upstream as PR #12).\n- **AuditKIT** (`9da5838`, Lexsi Labs, 2026). We used its agent-evaluation framing: claimed versus verified success, silent errors and pass^k. We reimplemented the statistics in `evaluate.py` so that they run on our trajectory format.\n- **Not used:** CuratorKIT, AlignTune, SafeTune and CircuitKIT. Our data is generated by code, so there was nothing to curate, and the work does not touch safety tuning or interpretability.\n\n### 3.6 Metrics\n\n| Metric | Better | Why we use it |\n|---|---|---|\n| Buggy gold-mutants paid by a reward | lower | how often the reward pays a realistic wrong query (reward false positives) |\n| Reward hacks paid | lower | whether a policy can earn reward without solving the task |\n| Correct queries rejected | lower | a stricter reward must not punish right answers (reward false negatives) |\n| Verified accuracy | higher | task success: right on the original and all 4 copies of a test database |\n| Single-database accuracy | higher | the usual execution accuracy, reported so the gap to verified is visible |\n| Lucky pass | lower | right on the original only; what a single-database reward wrongly pays |\n| Silent error | lower | wrong answer with every tool call succeeding and a confident reply: the failure a user cannot see |\n| pass^k | higher | all k samples verified; reliability, not one lucky draw |\n| Tool calls per trajectory, invalid tool calls | invalid lower | whether the agent uses its tool correctly |\n| Spider EX / test-suite accuracy (TS) | higher | transfer to real databases; TS is a check built by others |\n| $ per 1k queries, $ per 1k verified answers | lower | cost against a closed model |\n\n**What must not get worse.** Correct queries rejected stays at 0 for every reward we compare. No LLM or classifier judges any output: every score is deterministic SQL execution against gold. We checked the grader by hand on 48 of our wrong test answers and 40 Spider lucky passes (Sections 4.4 and 5.2).\n\n### 3.7 Threat model\n\nThe adversary is the policy under optimisation. It sees the question, the schema and the original database, can run any read-only query on it, and is paid by the reward. It cannot see or query the copies, and it cannot change the grader. In scope: queries that are right on the original database by luck, hard-coded answers seen while exploring (literal smuggling), hard-coded row IDs, comparator holes such as duplicate rows that a set comparison ignores, and format exploits of the reward (AgentTune's `WHERE 1 = 1` plus a fixed answer). Out of scope: tampering with the grading code or the databases, and bugs in the gold queries themselves.\n\n## 4. Auditing the reward\n\nThis section needs no GPU. On the 1302 audit items we generate buggy versions of each gold query (\"gold-mutants\"), typical reward hacks, and correct queries (gold and paraphrases). We then count how often each reward pays them in full.\n\n### 4.1 Buggy queries and hacks\n\n![Figure 1: per bug family F1\u2013F4, share of gold-mutants and of hacks each reward fully pays (false positives), and share of correct SQL it does not fully pay (false negatives, all zero)](results/figure1_reward_audit.png)\n\n**Table 1.** How often each reward pays buggy queries, hacks and correct queries (95% CI).\n\n| Reward | Buggy gold-mutants paid (n = 1814) | Hacks paid (n = 2839) | Correct queries rejected (n = 2847) |\n|---|---|---|---|\n| `R_exec_set` (BIRD set match, one DB) | 37.5% [35.3, 39.7] | 54.1% [52.3, 56.0] | 0 |\n| `R_exec_rex` (strict multiset, one DB) | 29.5% [27.4, 31.6] | 54.1% | 0 |\n| **Counterfactual**, copies chosen blind to the mutants | **16.8%** [15.2, 18.6] | **1.9%** [1.5, 2.5] | **0** |\n| Counterfactual, copies chosen to kill these mutants (optimistic bound) | 1.5% [1.0, 2.2] | 2.0% | 0 |\n\nOn one database, almost a third of the buggy queries get paid. With copies chosen without looking at the mutants, about one in six do. That is a 43% relative cut, short of the 50% we pre-registered (**H2 not met**). Hacks almost disappear. The hacks that still get paid all hard-code row IDs, and they survive only when the answer-changing copy happens to keep the same IDs. Appendix B lists every hack we tried.\n\n### 4.2 When are bugs invisible on one database?\n\n![Figure 2: share of gold-mutants that pass the single-DB check, by number of customers in the DB and bug family](results/figure2_blind_rate.png)\n\nHow often a bug hides depends mostly on how small the question's slice of the data is (one city, one year, one category). It depends much less on the size of the database. With 50 customers, 29.2% of mutants pass the single-database check; with 500, under 1% do. Small slices are common in real questions, so this is not an artefact of tiny databases.\n\n### 4.3 Two public rewards\n\n**AgentTune's built-in SQL reward (H3).** Unlike the other rewards here, it does not compare query results: it scores the query text with heuristics (penalties for many queries, `LIMIT 1` or no `WHERE`) and checks a yes/no answer. On 360 yes/no items, a query padded with `WHERE 1 = 1` plus an answer that is always \"yes\" earns the same score as honest SQL on every yes item (3.10). We wrote a patch that brings the exploit down to 2.20, with a reproduction script and a regression test. We reported it upstream as [Lexsi-Labs/AgentTune#11](https://github.com/Lexsi-Labs/AgentTune/issues/11). The maintainers merged a fix the next day ([#12](https://github.com/Lexsi-Labs/AgentTune/pull/12)): the WHERE check and an opt-in result-matching reward, both as we proposed, with a regression test.\n\n**SkyRL-SQL's reward** (NovaSky-AI, 2025) runs the predicted and gold SQL on one database and pays 1 if the two *sets* of rows are equal. Re-implemented from its source and run on the same candidates, it has two holes of about the same size:\n- **The database hole.** Adding copies closes this one.\n- **The comparator hole.** A strict comparator closes this one, and nearly all of it is a dropped `DISTINCT`.\n\nWith both fixes it pays exactly what our reward pays (Table B3).\n\n### 4.4 Published systems on Spider\n\nDo real systems produce lucky passes, or only our synthetic mutants? We took the public Spider-dev predictions of 8 systems (from the ESM+ repository) and scored each on the original Spider databases and on the test-suite databases of Zhong et al. Between 8.1% and 14.9% of each system's correct-looking answers fail a test-suite database. The highest rates belong to the prompted LLMs (Claude 14.9%, C3 14.1%).\n\nWe checked 40 of these lucky cases by hand. 27 return a different number of rows on the suite database and 13 return different values. Typical causes:\n- `GROUP BY name` where names repeat (gold groups by ID);\n- `EXCEPT` versus `NOT IN` with duplicates;\n- `ORDER BY ... LIMIT 1` with ties;\n- a `LEFT JOIN` that adds zero-count rows.\n\nMany are real bugs. Some are ties or ambiguous gold, so these rates are upper bounds. Full table: Appendix B.\n\n## 5. Training with the reward\n\n### 5.1 What the training reward paid for\n\n![Figure 3: per GRPO step for runs B and C, share of rollouts passing the original DB (left) and share of those that fail a counterfactual, i.e. lucky passes the single-DB reward pays (right)](results/figure3_training_rewards.png)\n\nWe logged every one of the 1440 rollouts in each run, with its result on every database:\n- **The prompts were easy.** About 86% of rollouts passed the original database, in both runs.\n- **Few passes were lucky.** About 8% of those passes failed a copy.\n- **So the rewards rarely disagreed.** B and C gave a different reward on only about 7% of rollouts. Even there, C gave partial credit rather than zero.\n- **Lucky passes fell in both runs.** The single-database reward was not drifting towards lucky queries in the first place.\n\n### 5.2 Results on our test set\n\n**Table 2.** Test set, 331 items \u00d7 8 samples. Verified means right on the original and all 4 copies.\n\n| Model | Verified [95% CI] | Lucky pass | Silent error | pass^4 | Held-out F4 verified |\n|---|---|---|---|---|---|\n| Tiny Aya base | 0.9% | 0.5% | 4.5% | 0.0% | 0.6% |\n| SFT (init of B and C) | 9.4% [8.4, 10.6] | 5.1% | 65.4% | 2.1% | 9.2% |\n| **GRPO-B** (single-DB reward) | **12.7%** [11.5, 14.0] | 5.1% | 64.0% | 3.9% | 9.9% |\n| GRPO-C (counterfactual reward) | 9.0% [7.9, 10.1] | 5.1% | 61.2% | 2.2% | 8.8% |\n| Command A (1 sample, no tools) | 81.0% [76.4, 84.8] | 1.8% | n/a | n/a | 94.9% |\n\n**Trajectory metrics** (same runs; task success is the verified column above).\n\n| Model | Tool calls per trajectory | Invalid tool calls | Throughput / latency |\n|---|---|---|---|\n| Tiny Aya base | 1.44 | 63.7% | n/a |\n| SFT | 2.11 | 17.2% | 3.8 trajectories/s on one L4 (vLLM, batched) |\n| GRPO-B | 2.11 | 16.2% | 3.4 trajectories/s on one L4 (measured) |\n| GRPO-C | 2.10 | 19.5% | 4.0 trajectories/s on one L4 |\n| Command A | 0 (single turn) | n/a | median 9.7 s, p90 42.2 s per query (API) |\n\nAn invalid tool call is one whose tool response is an error (bad arguments or SQL that fails). SFT and C throughput are RTX 5070 Ti times scaled by B's measured L4-to-5070 Ti ratio; they are batched rates, not per-query latency.\n\n**By family** (verified, 8 samples per item):\n\n| Model | F1 (36 items) | F2 (36) | F3 (45) | F4, held out (214) |\n|---|---|---|---|---|\n| Tiny Aya base | 5.2% | 0.0% | 0.0% | 0.6% |\n| SFT | 30.9% | 1.0% | 0.0% | 9.2% |\n| GRPO-B | 56.6% | 1.0% | 0.0% | 9.9% |\n| GRPO-C | 28.8% | 1.4% | 0.0% | 8.8% |\n| Command A (1 sample) | 100% | 0.0% | 64.4% | 94.9% |\n\nNearly all of B's gain over SFT is on F1. No Tiny Aya model solves F3 (net amounts after refunds) or F2 on this test set.\n\n**A sample trajectory** (GRPO-B, test item `F1_category_buyers|books|2024|S_s501`, verified). Question: *\"How many distinct customers bought a books product in 2024? Count orders of any status.\"*\n\n```\nassistant [{\"tool_name\": \"run_sql\", \"parameters\": {\"database\": \"S_s501\",\n \"sql_command\": \"SELECT status, COUNT(*) FROM orders GROUP BY status\"}}]\ntool cancelled | 20\n placed | 24\n shipped | 65\n (3 rows)\nassistant [{\"tool_name\": \"run_sql\", \"parameters\": {\"database\": \"S_s501\",\n \"sql_command\": \"SELECT COUNT(DISTINCT o.customer_id) FROM order_items oi JOIN orders o ON o.order_id = oi.order_id JOIN products p ON p.product_id = oi.product_id WHERE p.category = 'books' AND substr(o.order_date, 1, 4) = '2024'\"}}]\ntool 19\n (1 rows)\nassistant Here is the query that answers the question.\n SELECT COUNT(DISTINCT o.customer_id) FROM order_items oi JOIN orders o ... WHERE p.category = 'books' AND substr(o.order_date, 1, 4) = '2024'\n```\n\n**B and SFT.** B improves on SFT by 3.2 points [1.8, 4.8] (McNemar p = 0.006), and its lucky-pass rate does not change.\n\n**C.** C does not improve on SFT, and it is 3.7 points below B. The pre-registered hypotheses all fail:\n- **H4:** C's silent-error rate is lower, but McNemar on the first sample gives p = 0.61, and C also gets fewer answers right.\n- **H5:** no model gains on the held-out family F4.\n- **H6:** no trained model smuggles a literal at test time (0 of 2648 trajectories each), so there was nothing for C to remove.\n\n**What the errors are.** We hand-labelled 48 wrong test trajectories, 16 per model, using the NL2SQL-BUGs categories. Every lucky pass is an honest semantic bug that happens to give the right number on one database, such as an extra or missing filter, a wrong column, a missing table or the wrong aggregate. None is reward hacking. The most common error is filtering on the order date when the question says \"by payment date\". Details: Appendix B.\n\n### 5.3 Why the comparison came out null\n\nThe gap between B and C is real for these two adapters. With one training seed each, though, we cannot tell how much of it is run-to-run variance. Three things explain why we expected little difference:\n\n1. **C really was stricter.** Replaying all training rollouts, C's reward was lower than B's on about 7% of them because they passed the original database by luck. It was higher on almost none.\n2. **The training prompts were too easy for that to matter.** How fast RL learns depends on how much the reward varies under the policy, not on how accurate the reward is (Razin et al., 2025). RLVR also tolerates far more reward noise than 7% (Lv et al., 2025).\n3. **Others see the same pattern.** DualSQL trained Qwen3-8B with a stricter comparator and saw almost no change on its headline metric.\n\nA fair test needs prompts where lucky passes are common. Section 8 runs one, with the prediction fixed in advance.\n\n### 5.4 Transfer to a real benchmark: Spider\n\n**Setup.** Section 4.4 used other systems' predictions. Here we ran our own four models on all 1034 Spider-dev questions (20 real databases) and graded them the same way:\n- **EX:** the answer matches gold on the original Spider database.\n- **TS:** it also matches on every test-suite database of Zhong et al. Those copies were built by someone else, for evaluation.\n\nThe agent loop, tool and sampling are the same as on our test set: up to 5 `run_sql` steps and 4 samples per question at temperature 0.7. The system prompt is our training prompt with the question's Spider schema in place of the retail schema. Our models only ever saw one retail schema, so this is a transfer test of the reward, not a Spider score. The run took 36 minutes on a local RTX 5070 Ti.\n\n**Table 3.** Spider dev, 1034 questions \u00d7 4 samples.\n\n| Model | EX [95% CI] | TS [95% CI] | Lucky share of EX passes |\n|---|---|---|---|\n| Tiny Aya base | 15.4% [14.4, 16.6] | 12.6% [11.6, 13.7] | 18.3% [15.5, 21.5] |\n| SFT | 38.5% [37.1, 40.0] | 29.1% [27.8, 30.5] | 24.4% [22.4, 26.6] |\n| GRPO-B (single-DB reward) | 38.0% [36.5, 39.5] | 27.9% [26.6, 29.3] | 26.4% [24.3, 28.7] |\n| **GRPO-C** (counterfactual reward) | 39.1% [37.6, 40.6] | **29.9%** [28.5, 31.3] | **23.6%** [21.6, 25.7] |\n\nThe intervals in the table treat the 4 samples of a question as independent, so they are a little too narrow. The comparisons below use a paired bootstrap over questions.\n\n- **SFT transfers, mostly through format.** SFT more than doubles test-suite accuracy over the base model. Nearly all SFT answers end with a final query, against 41% for the base model. Among answers that do end with one, the two are about equally accurate.\n- **On Spider, C comes out ahead of B.** On our own test set C was below B. On Spider, C's test-suite accuracy is 1.9 points higher [+0.6, +3.3], while the gap on EX is smaller and not significant. C also has fewer lucky passes than B. So C's advantage is in answers that hold up on the extra databases, which is what a counterfactual reward should teach. B shows the opposite: compared with SFT it loses a little test-suite accuracy and gains lucky passes.\n- **This is a hint, not proof.**\n - Each reward has one training run, and we did not pre-register this comparison.\n - An exact McNemar test on the first sample alone gives p = 0.14.\n - C is not significantly better than SFT. Much of the gap comes from B getting worse.\n - A second seed of this pair finished training but could not be evaluated before the deadline. Section 8 is a different, independent pair of runs.\n- **Small models are luckier.** Between 18% and 26% of our models' EX passes fail a test-suite database, more than for any of the eight published systems. The base model is already at 18%, so much of this comes with model size. Training on the single-database reward raises it the most.\n\n## 6. Cost against Command A\n\nWe compared against Cohere Command A (`command-a-03-2025`; Team Cohere, 2025). It answers each test item once, in a single turn, with the schema and column definitions in the prompt and no tools. Its cost is billed tokens at list price; our logged token counts match the Cohere dashboard within 1%. Tiny Aya's cost is our L4's hourly price times the measured generation time.\n\n| Model | Verified | $ per 1k queries | $ per 1k verified-correct answers |\n|---|---|---|---|\n| Command A | 81.0% | $4.24 | $5.24 |\n| Tiny Aya GRPO-B | 12.7% | $0.061 | $0.48 |\n\nThe small model is about 70 times cheaper per query and about 11 times cheaper per verified answer. At its accuracy, though, it is not a usable replacement. What counterfactual rewards offer is a cheap way to make a small model's training signal trustworthy; they cannot close the gap to a frontier model in 60 GRPO steps.\n\nCommand A falls into the fan-out trap too. It gets none of the 36 items of one F2 template right. It joins `payments` directly, so an order with two payments counts its units twice, and it adds a `status != 'cancelled'` filter the question does not ask for. The single-database check rejects 32 of these 36 answers; the copies reject the other 4, which match on the original database by luck.\n\n**Compute used.** The L4 is a Google Cloud `g2-standard-4` VM at $0.74 per hour; the RTX 5070 Ti is a local workstation, so its runs cost nothing extra.\n\n| Stage | Hardware | Time | Cost |\n|---|---|---|---|\n| Data, copies, reward audit, scoring, `verify_all.py` | CPU | minutes | $0 |\n| SFT (353 trajectories, 2 epochs) | RTX 5070 Ti | 72 min | $0 |\n| GRPO-B and GRPO-C, 60 steps each | L4 | about 7 h | about $5 |\n| Curriculum GRPO-B and GRPO-C-min (Section 8) | L4 | about 8.5 h | about $6 |\n| Test-set generation, 331 \u00d7 8 per model | RTX 5070 Ti (B also on L4: 785 s) | about 13 min per model | $0 |\n| Spider transfer, 4 models | RTX 5070 Ti | 36 min | $0 |\n| DPO runs (Section 7) | RTX 5070 Ti | short | $0 |\n| Command A, 331 test items | Cohere trial API | about 1 h | $0 (list price $1.40) |\n\nThe total paid compute for the runs in this article is about $11, plus earlier attempts that crashed or were superseded.\n\n## 7. Other things we tried\n\nAll of these are reported in full in Appendix B. Most are nulls.\n\n- **Voting without gold (self-consistency).** Picking each item's most common answer out of 8 samples helps every model; it raises B to 16.9%. Voting on all 5 databases adds nothing over voting on one, because at this accuracy wrong answers rarely agree with each other anyway.\n- **DPO from one checkpoint.** We labelled one fixed set of SFT rollouts two ways, by the single-database verifier and by the copies, and ran DPO (Rafailov et al., 2023) on each. Neither beat SFT, and lucky passes did not move. Only a small share of the counterfactual pairs actually teach \"a lucky answer is worse\", for the same reason as in Section 5.3.\n- **Partial credit.** Giving each database a score between 0 and 1 gives GRPO more groups with some signal. But it pays buggy queries more than twice as much as the binary reward, and it pays a hard-coded number that is close to the answer. It ranks queries no better, so we did not train with it. The same analysis shows that our C reward still put lucky rollouts above their group's mean more than half the time, because getting 1 of 2 copies right still earns half. The strict version never does.\n- **Period-edge copies.** Our copies edit cities, statuses and tiers, not dates. One extra copy that moves orders to the edges of the asked year catches about two thirds of date bugs, with no correct query rejected. Our models rarely make these bugs, though, so it would change our accuracy numbers by under 1 point. A verifier needs the bug families the policy actually produces.\n\n## 8. A curriculum of hard questions\n\nSection 5.3 blamed easy prompts. So we built a training set where lucky passes are common, and fixed the prediction before training.\n\n- **Curriculum.** We sampled the SFT model 8 times on each of the 445 GRPO training items. We kept the items where some but not all samples pass the original database, and at least 20% of those passes fail a copy. That leaves 36 items from three templates; 89% of the SFT model's single-database passes on them are lucky.\n- **Runs.** GRPO-B (single-database reward) against GRPO-C-min, whose reward pays only when the query is right on the original and all copies. Both start from the same SFT adapter, with the same settings as Section 3.3 except one: completions are capped at 768 tokens, because longer ones ran out of memory on the 24 GB L4. The cap cuts 0.9% of the SFT model's trajectories on these items. For time, we ran one seed and 60 steps per arm, and dropped a third arm with a penalty for lucky passes.\n- **Prediction.** C-min beats B by at least 2 points verified on our test set, and B's share of lucky passes rises during training while C-min's does not. Otherwise we report a second null.\n\n**Table 4.** Curriculum runs. Training rows cover steps 11 to 60 (960 rollouts per run). Test set as in Table 2; Spider as in Section 5.4.\n\n| | GRPO-B, curriculum | GRPO-C-min, curriculum | C-min minus B [95% CI] |\n|---|---|---|---|\n| Training: rollouts passing the original database | 87.5% | 77.9% | |\n| Training: rollouts passing all databases | 23.6% | 31.8% | |\n| Training: lucky share of original-database passes | 73.0% | 59.1% | |\n| Test set: verified (SFT: 9.4%) | 7.1% | 6.3% | \u22120.8 pts [\u22121.8, +0.2] |\n| Test set: silent error | 63.7% | 56.5% | |\n| Spider: test-suite accuracy (SFT: 29.1%) | 27.6% | 29.7% | **+2.1 pts [+0.9, +3.4]** |\n| Spider: lucky share of passes | 28.3% | 25.3% | lucky rate \u22120.9 pts [\u22121.9, +0.1] |\n\n**The prediction failed on our test set.** C-min is not ahead of B, so this is a second null there. Both curriculum models fell below the SFT model they started from: 60 steps on 36 items from three templates is narrow training.\n\n**The rewards did pull apart this time.** B learned to pass the database it was paid on, but about three in four of those passes stayed lucky, from 84% in the first 10 steps. Its lucky share did not rise, as we had predicted; it just stayed high. Under C-min the lucky share fell from 78% to 59%, and more rollouts were right on every database.\n\n**On Spider the gap came back.** C-min beats B by 2.1 points of test-suite accuracy, with fewer lucky passes. The main runs gave 1.9 points (Section 5.4). This pair used different prompts and a stricter reward, so it is a second, independent sample of the same comparison, though each pair is still one run per reward. The curriculum B is the only one of our models below SFT on Spider (\u22121.6 points [\u22122.8, \u22120.3]), and it has the highest lucky share of any of them.\n\n## 9. Limitations\n\n- **One synthetic schema.** Training and our test set use one generated e-commerce schema. Spider is the only real data, and our models see it only at test time.\n- **Few training runs.** Each training comparison rests on one run per reward (in each of two setups, main and curriculum), and the Spider comparison of the main pair was not pre-registered. The curriculum prediction was pre-registered, and it failed on our test set.\n- **Short, small training.** 60 GRPO steps of a rank-16 LoRA on a 3.35B model. Absolute accuracy stays low (12.7% verified against 81.0% for Command A), so these are statements about the reward, not about a usable SQL agent.\n- **The copies only cover bugs we wrote down.** Copies chosen blind to the mutants still pay 16.8% of them, and our copies do not edit dates (Section 7).\n- **Upper bounds and small hand checks.** Lucky-pass rates on Spider include ties and ambiguous gold queries, so they are upper bounds. The hand audits cover 48 of our trajectories and 40 Spider cases.\n- **A one-shot baseline.** Command A answers once, in one turn, without tools; our agent gets 5 tool steps and is scored over 8 samples.\n\n## 10. With more compute\n\n| Next step | Why | Expected result |\n|---|---|---|\n| 3\u20135 seeds per reward, main and curriculum setups | Separate the reward's effect from run-to-run variance | A seed-averaged Spider gap of about +2 test-suite points for the copies, with an interval that excludes zero if the effect is real |\n| Generate copies for Spider and BIRD training databases, and train on them | Our copy generator knows one schema; real schemas give real, varied lucky passes | Fewer lucky passes on Spider (from about 24% of EX passes towards the 8\u201315% of published systems) and a test-suite gain over the single-database reward of 2\u20134 points |\n| 300+ GRPO steps, an 8B-class policy | Our runs are short and small | Higher absolute accuracy for both rewards; a larger B-vs-C gap, because a stronger policy finds more lucky passes for B to be paid for |\n| Copies for dates, and for the bug families the policy actually makes | Close the 16.8% blind spot found in Section 4.1 and Section 7 | Buggy queries paid falls from 16.8% towards the 1.5% bound of copies chosen against the mutants |\n| Questions in other languages | Tiny Aya Global is multilingual; the reward does not depend on the question language | The audit numbers carry over unchanged; training effects untested |\n\nThese are forecasts, not results.\n\n## 11. Conclusion\n\nA reward that checks an SQL answer on one database pays for bugs that the data happens to hide. In our audit that happened to almost a third of realistic bugs, in two public rewards, and in the outputs of published systems on Spider. Running the same query on a few generated copies closes most of that hole and turns away nearly every reward hack, without rejecting correct queries.\n\nWhether the stricter reward also trains a better agent is still open. On our own prompts the two rewards rarely disagreed, so the training comparison could not separate them. On Spider, the first data we did not build, the model trained with the copies came out about two points ahead and made fewer lucky passes, in two independent pairs of runs. On our own test set neither pair put it ahead, including the one we pre-registered. We release everything needed to check our numbers.\n\n## References\n\n- Ai (2026). Bounded semantic planning and deterministic compilation for reliable enterprise text-to-SQL. [arXiv:2608.16663](https://arxiv.org/abs/2608.16663).\n- Chandra, Chawda, Kar, Reddy, et al. (2014). Data generation for testing and grading SQL queries (XData). *The VLDB Journal*. [arXiv:1411.6704](https://arxiv.org/abs/1411.6704).\n- Chen, Gan, Chung, Zhang, et al. (2026). DualSQL: Text-to-SQL with multi-agent reinforcement learning. [arXiv:2609.18135](https://arxiv.org/abs/2609.18135).\n- ESM+ evaluation code and Spider-dev predictions of published systems. GitHub repository [`bossben/ESMp`](https://github.com/bossben/ESMp), commit `91dd85b`.\n- Fu, Sun, Wei, Wei, et al. (2026). SQLoom: Set-contrastive learning of syntactic, semantic, and logical dependencies for text-to-SQL. [arXiv:2611.03298](https://arxiv.org/abs/2611.03298).\n- Habibollah, Rafiei (2026). SynSQL: re-grading text-to-SQL systems on synthesized databases. [arXiv:2604.27261](https://arxiv.org/abs/2604.27261).\n- He, Choi, Zhang, Ji, et al. (2025). HardTests: A high-quality RL verifier generation pipeline for LLM algorithmic coding. [arXiv:2505.24098](https://arxiv.org/abs/2505.24098).\n- He, Zhao, Wang, et al. (2024). VeriEQL: Bounded equivalence verification for complex SQL queries with integrity constraints. [arXiv:2403.03193](https://arxiv.org/abs/2403.03193).\n- Hu, Shen, Wallis, Allen-Zhu, et al. (2021). LoRA: Low-rank adaptation of large language models. *ICLR 2022*. [arXiv:2106.09685](https://arxiv.org/abs/2106.09685).\n- Jin, Choi, Zhu, Kang (2026). Pervasive annotation errors break text-to-SQL benchmarks and leaderboards. [arXiv:2601.08778](https://arxiv.org/abs/2601.08778).\n- Klopfenstein, He, Tremante, Wang, et al. (2025). SpotIt: Evaluating text-to-SQL evaluation with formal verification. *ICLR 2026*. [arXiv:2510.26840](https://arxiv.org/abs/2510.26840).\n- Lexsi Labs (2026). AuditKIT: agent evaluation toolkit. GitHub repository [`Lexsi-Labs/AuditKIT`](https://github.com/Lexsi-Labs/AuditKIT), commit `9da5838`.\n- Li, Hui, Qu, Yang, et al. (2023). Can LLM already serve as a database interface? A big bench for large-scale database grounded text-to-SQLs (BIRD). *NeurIPS 2023*. [arXiv:2305.03111](https://arxiv.org/abs/2305.03111).\n- Liu, Shen, Li, Tang, et al. (2025). NL2SQL-BUGs: A benchmark for detecting semantic errors in NL2SQL translation. *KDD 2025*. [arXiv:2503.11984](https://arxiv.org/abs/2503.11984).\n- Lv, Xie, Sun, Kang, et al. (2025). The climb carves wisdom deeper than the summit: On the noisy rewards in learning to reason. [arXiv:2505.22653](https://arxiv.org/abs/2505.22653).\n- Lyngkhoi, Gupta, Vats, Kadiyala, et al. (2026). AgentTune: A unified training lifecycle for tool-using LLM agents. [arXiv:2610.13335](https://arxiv.org/abs/2610.13335). Code: [`Lexsi-Labs/AgentTune`](https://github.com/Lexsi-Labs/AgentTune), commit `d4db4c1`. Our report: issue [Lexsi-Labs/AgentTune#11](https://github.com/Lexsi-Labs/AgentTune/issues/11).\n- Ma, Zhuang, Xu, Jiang, et al. (2025). SQL-R1: Training natural language to SQL reasoning model by reinforcement learning. *NeurIPS 2025*. [arXiv:2504.08600](https://arxiv.org/abs/2504.08600).\n- NovaSky-AI (2025). SkyRL, including the SkyRL-SQL reward. GitHub repository [`NovaSky-AI/SkyRL`](https://github.com/NovaSky-AI/SkyRL), commit `d870046`.\n- Papicchio, Rossi, Cagliero, Papotti (2025). Think2SQL: Reinforce LLM reasoning capabilities for Text2SQL. [arXiv:2504.15077](https://arxiv.org/abs/2504.15077).\n- ParSEval (2025). *PVLDB* 18(11).\n- Pasupat, Liang (2016). Inferring logical forms from denotations. *ACL 2016*. [arXiv:1606.06900](https://arxiv.org/abs/1606.06900).\n- Pourreza, Talaei, Sun, Wan, et al. (2025). Reasoning-SQL: Reinforcement learning with SQL tailored partial rewards for reasoning-enhanced text-to-SQL. [arXiv:2503.23157](https://arxiv.org/abs/2503.23157).\n- Rafailov, Sharma, Mitchell, Ermon, et al. (2023). Direct preference optimization: Your language model is secretly a reward model. *NeurIPS 2023*. [arXiv:2305.18290](https://arxiv.org/abs/2305.18290).\n- Razin, Wang, Strauss, Wei, et al. (2025). What makes a reward model a good teacher? An optimization perspective. *NeurIPS 2025*. [arXiv:2503.15477](https://arxiv.org/abs/2503.15477).\n- Salamanca, Abagyan, D'souza, et al. (2026). Tiny Aya: Bridging scale and multilingual depth. [arXiv:2603.11510](https://arxiv.org/abs/2603.11510).\n- Shao, Wang, Zhu, Xu, et al. (2024). DeepSeekMath: Pushing the limits of mathematical reasoning in open language models (introduces GRPO). [arXiv:2402.03300](https://arxiv.org/abs/2402.03300).\n- shro-2002 (2026). RewardShield. GitHub repository [shro-2002/rewardshield](https://github.com/shro-2002/rewardshield).\n- SiddharthaRaja (2026). Mirage: GRPO for text-to-SQL with a perturbed-database reward suite. GitHub repository [SiddharthaRaja/Mirage](https://github.com/SiddharthaRaja/Mirage).\n- Team Cohere (2025). Command A: An enterprise-ready large language model. [arXiv:2504.00698](https://arxiv.org/abs/2504.00698).\n- Wang, Liu, Sun, Li, et al. (2025). CodeContests+: High-quality test case generation for competitive programming. [arXiv:2506.05817](https://arxiv.org/abs/2506.05817).\n- Weng, Wu, Cui, Zhan, et al. (2025). Graph-Reward-SQL: Execution-free reinforcement learning for text-to-SQL via graph matching and stepwise reward. [arXiv:2505.12380](https://arxiv.org/abs/2505.12380).\n- Yao, Shinn, Razavi, Narasimhan (2024). \u03c4-bench: A benchmark for tool-agent-user interaction in real-world domains. [arXiv:2406.12045](https://arxiv.org/abs/2406.12045).\n- Yao, Sun, Borchmann, Nuti, et al. (2025). Arctic-Text2SQL-R1: Simple rewards, strong reasoning in text-to-SQL. [arXiv:2505.20315](https://arxiv.org/abs/2505.20315).\n- Yu, Zhang, Yang, Yasunaga, et al. (2018). Spider: A large-scale human-labeled dataset for complex and cross-domain semantic parsing and text-to-SQL task. *EMNLP 2018*. [arXiv:1809.08887](https://arxiv.org/abs/1809.08887).\n- Zhang, Wang, Liu, Lan, et al. (2026a). Progress-SQL: Improving reinforcement learning for text-to-SQL via progressive rewards. [arXiv:2606.06825](https://arxiv.org/abs/2606.06825).\n- Zhang, Yan, Huang, Zhao, et al. (2026b). Robust code RL via faulty-code-driven test case synthesis and dense reward shaping (RobustTests). *EMNLP 2026*. [arXiv:2608.24135](https://arxiv.org/abs/2608.24135).\n- Zhang (2026c). When the reward suite is leaky. [arXiv:2607.11022](https://arxiv.org/abs/2607.11022).\n- Zhong, Yu, Klein (2020). Semantic evaluation for text-to-SQL with distilled test suites. *EMNLP 2020*. [arXiv:2010.02840](https://arxiv.org/abs/2010.02840).\n- Zhu, Jin, Choi, Kang (2026). Human-level text-to-SQL via reinforcement learning on verified data, without pipeline engineering (ReViSQL). [arXiv:2603.20004](https://arxiv.org/abs/2603.20004).\n\n## Appendix A. Artifacts, licences and Lexsi tools\n\n**Lexsi repositories used.**\n- **AgentTune** (`d4db4c1`) provides the multi-turn GRPO trainer with tool calls, its tool-prompt fallback, its render, parser and fold functions, and `lexsi_provenance.json`.\n - Our trainer fix: a patch to the multi-turn token stream. Before it, the importance ratio was about 1e-5.\n - Our reward fix: a patch, with a reproduction script and a regression test. We reported the exploit as issue [Lexsi-Labs/AgentTune#11](https://github.com/Lexsi-Labs/AgentTune/issues/11); the maintainers fixed it upstream the next day in [PR #12](https://github.com/Lexsi-Labs/AgentTune/pull/12) (`e56a109`).\n- **AuditKIT** (`9da5838`) provides the agent-evaluation framing (claimed versus verified success, silent errors) and pass^k. We reimplemented the statistics in our own evaluation code.\n\n**Models.**\n- `CohereLabs/tiny-aya-global` (3.35B, CC-BY-NC-4.0, gated) is the policy we train.\n- Cohere Command A (`command-a-03-2025`, trial API) is used for the cost comparison.\n\n**Model collection.** [Collection](https://huggingface.co/collections/sqlcf-lexsi/counterfactual-db-rewards-for-a-small-sql-agent-6af174f77dbae0bf0a54c3c4): the in-browser [demo](https://huggingface.co/spaces/sqlcf-lexsi/counterfactual-sql-demo) and three LoRA adapters.\n\n| Repo | Model | Label |\n|---|---|---|\n| [`sqlcf-lexsi/tiny-aya-global-sqlcf-sft`](https://huggingface.co/sqlcf-lexsi/tiny-aya-global-sqlcf-sft) | SFT, the init of both GRPO runs | baseline, 9.4% verified |\n| [`sqlcf-lexsi/tiny-aya-global-sqlcf-grpo-b`](https://huggingface.co/sqlcf-lexsi/tiny-aya-global-sqlcf-grpo-b) | GRPO-B, single-DB reward | improved: 12.7%, beats SFT (p = 0.006) |\n| [`sqlcf-lexsi/tiny-aya-global-sqlcf-grpo-c`](https://huggingface.co/sqlcf-lexsi/tiny-aya-global-sqlcf-grpo-c) | GRPO-C, counterfactual reward | null: 9.0%, no gain over SFT (p = 0.21) |\n\nThe two curriculum adapters of Section 8 are not released; their test and Spider trajectories are in the data collection.\n\n**Adapter licence and use.**\n- The adapters are derivatives of a gated, non-commercial model. Each repo ships the base model's CC-BY-NC-4.0 licence and a notice pointing to Cohere's licence addendum and Acceptable Use Policy. The repos are gated with automatic approval, as the organisers ruled on Nov 9.\n- Using them requires access to `CohereLabs/tiny-aya-global`.\n- vLLM's LoRA path crashes on Tiny Aya's tied embeddings, so merge first (fp32 merge, saved as fp16).\n\n**Data collection.** [Collection](https://huggingface.co/collections/sqlcf-lexsi/counterfactual-sql-data-rollouts-and-traces-6af479430243b7731f85d144) with [`sqlcf-lexsi/counterfactual-sql-data`](https://huggingface.co/datasets/sqlcf-lexsi/counterfactual-sql-data). It contains the items, all SQLite databases, SFT trajectories, results, test trajectories for every model, and every GRPO rollout with its per-database pass vector and reward. Released under CC-BY-NC-4.0, because the rollouts contain model outputs.\n\n**Spider data.** The Spider data is not ours: questions from `xlangai/spider` (CC BY-SA 4.0), gold SQL from ESM+, and test-suite databases from Zhong et al. We do not redistribute any of it. Our released Spider trajectories keep only our models' turns and the tool outputs. The tool outputs include up to 20 rows from a Spider database, so those files fall under CC BY-SA 4.0.\n\n## Appendix B. Detailed results\n\n**B1. Hack catalogue.** Every reward hack we tried, and how often each reward pays it.\n\n| Hack | Example | Single-DB reward pays | Counterfactual reward pays |\n|---|---|---|---|\n| Literal smuggling: return the answer seen while exploring | `SELECT 5` | 1302 / 1302 | **0 / 1302** |\n| Hard-coded IDs: list the gold rows' IDs | `SELECT customer_id FROM customers WHERE customer_id IN (3, 17, 42)` | 235 / 235 | 54 / 235 (the random answer-changing copy kept the same IDs) |\n| Empty result | `SELECT NULL WHERE 0` | 0 / 1302 | 0 / 1302 |\n| WHERE padding plus a fixed answer (AgentTune reward only) | `SELECT * FROM customers WHERE 1 = 1`, then `*yes*` | 3.10, equal to honest SQL, on all 314 yes items | n/a (yes/no reward); our patch: 2.20 |\n| Duplicate rows instead of DISTINCT (set-comparison rewards only) | gold `SELECT DISTINCT o.customer_id ...`, hack drops `DISTINCT` | SkyRL pays 203 / 203 | set comparison with copies still pays 203 / 203; strict comparator with copies 15 / 203 |\n\nOn the 360 yes/no items, the AgentTune exploit scores +2.45 overall against +3.10 for honest SQL.\n\n**B2. Bugs invisible on one database (Figure 2).** Share of gold-mutants that pass the single-database check: 29.2% at 50 customers, 0.8% at 500, 0.2% at 5,000.\n\n**B3. SkyRL-SQL reward** (`NovaSky-AI/SkyRL` at `d870046`, re-implemented from its source).\n\n| Reward | Mutants paid (n = 1814) | Hacks paid (n = 2839) | Correct rejected (n = 2847) |\n|---|---|---|---|\n| SkyRL as shipped (set compare, one DB) | 37.5% [35.3, 39.7] | 53.2% | 0 |\n| SkyRL's set compare on original and copies | 27.2% [25.2, 29.3] | 1.9% | 0 |\n| Strict multiset, one DB | 29.5% | 54.1% | 0 |\n| Strict multiset on original and copies (ours) | **16.8%** | **1.9%** | 0 |\n\nCopies close the database hole (10.3 points); a strict comparator closes the comparator hole (10.4 points more). 188 of the 203 dropped-`DISTINCT` mutants are paid by set comparison on every database.\n\n**B4. Published systems on Spider dev** (1034 questions). EX = original database; TS = original plus every test-suite database.\n\n| System | EX | TS | Lucky share of EX passes [95% CI] |\n|---|---|---|---|\n| Claude | 80.6% | 68.6% | **14.9%** [12.6, 17.5] |\n| C3 (ChatGPT) | 79.7% | 68.5% | 14.1% [11.9, 16.6] |\n| DIN-SQL | 81.5% | 72.2% | 11.4% [9.4, 13.7] |\n| GPT-4 Turbo | 72.3% | 64.7% | 10.6% [8.6, 13.0] |\n| DAIL-SQL | 82.7% | 75.3% | 8.9% [7.2, 11.0] |\n| RASAT+PICARD | 76.5% | 70.1% | 8.3% [6.6, 10.5] |\n| RESDSQL | 82.6% | 75.8% | 8.2% [6.5, 10.2] |\n| Graphix | 79.9% | 73.4% | 8.1% [6.4, 10.2] |\n\nOur scorer keeps `DISTINCT` and uses our strict comparator, so our EX and TS run a few points below the official script's. The lucky share uses the same comparator on both sides.\n\n**B5. Training rollouts (Figure 3)**, 1440 rollouts per run.\n\n| Run | Rollouts passing the original DB | Share of those failing a copy | First half | Second half |\n|---|---|---|---|---|\n| B (`R_exec_rex`) | 86.4% | 8.2% | 10.7% | 5.9% |\n| C (`R_cf_mean`) | 86.5% | 8.3% | 11.1% | 5.7% |\n\nReplaying all rollouts, C's reward was lower than B's on 7.1% (B run) and 7.2% (C run) of rollouts, and higher on 0.1% and 0.3%.\n\n**B6. Full test results** (331 items \u00d7 8 samples).\n\n| Model | Verified | Single-DB | Lucky | Silent error | pass^2 | pass^4 | Held-out F4 verified |\n|---|---|---|---|---|---|---|---|\n| base | 0.9% | 1.4% | 0.5% | 4.5% | 0.1% | 0.0% | 0.6% |\n| SFT | 9.4% | 14.5% | 5.1% | 65.4% | 4.9% | 2.1% | 9.2% |\n| GRPO-B | 12.7% | 17.8% | 5.1% | 64.0% | 8.1% | 3.9% | 9.9% |\n| GRPO-C | 9.0% | 14.0% | 5.1% | 61.2% | 4.7% | 2.2% | 8.8% |\n| Command A | 81.0% | 82.8% | 1.8% | n/a | n/a | n/a | 94.9% |\n\n- **B against SFT:** +3.2 points [1.8, 4.8], p = 0.006.\n- **C against SFT:** \u22120.5 points [\u22121.5, +0.6], p = 0.21.\n- **C against B:** \u22123.7 points [\u22125.3, \u22122.3], p < 0.001.\n- **Silent errors, C against B:** \u22122.9 points [\u22125.3, \u22120.5] by bootstrap over all 8 samples; McNemar on sample 0 gives p = 0.61.\n- **Release labels (gate G3):** B beats SFT and is released as improved; C is released labelled as a null.\n\n**B7. Hand audit of wrong answers**. 48 wrong test trajectories, 16 per model (SFT, B, C), sampled with seed 0 and stratified by family. Each model contributes 6 lucky passes and 10 other wrong answers.\n\n| Error | NL2SQL-BUGs category | All (48) | Lucky passes (18) |\n|---|---|---|---|\n| `order_date` used where the question says payment date | Attribute | 11 | 4 |\n| Product category matched against customer columns, or a hallucinated value | Value | 7 | 0 |\n| Extra status filter the question does not ask for | Condition | 6 | 5 |\n| No latest-row filter for \"current tier\" | Condition | 5 | 2 |\n| AVG over order items instead of per-order totals | Clause | 5 | 0 |\n| `invoice_adjustments` left out of net invoice | Table | 4 | 3 |\n| Does not run (undefined alias, unknown column or function) | Others | 4 | 0 |\n| `COUNT(*)` instead of `SUM(quantity)` | Function | 3 | 3 |\n| Incoherent, or an exploratory query given as the answer | Others | 2 | 0 |\n| Wrong table | Table | 1 | 1 |\n\nThe three models' error mixes are similar. With 16 per model, the audit cannot show a difference between B and C.\n\n**B8. Cost.**\n- **Command A** (trial key, so the real charge was $0). The Cohere dashboard shows 222,578 input and 85,815 output tokens for the run day, against our logged 220,580 and 85,348. That prices at $4.27 per 1k queries against our $4.24. List prices are $2.50 per 1M input tokens and $10.00 per 1M output tokens.\n- **Tiny Aya** is priced at our L4 VM's on-demand rate ($0.74/hr, g2-standard-4).\n\n| Model | Verified | Single-DB | $ per 1k queries | Speed |\n|---|---|---|---|---|\n| Command A | 81.0% [76.4, 84.8] | 82.8% | $4.24 | median 9.7 s, p90 42 s per query |\n| Tiny Aya SFT | 9.4% [8.4, 10.6] | 14.5% | $0.054 | 3.8 trajectories/s batched on L4 (scaled) |\n| Tiny Aya GRPO-B | 12.7% [11.5, 14.0] | 17.8% | $0.061 | 3.4 trajectories/s batched on L4 (measured) |\n| Tiny Aya GRPO-C | 9.0% [7.9, 10.1] | 14.0% | $0.052 | 4.0 trajectories/s batched on L4 (scaled) |\n\nScaling: GRPO-B's test generation took 785 s on the L4 and 524 s on an RTX 5070 Ti, with the same accuracy. SFT and C were timed on the 5070 Ti only and scaled by that ratio (1.50). Per-query latency was not measured for Tiny Aya.\n\n**B9. Voting without gold** (paired bootstrap over items).\n\n| Model | First sample | Vote, 1 DB | Vote, 5 DBs | Any of 8 (oracle) | 5-DB vote minus first [95% CI] |\n|---|---|---|---|---|---|\n| SFT | 8.5% | 10.9% | 11.8% | 21.5% | +3.3 [+0.6, +6.3] |\n| GRPO-B | 13.9% | 16.9% | 16.3% | 22.4% | +2.4 [\u22120.6, +5.4] |\n| GRPO-C | 6.3% | 13.6% | 13.9% | 21.5% | +7.6 [+4.8, +10.6] |\n\n- **Shaky-answer flag.** Flagging answers where fewer than half of the samples agree catches 53\u201359% of wrong answers at 94\u201398% precision. The extra databases add nothing here either (AUROC 0.67\u20130.74 either way).\n- **Pass@8.** All three trained models reach about the same pass@8, so B's gain is in picking its right answer more often, not in finding new ones.\n\n**B10. DPO from one checkpoint**.\n\n**Source rollouts.** SFT's 2670 rollouts on the 445 GRPO prompts (6 per prompt, T = 1.0) grade into 1971 verified, 170 lucky and 529 failed.\n\n**Labels:**\n- **Single-DB labels:** chosen from verified and lucky, rejected from failed. This gives 420 pairs, 96 of them with a lucky answer as the chosen one.\n- **Counterfactual labels:** chosen from verified, rejected from lucky and failed, lucky first. This gives 351 pairs, 30 of them with a lucky answer rejected.\n\n**Training.** At most 3 pairs per prompt; sigmoid DPO with \u03b2 = 0.1, learning rate 2e-5, 1 epoch, effective batch 16, seed 0.\n\n| Model | Verified [95% CI] | Single DB | Lucky pass | Invalid tool calls | Silent error |\n|---|---|---|---|---|---|\n| SFT (start) | 9.4% [8.4, 10.6] | 14.5% | 5.1% | 17.2% | 65.4% |\n| DPO, single-DB labels | 9.7% [8.6, 10.9] | 14.1% | 4.5% | 20.0% | 62.4% |\n| DPO, counterfactual labels | 8.1% [7.1, 9.2] | 12.5% | 4.5% | 27.4% | 55.1% |\n\n**Comparisons.**\n- **Single-DB labels against SFT:** +0.2 [\u22121.0, +1.5].\n- **Counterfactual against single-DB labels:** \u22121.6 [\u22122.6, \u22120.6]; McNemar on the first sample gives p = 0.40.\n\n**Why the lower silent-error rate is not a gain.** An error only counts as silent if every tool call succeeded, and the counterfactual-label run makes far more failing tool calls.\n\n**By family.** Both runs lose on F1 (from 30.9% to 21.2% and 10.4%) and gain a little on F2 (from 1.0% to 4.5%). There is one seed each, and the counterfactual run has fewer pairs.\n\n**B11. Partial credit**.\n\n**How the credit is computed.** Each database gives 1 for a match; for a single number, its closeness, 1 \u2212 |pred \u2212 gold| / |gold|; otherwise the F1 of the result rows. `P_single` is the credit on the original; `P_cf` is the mean over the original and every copy.\n\n| Reward | Mean pay: mutants / hacks | AUROC, correct vs mutant / hack | GRPO groups with any signal | Lucky rollouts above group mean | Wrong rollouts paid > 0 |\n|---|---|---|---|---|---|\n| `R_exec_rex` (one DB, 0/1; B) | 0.30 / 0.54 | 0.85 / 0.73 | 31.5% | 63.5% | 24.3% |\n| `R_cf_mean` (copies, 0/1 mean; C) | 0.23 / 0.28 | 0.92 / 0.99 | 32.1% | 58.8% | 23.3% |\n| `R_cf_strict` (all DBs) | 0.17 / 0.02 | 0.92 / 0.99 | 26.3% | 0% | 0% |\n| `P_single` (one DB, partial) | 0.66 / 0.54 | 0.85 / 0.73 | 37.8% | 63.5% | 52.5% |\n| `P_cf` (copies, partial) | 0.65 / 0.47 | 0.92 / 0.99 | 39.8% | 69.4% | 56.2% |\n\nA query that hard-codes a number close to the answer earns 0.86 under `P_cf`, against 0 under `R_cf_strict`.\n\n**B12. Period-edge copies** (19 s on a CPU).\n\n**How each copy is built:**\n- Half of the slice's orders in year Y move to Jan 1 and Dec 31 of Y, alternating.\n- Half of its orders in the other year move to the day just outside Y.\n- Payments, refunds and adjustments move with their order.\n\n**Date bugs.** They come from the 777 year-filtered audit queries: Dec 31 excluded, Jan 1 excluded, Jan 1 of the next year included, and order date swapped with payment date.\n\n| Date bugs paid (n = 2535) [95% CI] | |\n|---|---|\n| Original DB only | 92.1% [91.0, 93.1] |\n| Original plus the 2 mutant-blind copies | 91.4% [90.2, 92.4] |\n| Original plus the training-reward copies | 90.9% [89.7, 92.0] |\n| **Original plus the 2 copies plus the period-edge copy** | **32.9%** [31.1, 34.7] |\n| Correct queries rejected by the new copy | **0 / 1554** |\n\n**Effect on real answers.** On real answers to the year items, the new copy rejects 2 of B's 305 verified test answers, 2 of C's 218, 1 of SFT's 223, and 4 of SFT's 998 verified GRPO rollouts. Each is the same bug: \"by payment date\" filtered on the order date.\n\n## Appendix C. Implementation notes\n\n- **Why some templates are held out of SFT.** With all 11 train templates in SFT, only 8 of 2670 GRPO rollouts passed the original database but failed a copy. B and C would have seen almost the same reward. We held 3 templates out of SFT so that the rewards differ. Test templates were never touched.\n- **Merging before evaluation.** vLLM's LoRA path crashes on Tiny Aya's tied embeddings, so GRPO adapters are merged (fp32 merge, saved as fp16) before evaluation.\n- **No vLLM sleep mode.** It segfaults on wake-up (vLLM 0.22.1), so we leave it off.\n\n## Appendix D. Prompts\n\n**Agent system prompt** (`sqlcf/agent.py`; the schema and business definitions are filled in from `sqlcf/db.py`):\n\n```\nYou are a careful data analyst working with a retail SQLite database.\n\nSchema:\n{SCHEMA}\n\nBusiness definitions:\n{GLOSSARY}\n\nUse the run_sql tool to explore the data and check your query before answering. Watch for\nduplicate rows, joins that multiply rows, missing business filters and superseded records.\nWhen you are done, reply with one short sentence and your final query in a ```sql block.\nThe final query must compute the answer from the tables; do not hard-code values.\n```\n\nAgentTune then appends its fallback tool prompt with the `run_sql` schema. The user turn is `Database: <id>` followed by `Question: <question>`.\n\n**Command A prompt** (`scripts/e5_command_a.py`): the same system prompt, with the tool sentence replaced by *\"You cannot run queries; answer from the schema and definitions.\"* Single turn, one sample.\n\n## Reproduction\n\n**Environment.** Python 3.12 for training (AgentTune `d4db4c1` with our patch, TRL, PEFT, vLLM 0.22.1) and any Python 3.11+ for the CPU parts (pandas, numpy, scipy, matplotlib, pytest). SFT and evaluation fit a 16 GB GPU; GRPO needs a 24 GB GPU (we used an L4). The data, the reward audit and every scoring step run on a CPU. Tiny Aya is gated: accept its licence and use a Hugging Face read token.\n\n**Repository layout.** `sqlcf/` holds the data generator, templates, counterfactual copies, comparator, rewards, audit and agent tool; `train.py` runs SFT, GRPO and merging; `evaluate.py` generates and scores test trajectories; `scripts/` holds one script per experiment; `patches/` the AgentTune fixes; `tests/` the reward and regression tests; `results/` every stored result. The [README](https://github.com/RishiiGamer2201/sexsi_hack#readme) lists every command; the core path is:\n\n```bash\npython -m sqlcf.cf --name train --split train --seeds 101 102 103 104 105 106 --size S # data + copies\npython -m sqlcf.audit e0 # reward audit (Section 4)\npython train.py sft --out runs/sft # SFT\npython train.py grpo --reward R_exec_rex --out runs/grpo_B # GRPO-B\npython train.py grpo --reward R_cf_mean --out runs/grpo_C # GRPO-C\npython evaluate.py gen --model runs/grpo_B/merged --name B # test trajectories\npython evaluate.py score --traj results/traj_*.jsonl # Tables 2 and B6\npython verify_all.py # re-derive every results/*.json\n```\n\n## Acknowledgements\n\nLexsi Labs organised the hackathon and maintains AgentTune and AuditKIT; its maintainers reviewed and merged our reward fix within a day. Cohere provided Tiny Aya Global and a trial API key for Command A. Spider (Yu et al., 2018), the test-suite databases (Zhong et al., 2020) and the ESM+ prediction files made the real-data checks possible.\n\n## Declaration of work done outside the hackathon\n\nNone. All code, data and experiments were made during the hackathon; the repository's first commit is Nov 8, 2026. Third-party code and data are listed in Appendix A and used under their licences.\n\n## Teammates tagged\n\n- **Shreshth Rai** (team lead): Hugging Face [@hi-phi-r25i](https://huggingface.co/hi-phi-r25i) \u00b7 GitHub [@shreshth3000](https://github.com/shreshth3000)\n- **Rishii Kumar Singh**: Hugging Face [@LogicPalette](https://huggingface.co/LogicPalette) \u00b7 GitHub [@RishiiGamer2201](https://github.com/RishiiGamer2201)\n- **Aksh Aggarwal**: Hugging Face [@AkshAgg](https://huggingface.co/AkshAgg) \u00b7 GitHub [@Aksh2906](https://github.com/Aksh2906)\n", MATH = [["q", false], ["g", false], ["D_0", false], ["D_1,\\dots,D_K", false], ["C_k = \\mathrm{match}\\big(q(D_k),\\, g(D_k)\\big) \\in \\{0,1\\}, \\qquad k = 0,\\dots,K", true], ["R_{\\mathrm{rex}} = C_0, \\qquad R_{\\mathrm{cf}}^{\\mathrm{mean}} = C_0 \\cdot \\frac{1}{K}\\sum_{k=1}^{K} C_k, \\qquad R_{\\mathrm{cf}}^{\\mathrm{strict}} = \\prod_{k=0}^{K} C_k, \\qquad R_{\\mathrm{cf}}^{\\mathrm{soft}} = C_0\\big(1 - 0.2\\cdot\\mathbf{1}[\\exists k \\ge 1 : C_k = 0]\\big)", true], ["R_{\\mathrm{cf}}^{\\mathrm{soft}}", false], ["R_{\\mathrm{cf}}^{\\mathrm{mean}}", false], ["R_{\\mathrm{rex}}", false], ["R_{\\mathrm{cf}}^{\\mathrm{mean}}", false]];
let html = marked.parse(MD, {gfm: true}).replace(/MATHPH(\d+)END/g, (_, i) =>
katex.renderToString(MATH[i][0], {displayMode: MATH[i][1], throwOnError: false}));
const el = document.getElementById("body");
el.innerHTML = html;
el.querySelectorAll("table").forEach(t => { const w = document.createElement("div"); w.className = "tw"; t.before(w); w.append(t); });
// figure + caption from the image alt text
el.querySelectorAll("p > img:only-child").forEach(img => {
const f = document.createElement("figure"), c = document.createElement("figcaption");
c.textContent = img.alt; img.parentElement.replaceWith(f); f.append(img, c); });
// clickable citations: "Zhu et al., 2026", "Zhong, Yu & Klein (2020)", "Ai, 2026" -> #ref-<surname><year> in References
const rh = [...el.querySelectorAll("h2")].find(h => h.textContent.trim() === "References");
const refs = rh && rh.nextElementSibling, keys = new Set();
if (refs) refs.querySelectorAll("li").forEach(li => {
const m = li.textContent.match(/^(.+?)(?:,.*?)? \((\d{4}[ab]?)\)/);
if (m) { const k = m[1].trim().replace(/\W/g, "") + m[2]; li.id = "ref-" + k; keys.add(k); } });
const CITE = /\b(Team Cohere|Lexsi Labs|[A-Z][A-Za-z'-]+)((?: et al\.)|(?:,? [A-Z][a-z]+)*(?: & [A-Z][A-Za-z]+)?),? (?:\((\d{4}[ab]?)\)|(\d{4}[ab]?)\b)/g;
const tw = document.createTreeWalker(el, NodeFilter.SHOW_TEXT, { acceptNode: n =>
n.parentElement.closest("a, code, pre, .katex") || (refs && refs.contains(n)) ? 2 : 1 });
const texts = []; while (tw.nextNode()) texts.push(tw.currentNode);
texts.forEach(n => {
const s = n.nodeValue; let last = 0, frag = null;
for (const m of s.matchAll(CITE)) {
const k = m[1].replace(/\W/g, "") + (m[3] || m[4]); if (!keys.has(k)) continue;
frag = frag || document.createDocumentFragment();
frag.append(s.slice(last, m.index));
const a = document.createElement("a"); a.href = "#ref-" + k; a.className = "cite"; a.textContent = m[0];
frag.append(a); last = m.index + m[0].length; }
if (frag) { frag.append(s.slice(last)); n.replaceWith(frag); } });
// abstract block: the "Abstract" heading and its paragraphs, up to the next h2
const ah = [...el.querySelectorAll("h2")].find(h => h.textContent.trim() === "Abstract");
if (ah) { const box = document.createElement("section"); box.className = "abstract"; ah.before(box);
let n = ah; while (n && !(n !== ah && n.tagName === "H2")) { const nx = n.nextElementSibling; box.append(n); n = nx; } }
el.querySelectorAll('a[href^="http"]').forEach(a => { a.target = "_blank"; a.rel = "noopener"; });
</script>
</body></html>