| { | |
| "n": 100, | |
| "table": "penguins", | |
| "table_rows": 344, | |
| "server": "http://127.0.0.1:8099", | |
| "tuned": { | |
| "exec_accuracy": 1.0, | |
| "sql_error_rate": 0.0, | |
| "exact_match": 1.0, | |
| "seconds": 136.9, | |
| "sample_misses": [] | |
| }, | |
| "base_instructed": { | |
| "exec_accuracy": 0.93, | |
| "sql_error_rate": 0.07, | |
| "exact_match": 0.93, | |
| "seconds": 134.8, | |
| "sample_misses": [ | |
| { | |
| "q": "What is the highest bill length mm where island is Torgersen?", | |
| "gold": "SELECT MAX(bill_length_mm) FROM penguins WHERE island = 'Torgersen'", | |
| "pred": "SELECT MAX bill_length_mm FROM penguins WHERE island = 'Torgersen'", | |
| "why": "error" | |
| }, | |
| { | |
| "q": "What is the highest bill length mm where island is Dream?", | |
| "gold": "SELECT MAX(bill_length_mm) FROM penguins WHERE island = 'Dream'", | |
| "pred": "SELECT MAX bill_length_mm FROM penguins WHERE island = 'Dream'", | |
| "why": "error" | |
| }, | |
| { | |
| "q": "What is the highest bill length mm where sex is FEMALE?", | |
| "gold": "SELECT MAX(bill_length_mm) FROM penguins WHERE sex = 'FEMALE'", | |
| "pred": "SELECT MAX bill_length_mm FROM penguins WHERE sex = 'FEMALE'", | |
| "why": "error" | |
| }, | |
| { | |
| "q": "What is the highest flipper length mm where island is Dream?", | |
| "gold": "SELECT MAX(flipper_length_mm) FROM penguins WHERE island = 'Dream'", | |
| "pred": "SELECT MAX flipper_length_mm FROM penguins WHERE island = 'Dream'", | |
| "why": "error" | |
| }, | |
| { | |
| "q": "What is the highest bill length mm where species is Gentoo?", | |
| "gold": "SELECT MAX(bill_length_mm) FROM penguins WHERE species = 'Gentoo'", | |
| "pred": "SELECT MAX bill_length_mm FROM penguins WHERE species = 'Gentoo'", | |
| "why": "error" | |
| }, | |
| { | |
| "q": "What is the highest bill length mm where island is Biscoe?", | |
| "gold": "SELECT MAX(bill_length_mm) FROM penguins WHERE island = 'Biscoe'", | |
| "pred": "SELECT MAX bill_length_mm FROM penguins WHERE island = 'Biscoe'", | |
| "why": "error" | |
| } | |
| ] | |
| }, | |
| "model": "Llama-3.1-8B-Instruct (NousResearch mirror)", | |
| "pipeline": "6 public seaborn tables (titanic, mpg, tips, car_crashes, planets, healthexp) -> data tabular (1800 verified examples) -> tinyforge worker run backend=soup, NF4, 200 steps, batch 1 x accum 2, 9m29s, 2 examples/step -> adapter -> GGUF (llama.cpp b11380: adapter f16, base Q4_K_M) -> llama-server -ngl 24 on RTX 3050 Ti 4 GB, ~18 tok/s -> eval on 100 questions about an UNSEEN table (penguins)", | |
| "caveats": [ | |
| "tuned = 100/100 is a ceiling; base_instructed is already 93%, so this test cannot show how much headroom fine-tuning adds", | |
| "all 7 base misses are the same malformed MAX/MIN call (MAX bill_length_mm without parentheses): the gain here is consistency, not new capability", | |
| "test questions use the same 12 templates as training on a new schema, so it measures transfer to new tables, not to new kinds of questions", | |
| "adapter trained on the NF4 base, served on a Q4_K_M base: a small quantisation mismatch that did not hurt here", | |
| "n=100, one run, greedy decoding, no confidence interval; the same test on the 3B model was not run" | |
| ] | |
| } |