// MAIN Lab benchmark registry. Newest first. // To add a benchmark: copy one object (same format as brand/benchmarks/bNNN.js, plus // num, date, kind, verdict ("win" | "negative" | "mixed"), headline {v,l} (big number on the card), summary (one short line), links) and put it at the top. // tests: [title, dataKey, scale, decimals, lowerWins, note, mode] mode: "tie" = not significant, "none" = no winner window.MAIN_BENCHMARKS = [ { "id": "Benchmark #003", "title": "Weight-space programming,", "accent": "working.", "sub": "Benchmark #003 · Qwen2.5-0.5B-Instruct · 197 unseen Azerbaijani prompts · no training, weights edited directly", "note": "Edit chosen on practice prompts only (19 of 50 probes) · fluency judged by SIBA Meristem (native Azerbaijani LM) · #1 = winner", "models": [ { "k": "base", "short": "Qwen", "img": "qwen-color.svg" }, { "k": "prompted", "short": "+ Prompt", "img": "qwen-color.svg" }, { "k": "edited", "short": "Ours", "ours": true } ], "tests": [ [ "Clean Azerbaijani", "clean", 1, 1, false, "% of answers" ], [ "Answered in Turkish", "tr", 1, 1, true, "↓ lower wins" ], [ "Turkish words", "leak", 1, 1, true, "per 100 words ↓" ], [ "English kept English", "en", 1, 0, false, "% · selectivity" ], [ "Fluency", "bpb", 1, 3, true, "bits/byte ↓", "none" ], [ "Reading QA (Az)", "qa", 100, 1, false, "F1 · n=100", "tie" ], [ "Grammar (Az)", "mp", 100, 1, false, "chance 50", "tie" ] ], "data": { "base": { "clean": 55.32994923857868, "tr": 41.6243654822335, "leak": 8.395185384148109, "en": 100, "bpb": 1.3322746987828893, "qa": 0.1300687935298941, "mp": 0.515 }, "prompted": { "clean": 57.36040609137056, "tr": 36.54822335025381, "leak": 6.660425417629274, "en": 100, "bpb": 1.325934701873973, "qa": null, "mp": null }, "edited": { "clean": 77.15736040609137, "tr": 18.274111675126903, "leak": 3.900726450831416, "en": 100, "bpb": 1.3566729530293398, "qa": 0.10965533322886264, "mp": 0.485 } }, "tag": "EXPERIMENT · IT WORKED", "hero": [ { "v": "42% → 18%", "l": "answers in Turkish" }, { "v": "55% → 77%", "l": "clean Azerbaijani answers" }, { "v": "128", "l": "numbers changed, no training" } ], "num": "003", "date": "2026-11-17", "kind": "Weight-space programming", "verdict": "win", "summary": "128 numbers changed. No training. English untouched.", "links": [ { "label": "Model on Hugging Face", "href": "https://huggingface.co/MAIN-Lab/qwen2.5-0.5b-az-steer" }, { "label": "Before / after answers", "href": "https://huggingface.co/spaces/MAIN-Lab/az-steer" } ], "headline": { "v": "42→18%", "l": "answers in Turkish" } }, { "id": "Benchmark #002", "title": "Weight-space programming,", "accent": "tested.", "sub": "Benchmark #002 · Qwen2.5-0.5B-Instruct · MATH-500 · no gradients, weights edited directly", "note": "MATH-500: 500 problems, greedy, 1,024-token limit · practice: 100 MATH-train problems · #1 = winner", "models": [ { "k": "base", "short": "Qwen", "img": "qwen-color.svg" }, { "k": "edit", "short": "Ours", "ours": true } ], "tests": [ [ "MATH-500", "acc", 100, 1, false, "accuracy · p = 0.76", "tie" ], [ "Stuck in a loop", "truncated", 100, 1, true, "↓ lower wins" ], [ "No final answer", "no_box", 100, 1, true, "↓ lower wins" ], [ "Practice set", "acc_p", 100, 1, false, "accuracy" ], [ "Weights changed", "pct", 1, 3, true, "% of model", "none" ] ], "data": { "base": { "acc": 0.258, "truncated": 0.12, "no_box": 0.126, "acc_p": 0.32, "flip": 47, "pct": 0 }, "edit": { "acc": 0.264, "truncated": 0.154, "no_box": 0.164, "acc_p": 0.24, "flip": 50, "pct": 0.023 } }, "cols": 5, "theme": "dark", "tag": "EXPERIMENT · NEGATIVE RESULT", "hero": [ { "v": "TIE", "l": "MATH-500: 25.8% → 26.4% (p = 0.76)" }, { "v": "50 / 47", "l": "problems fixed / broken by the edit" }, { "v": "12 of 50", "l": "probes used, practice set only" } ], "num": "002", "date": "2026-11-17", "kind": "Weight-space programming", "verdict": "negative", "summary": "Editing weights for math did not work. Published anyway.", "links": [], "headline": { "v": "TIE", "l": "on MATH-500, p = 0.76" } }, { "id": "Benchmark #001", "tag": "BENCHMARK · LEADERBOARD", "title": "Small models,", "accent": "Azerbaijani.", "sub": "Benchmark #001 · 5 models · 11 tests", "note": "Same 11 tests for every model · #1 = winner", "models": [ { "k": "tuned", "short": "Ours", "ours": true }, { "k": "base", "short": "Gemma", "img": "gemma-color.svg" }, { "k": "gpt2az", "short": "GPT-2", "img": "openai.svg" }, { "k": "qwen3_aze", "short": "Qwen", "img": "qwen-color.svg" }, { "k": "meristem_native", "short": "SIBA", "txt": "SIBA" } ], "tests": [ [ "Az vs Turkish", "az_tr", 100, 1, false, "chance 50" ], [ "Grammar", "minimal_pairs", 100, 1, false, "chance 50" ], [ "Bits per byte", "bpb", 1, 2, true, "↓ lower wins" ], [ "Translation EN→AZ", "translate", 1, 1, false, "chrF++" ], [ "Reading QA", "extractive_qa", 100, 1, false, "F1" ], [ "Belebele", "belebele", 100, 1, false, "chance 25" ], [ "INCLUDE", "include", 100, 1, false, "chance 25" ], [ "Az facts", "knowledge", 100, 1, false, "chance 5" ], [ "Sentiment", "sentiment", 100, 1, false, "chance 33" ], [ "NLI", "nli", 100, 1, false, "chance 33" ], [ "RAG check", "rag_verify", 100, 1, false, "one-answer 50" ] ], "data": { "tuned": { "minimal_pairs": 0.8, "az_tr": 0.86, "bpb": 1.6082452111633438, "belebele": 0.31333333333333335, "include": 0.25547445255474455, "knowledge": 0.04666666666666667, "sentiment": 0.34, "nli": 0.34, "rag_verify": 0.25, "translate": 25.12997908359113, "extractive_qa": 0.1846032313724571 }, "base": { "minimal_pairs": 0.515, "az_tr": 0.49, "bpb": 2.8939, "belebele": 0.3244, "include": 0.2664, "knowledge": 0.06, "sentiment": 0.34, "nli": 0.34, "rag_verify": 0.25, "translate": 13.954647273559237, "extractive_qa": 0.04242065027049547 }, "gpt2az": { "minimal_pairs": 0.495, "az_tr": 0.895, "bpb": null, "belebele": 0.24, "include": 0.24087591240875914, "knowledge": 0.04666666666666667, "sentiment": 0.26, "nli": 0.32, "rag_verify": 0.5, "translate": 1.7315151459110896, "extractive_qa": 0 }, "qwen3_aze": { "minimal_pairs": 0.59, "az_tr": 0.58, "bpb": 2.396953399250009, "belebele": 0.30444444444444446, "include": 0.29927007299270075, "knowledge": 0.06, "sentiment": 0.34, "nli": 0.37, "rag_verify": 0.25, "translate": 15.237250307254, "extractive_qa": 0.24139966282309083 }, "meristem_native": { "minimal_pairs": 0.995, "az_tr": 0.99, "bpb": 1.0795030482726, "belebele": 0.29777777777777775, "include": 0.29927007299270075, "knowledge": 0.05333333333333334, "sentiment": 0.34, "nli": 0.32, "rag_verify": 0.5, "translate": 7.341418029205259, "extractive_qa": 0.03356658901096682 } }, "num": "001", "date": "2026-11-16", "kind": "Fine-tuning", "verdict": "win", "summary": "Gemma 3 270M, fine-tuned by Claude on a CPU in 2 hours.", "links": [ { "label": "Model on Hugging Face", "href": "https://huggingface.co/MAIN-Lab/gemma-3-270m-az" } ], "headline": { "v": "#1", "l": "of 5 in EN→AZ translation" } } ];