Download benchmarks.js from MAIN-Lab/ai: direct link, hf CLI and curl.
- Browser
- Download file 8.56 kB
-
https://huggingface.co/spaces/MAIN-Lab/ai/resolve/main/benchmarks.js
- Command line
-
hf download hf://spaces/MAIN-Lab/ai/benchmarks.js
-
curl -L -o benchmarks.js https://huggingface.co/spaces/MAIN-Lab/ai/resolve/main/benchmarks.js
8.56 kB
| // MAIN Lab benchmark registry. Newest first. | |
| // To add a benchmark: copy one object (same format as brand/benchmarks/bNNN.js, plus | |
| // num, date, kind, verdict ("win" | "negative" | "mixed"), headline {v,l} (big number on the card), summary (one short line), links) and put it at the top. | |
| // tests: [title, dataKey, scale, decimals, lowerWins, note, mode] mode: "tie" = not significant, "none" = no winner | |
| window.MAIN_BENCHMARKS = [ | |
| { | |
| "id": "Benchmark #003", | |
| "title": "Weight-space programming,", | |
| "accent": "working.", | |
| "sub": "Benchmark #003 · Qwen2.5-0.5B-Instruct · 197 unseen Azerbaijani prompts · no training, weights edited directly", | |
| "note": "Edit chosen on practice prompts only (19 of 50 probes) · fluency judged by SIBA Meristem (native Azerbaijani LM) · <b>#1</b> = winner", | |
| "models": [ | |
| { | |
| "k": "base", | |
| "short": "Qwen", | |
| "img": "qwen-color.svg" | |
| }, | |
| { | |
| "k": "prompted", | |
| "short": "+ Prompt", | |
| "img": "qwen-color.svg" | |
| }, | |
| { | |
| "k": "edited", | |
| "short": "Ours", | |
| "ours": true | |
| } | |
| ], | |
| "tests": [ | |
| [ | |
| "Clean Azerbaijani", | |
| "clean", | |
| 1, | |
| 1, | |
| false, | |
| "% of answers" | |
| ], | |
| [ | |
| "Answered in Turkish", | |
| "tr", | |
| 1, | |
| 1, | |
| true, | |
| "↓ lower wins" | |
| ], | |
| [ | |
| "Turkish words", | |
| "leak", | |
| 1, | |
| 1, | |
| true, | |
| "per 100 words ↓" | |
| ], | |
| [ | |
| "English kept English", | |
| "en", | |
| 1, | |
| 0, | |
| false, | |
| "% · selectivity" | |
| ], | |
| [ | |
| "Fluency", | |
| "bpb", | |
| 1, | |
| 3, | |
| true, | |
| "bits/byte ↓", | |
| "none" | |
| ], | |
| [ | |
| "Reading QA (Az)", | |
| "qa", | |
| 100, | |
| 1, | |
| false, | |
| "F1 · n=100", | |
| "tie" | |
| ], | |
| [ | |
| "Grammar (Az)", | |
| "mp", | |
| 100, | |
| 1, | |
| false, | |
| "chance 50", | |
| "tie" | |
| ] | |
| ], | |
| "data": { | |
| "base": { | |
| "clean": 55.32994923857868, | |
| "tr": 41.6243654822335, | |
| "leak": 8.395185384148109, | |
| "en": 100, | |
| "bpb": 1.3322746987828893, | |
| "qa": 0.1300687935298941, | |
| "mp": 0.515 | |
| }, | |
| "prompted": { | |
| "clean": 57.36040609137056, | |
| "tr": 36.54822335025381, | |
| "leak": 6.660425417629274, | |
| "en": 100, | |
| "bpb": 1.325934701873973, | |
| "qa": null, | |
| "mp": null | |
| }, | |
| "edited": { | |
| "clean": 77.15736040609137, | |
| "tr": 18.274111675126903, | |
| "leak": 3.900726450831416, | |
| "en": 100, | |
| "bpb": 1.3566729530293398, | |
| "qa": 0.10965533322886264, | |
| "mp": 0.485 | |
| } | |
| }, | |
| "tag": "EXPERIMENT · IT WORKED", | |
| "hero": [ | |
| { | |
| "v": "42% → 18%", | |
| "l": "answers in Turkish" | |
| }, | |
| { | |
| "v": "55% → 77%", | |
| "l": "clean Azerbaijani answers" | |
| }, | |
| { | |
| "v": "128", | |
| "l": "numbers changed, no training" | |
| } | |
| ], | |
| "num": "003", | |
| "date": "2026-11-17", | |
| "kind": "Weight-space programming", | |
| "verdict": "win", | |
| "summary": "128 numbers changed. No training. English untouched.", | |
| "links": [ | |
| { | |
| "label": "Model on Hugging Face", | |
| "href": "https://huggingface.co/MAIN-Lab/qwen2.5-0.5b-az-steer" | |
| }, | |
| { | |
| "label": "Before / after answers", | |
| "href": "https://huggingface.co/spaces/MAIN-Lab/az-steer" | |
| } | |
| ], | |
| "headline": { | |
| "v": "42→18%", | |
| "l": "answers in Turkish" | |
| } | |
| }, | |
| { | |
| "id": "Benchmark #002", | |
| "title": "Weight-space programming,", | |
| "accent": "tested.", | |
| "sub": "Benchmark #002 · Qwen2.5-0.5B-Instruct · MATH-500 · no gradients, weights edited directly", | |
| "note": "MATH-500: 500 problems, greedy, 1,024-token limit · practice: 100 MATH-train problems · <b>#1</b> = winner", | |
| "models": [ | |
| { | |
| "k": "base", | |
| "short": "Qwen", | |
| "img": "qwen-color.svg" | |
| }, | |
| { | |
| "k": "edit", | |
| "short": "Ours", | |
| "ours": true | |
| } | |
| ], | |
| "tests": [ | |
| [ | |
| "MATH-500", | |
| "acc", | |
| 100, | |
| 1, | |
| false, | |
| "accuracy · p = 0.76", | |
| "tie" | |
| ], | |
| [ | |
| "Stuck in a loop", | |
| "truncated", | |
| 100, | |
| 1, | |
| true, | |
| "↓ lower wins" | |
| ], | |
| [ | |
| "No final answer", | |
| "no_box", | |
| 100, | |
| 1, | |
| true, | |
| "↓ lower wins" | |
| ], | |
| [ | |
| "Practice set", | |
| "acc_p", | |
| 100, | |
| 1, | |
| false, | |
| "accuracy" | |
| ], | |
| [ | |
| "Weights changed", | |
| "pct", | |
| 1, | |
| 3, | |
| true, | |
| "% of model", | |
| "none" | |
| ] | |
| ], | |
| "data": { | |
| "base": { | |
| "acc": 0.258, | |
| "truncated": 0.12, | |
| "no_box": 0.126, | |
| "acc_p": 0.32, | |
| "flip": 47, | |
| "pct": 0 | |
| }, | |
| "edit": { | |
| "acc": 0.264, | |
| "truncated": 0.154, | |
| "no_box": 0.164, | |
| "acc_p": 0.24, | |
| "flip": 50, | |
| "pct": 0.023 | |
| } | |
| }, | |
| "cols": 5, | |
| "theme": "dark", | |
| "tag": "EXPERIMENT · NEGATIVE RESULT", | |
| "hero": [ | |
| { | |
| "v": "TIE", | |
| "l": "MATH-500: 25.8% → 26.4% (p = 0.76)" | |
| }, | |
| { | |
| "v": "50 / 47", | |
| "l": "problems fixed / broken by the edit" | |
| }, | |
| { | |
| "v": "12 of 50", | |
| "l": "probes used, practice set only" | |
| } | |
| ], | |
| "num": "002", | |
| "date": "2026-11-17", | |
| "kind": "Weight-space programming", | |
| "verdict": "negative", | |
| "summary": "Editing weights for math did not work. Published anyway.", | |
| "links": [], | |
| "headline": { | |
| "v": "TIE", | |
| "l": "on MATH-500, p = 0.76" | |
| } | |
| }, | |
| { | |
| "id": "Benchmark #001", | |
| "tag": "BENCHMARK · LEADERBOARD", | |
| "title": "Small models,", | |
| "accent": "Azerbaijani.", | |
| "sub": "Benchmark #001 · 5 models · 11 tests", | |
| "note": "Same 11 tests for every model · <b>#1</b> = winner", | |
| "models": [ | |
| { | |
| "k": "tuned", | |
| "short": "Ours", | |
| "ours": true | |
| }, | |
| { | |
| "k": "base", | |
| "short": "Gemma", | |
| "img": "gemma-color.svg" | |
| }, | |
| { | |
| "k": "gpt2az", | |
| "short": "GPT-2", | |
| "img": "openai.svg" | |
| }, | |
| { | |
| "k": "qwen3_aze", | |
| "short": "Qwen", | |
| "img": "qwen-color.svg" | |
| }, | |
| { | |
| "k": "meristem_native", | |
| "short": "SIBA", | |
| "txt": "SIBA" | |
| } | |
| ], | |
| "tests": [ | |
| [ | |
| "Az vs Turkish", | |
| "az_tr", | |
| 100, | |
| 1, | |
| false, | |
| "chance 50" | |
| ], | |
| [ | |
| "Grammar", | |
| "minimal_pairs", | |
| 100, | |
| 1, | |
| false, | |
| "chance 50" | |
| ], | |
| [ | |
| "Bits per byte", | |
| "bpb", | |
| 1, | |
| 2, | |
| true, | |
| "↓ lower wins" | |
| ], | |
| [ | |
| "Translation EN→AZ", | |
| "translate", | |
| 1, | |
| 1, | |
| false, | |
| "chrF++" | |
| ], | |
| [ | |
| "Reading QA", | |
| "extractive_qa", | |
| 100, | |
| 1, | |
| false, | |
| "F1" | |
| ], | |
| [ | |
| "Belebele", | |
| "belebele", | |
| 100, | |
| 1, | |
| false, | |
| "chance 25" | |
| ], | |
| [ | |
| "INCLUDE", | |
| "include", | |
| 100, | |
| 1, | |
| false, | |
| "chance 25" | |
| ], | |
| [ | |
| "Az facts", | |
| "knowledge", | |
| 100, | |
| 1, | |
| false, | |
| "chance 5" | |
| ], | |
| [ | |
| "Sentiment", | |
| "sentiment", | |
| 100, | |
| 1, | |
| false, | |
| "chance 33" | |
| ], | |
| [ | |
| "NLI", | |
| "nli", | |
| 100, | |
| 1, | |
| false, | |
| "chance 33" | |
| ], | |
| [ | |
| "RAG check", | |
| "rag_verify", | |
| 100, | |
| 1, | |
| false, | |
| "one-answer 50" | |
| ] | |
| ], | |
| "data": { | |
| "tuned": { | |
| "minimal_pairs": 0.8, | |
| "az_tr": 0.86, | |
| "bpb": 1.6082452111633438, | |
| "belebele": 0.31333333333333335, | |
| "include": 0.25547445255474455, | |
| "knowledge": 0.04666666666666667, | |
| "sentiment": 0.34, | |
| "nli": 0.34, | |
| "rag_verify": 0.25, | |
| "translate": 25.12997908359113, | |
| "extractive_qa": 0.1846032313724571 | |
| }, | |
| "base": { | |
| "minimal_pairs": 0.515, | |
| "az_tr": 0.49, | |
| "bpb": 2.8939, | |
| "belebele": 0.3244, | |
| "include": 0.2664, | |
| "knowledge": 0.06, | |
| "sentiment": 0.34, | |
| "nli": 0.34, | |
| "rag_verify": 0.25, | |
| "translate": 13.954647273559237, | |
| "extractive_qa": 0.04242065027049547 | |
| }, | |
| "gpt2az": { | |
| "minimal_pairs": 0.495, | |
| "az_tr": 0.895, | |
| "bpb": null, | |
| "belebele": 0.24, | |
| "include": 0.24087591240875914, | |
| "knowledge": 0.04666666666666667, | |
| "sentiment": 0.26, | |
| "nli": 0.32, | |
| "rag_verify": 0.5, | |
| "translate": 1.7315151459110896, | |
| "extractive_qa": 0 | |
| }, | |
| "qwen3_aze": { | |
| "minimal_pairs": 0.59, | |
| "az_tr": 0.58, | |
| "bpb": 2.396953399250009, | |
| "belebele": 0.30444444444444446, | |
| "include": 0.29927007299270075, | |
| "knowledge": 0.06, | |
| "sentiment": 0.34, | |
| "nli": 0.37, | |
| "rag_verify": 0.25, | |
| "translate": 15.237250307254, | |
| "extractive_qa": 0.24139966282309083 | |
| }, | |
| "meristem_native": { | |
| "minimal_pairs": 0.995, | |
| "az_tr": 0.99, | |
| "bpb": 1.0795030482726, | |
| "belebele": 0.29777777777777775, | |
| "include": 0.29927007299270075, | |
| "knowledge": 0.05333333333333334, | |
| "sentiment": 0.34, | |
| "nli": 0.32, | |
| "rag_verify": 0.5, | |
| "translate": 7.341418029205259, | |
| "extractive_qa": 0.03356658901096682 | |
| } | |
| }, | |
| "num": "001", | |
| "date": "2026-11-16", | |
| "kind": "Fine-tuning", | |
| "verdict": "win", | |
| "summary": "Gemma 3 270M, fine-tuned by Claude on a CPU in 2 hours.", | |
| "links": [ | |
| { | |
| "label": "Model on Hugging Face", | |
| "href": "https://huggingface.co/MAIN-Lab/gemma-3-270m-az" | |
| } | |
| ], | |
| "headline": { | |
| "v": "#1", | |
| "l": "of 5 in EN→AZ translation" | |
| } | |
| } | |
| ]; | |