ai / benchmarks.js
Turan0k's picture
MAIN Lab website
ff587a6 verified
Raw History Blame Contribute Delete
8.56 kB
// MAIN Lab benchmark registry. Newest first.
// To add a benchmark: copy one object (same format as brand/benchmarks/bNNN.js, plus
// num, date, kind, verdict ("win" | "negative" | "mixed"), headline {v,l} (big number on the card), summary (one short line), links) and put it at the top.
// tests: [title, dataKey, scale, decimals, lowerWins, note, mode] mode: "tie" = not significant, "none" = no winner
window.MAIN_BENCHMARKS = [
{
"id": "Benchmark #003",
"title": "Weight-space programming,",
"accent": "working.",
"sub": "Benchmark #003 · Qwen2.5-0.5B-Instruct · 197 unseen Azerbaijani prompts · no training, weights edited directly",
"note": "Edit chosen on practice prompts only (19 of 50 probes) · fluency judged by SIBA Meristem (native Azerbaijani LM) · <b>#1</b> = winner",
"models": [
{
"k": "base",
"short": "Qwen",
"img": "qwen-color.svg"
},
{
"k": "prompted",
"short": "+ Prompt",
"img": "qwen-color.svg"
},
{
"k": "edited",
"short": "Ours",
"ours": true
}
],
"tests": [
[
"Clean Azerbaijani",
"clean",
1,
1,
false,
"% of answers"
],
[
"Answered in Turkish",
"tr",
1,
1,
true,
"↓ lower wins"
],
[
"Turkish words",
"leak",
1,
1,
true,
"per 100 words ↓"
],
[
"English kept English",
"en",
1,
0,
false,
"% · selectivity"
],
[
"Fluency",
"bpb",
1,
3,
true,
"bits/byte ↓",
"none"
],
[
"Reading QA (Az)",
"qa",
100,
1,
false,
"F1 · n=100",
"tie"
],
[
"Grammar (Az)",
"mp",
100,
1,
false,
"chance 50",
"tie"
]
],
"data": {
"base": {
"clean": 55.32994923857868,
"tr": 41.6243654822335,
"leak": 8.395185384148109,
"en": 100,
"bpb": 1.3322746987828893,
"qa": 0.1300687935298941,
"mp": 0.515
},
"prompted": {
"clean": 57.36040609137056,
"tr": 36.54822335025381,
"leak": 6.660425417629274,
"en": 100,
"bpb": 1.325934701873973,
"qa": null,
"mp": null
},
"edited": {
"clean": 77.15736040609137,
"tr": 18.274111675126903,
"leak": 3.900726450831416,
"en": 100,
"bpb": 1.3566729530293398,
"qa": 0.10965533322886264,
"mp": 0.485
}
},
"tag": "EXPERIMENT · IT WORKED",
"hero": [
{
"v": "42% → 18%",
"l": "answers in Turkish"
},
{
"v": "55% → 77%",
"l": "clean Azerbaijani answers"
},
{
"v": "128",
"l": "numbers changed, no training"
}
],
"num": "003",
"date": "2026-11-17",
"kind": "Weight-space programming",
"verdict": "win",
"summary": "128 numbers changed. No training. English untouched.",
"links": [
{
"label": "Model on Hugging Face",
"href": "https://huggingface.co/MAIN-Lab/qwen2.5-0.5b-az-steer"
},
{
"label": "Before / after answers",
"href": "https://huggingface.co/spaces/MAIN-Lab/az-steer"
}
],
"headline": {
"v": "42→18%",
"l": "answers in Turkish"
}
},
{
"id": "Benchmark #002",
"title": "Weight-space programming,",
"accent": "tested.",
"sub": "Benchmark #002 · Qwen2.5-0.5B-Instruct · MATH-500 · no gradients, weights edited directly",
"note": "MATH-500: 500 problems, greedy, 1,024-token limit · practice: 100 MATH-train problems · <b>#1</b> = winner",
"models": [
{
"k": "base",
"short": "Qwen",
"img": "qwen-color.svg"
},
{
"k": "edit",
"short": "Ours",
"ours": true
}
],
"tests": [
[
"MATH-500",
"acc",
100,
1,
false,
"accuracy · p = 0.76",
"tie"
],
[
"Stuck in a loop",
"truncated",
100,
1,
true,
"↓ lower wins"
],
[
"No final answer",
"no_box",
100,
1,
true,
"↓ lower wins"
],
[
"Practice set",
"acc_p",
100,
1,
false,
"accuracy"
],
[
"Weights changed",
"pct",
1,
3,
true,
"% of model",
"none"
]
],
"data": {
"base": {
"acc": 0.258,
"truncated": 0.12,
"no_box": 0.126,
"acc_p": 0.32,
"flip": 47,
"pct": 0
},
"edit": {
"acc": 0.264,
"truncated": 0.154,
"no_box": 0.164,
"acc_p": 0.24,
"flip": 50,
"pct": 0.023
}
},
"cols": 5,
"theme": "dark",
"tag": "EXPERIMENT · NEGATIVE RESULT",
"hero": [
{
"v": "TIE",
"l": "MATH-500: 25.8% → 26.4% (p = 0.76)"
},
{
"v": "50 / 47",
"l": "problems fixed / broken by the edit"
},
{
"v": "12 of 50",
"l": "probes used, practice set only"
}
],
"num": "002",
"date": "2026-11-17",
"kind": "Weight-space programming",
"verdict": "negative",
"summary": "Editing weights for math did not work. Published anyway.",
"links": [],
"headline": {
"v": "TIE",
"l": "on MATH-500, p = 0.76"
}
},
{
"id": "Benchmark #001",
"tag": "BENCHMARK · LEADERBOARD",
"title": "Small models,",
"accent": "Azerbaijani.",
"sub": "Benchmark #001 · 5 models · 11 tests",
"note": "Same 11 tests for every model · <b>#1</b> = winner",
"models": [
{
"k": "tuned",
"short": "Ours",
"ours": true
},
{
"k": "base",
"short": "Gemma",
"img": "gemma-color.svg"
},
{
"k": "gpt2az",
"short": "GPT-2",
"img": "openai.svg"
},
{
"k": "qwen3_aze",
"short": "Qwen",
"img": "qwen-color.svg"
},
{
"k": "meristem_native",
"short": "SIBA",
"txt": "SIBA"
}
],
"tests": [
[
"Az vs Turkish",
"az_tr",
100,
1,
false,
"chance 50"
],
[
"Grammar",
"minimal_pairs",
100,
1,
false,
"chance 50"
],
[
"Bits per byte",
"bpb",
1,
2,
true,
"↓ lower wins"
],
[
"Translation EN→AZ",
"translate",
1,
1,
false,
"chrF++"
],
[
"Reading QA",
"extractive_qa",
100,
1,
false,
"F1"
],
[
"Belebele",
"belebele",
100,
1,
false,
"chance 25"
],
[
"INCLUDE",
"include",
100,
1,
false,
"chance 25"
],
[
"Az facts",
"knowledge",
100,
1,
false,
"chance 5"
],
[
"Sentiment",
"sentiment",
100,
1,
false,
"chance 33"
],
[
"NLI",
"nli",
100,
1,
false,
"chance 33"
],
[
"RAG check",
"rag_verify",
100,
1,
false,
"one-answer 50"
]
],
"data": {
"tuned": {
"minimal_pairs": 0.8,
"az_tr": 0.86,
"bpb": 1.6082452111633438,
"belebele": 0.31333333333333335,
"include": 0.25547445255474455,
"knowledge": 0.04666666666666667,
"sentiment": 0.34,
"nli": 0.34,
"rag_verify": 0.25,
"translate": 25.12997908359113,
"extractive_qa": 0.1846032313724571
},
"base": {
"minimal_pairs": 0.515,
"az_tr": 0.49,
"bpb": 2.8939,
"belebele": 0.3244,
"include": 0.2664,
"knowledge": 0.06,
"sentiment": 0.34,
"nli": 0.34,
"rag_verify": 0.25,
"translate": 13.954647273559237,
"extractive_qa": 0.04242065027049547
},
"gpt2az": {
"minimal_pairs": 0.495,
"az_tr": 0.895,
"bpb": null,
"belebele": 0.24,
"include": 0.24087591240875914,
"knowledge": 0.04666666666666667,
"sentiment": 0.26,
"nli": 0.32,
"rag_verify": 0.5,
"translate": 1.7315151459110896,
"extractive_qa": 0
},
"qwen3_aze": {
"minimal_pairs": 0.59,
"az_tr": 0.58,
"bpb": 2.396953399250009,
"belebele": 0.30444444444444446,
"include": 0.29927007299270075,
"knowledge": 0.06,
"sentiment": 0.34,
"nli": 0.37,
"rag_verify": 0.25,
"translate": 15.237250307254,
"extractive_qa": 0.24139966282309083
},
"meristem_native": {
"minimal_pairs": 0.995,
"az_tr": 0.99,
"bpb": 1.0795030482726,
"belebele": 0.29777777777777775,
"include": 0.29927007299270075,
"knowledge": 0.05333333333333334,
"sentiment": 0.34,
"nli": 0.32,
"rag_verify": 0.5,
"translate": 7.341418029205259,
"extractive_qa": 0.03356658901096682
}
},
"num": "001",
"date": "2026-11-16",
"kind": "Fine-tuning",
"verdict": "win",
"summary": "Gemma 3 270M, fine-tuned by Claude on a CPU in 2 hours.",
"links": [
{
"label": "Model on Hugging Face",
"href": "https://huggingface.co/MAIN-Lab/gemma-3-270m-az"
}
],
"headline": {
"v": "#1",
"l": "of 5 in EN→AZ translation"
}
}
];