PugarHuda commited on
Commit
aa5e438
·
verified ·
1 Parent(s): 03d12fa

Benak space

Browse files
Files changed (2) hide show
  1. index.html +139 -120
  2. results.json +1 -1
index.html CHANGED
@@ -1,120 +1,139 @@
1
- <!DOCTYPE html>
2
- <html lang="en">
3
- <head>
4
- <meta charset="utf-8">
5
- <meta name="viewport" content="width=device-width, initial-scale=1">
6
- <title>Benak — what language does Beam think in?</title>
7
- <style>
8
- :root{--surface:#fcfcfb;--page:#f9f9f7;--ink:#0b0b0b;--ink2:#52514e;--muted:#898781;--grid:#e1e0d9;--axis:#c3c2b7;
9
- --s1:#2a78d6;--s2:#eb6834;--seq0:#cde2fb;--seq1:#0d366b;--mark:#fde7dc;--markline:#eb6834}
10
- @media (prefers-color-scheme:dark){:root{--surface:#1a1a19;--page:#0d0d0d;--ink:#f0efec;--ink2:#c3c2b7;--muted:#898781;
11
- --grid:#2c2c2a;--axis:#383835;--s1:#3987e5;--s2:#d95926;--seq0:#184f95;--seq1:#cde2fb;--mark:#3a2418;--markline:#d95926}}
12
- *{box-sizing:border-box}body{margin:0;background:var(--page);color:var(--ink);font:15px/1.55 system-ui,-apple-system,Segoe UI,sans-serif}
13
- main{max-width:1080px;margin:0 auto;padding:32px 20px 80px}
14
- h1{font-size:30px;margin:0 0 4px}h2{font-size:21px;margin:44px 0 6px}h3{font-size:16px;margin:20px 0 6px}
15
- .sub{color:var(--ink2);margin:0 0 18px}.card{background:var(--surface);border:1px solid var(--grid);border-radius:10px;padding:18px;margin:14px 0}
16
- .tiles{display:grid;grid-template-columns:repeat(auto-fit,minmax(200px,1fr));gap:12px}.tile .v{font-size:34px;font-weight:650}.tile .l{color:var(--ink2);font-size:13px}
17
- table{border-collapse:separate;border-spacing:2px;font-size:13px}th{font-weight:600;color:var(--ink2);text-align:left;padding:4px 8px}
18
- td.c{width:86px;height:34px;text-align:center;border-radius:4px;cursor:default;font-variant-numeric:tabular-nums}
19
- .legend{display:flex;align-items:center;gap:8px;color:var(--ink2);font-size:13px;margin:8px 0}
20
- .ramp{width:180px;height:10px;border-radius:5px;background:linear-gradient(90deg,var(--seq0),var(--seq1))}
21
- #tip{position:fixed;pointer-events:none;background:var(--surface);color:var(--ink);border:1px solid var(--axis);border-radius:8px;
22
- padding:8px 10px;font-size:13px;box-shadow:0 4px 14px rgba(0,0,0,.15);display:none;z-index:9;max-width:280px}
23
- select,button{font:inherit;padding:5px 8px;border-radius:6px;border:1px solid var(--axis);background:var(--surface);color:var(--ink)}
24
- .trace{white-space:pre-wrap;font-size:14px;max-height:420px;overflow:auto;border-top:1px solid var(--grid);padding-top:10px}
25
- .seg-eng{background:var(--mark);border-bottom:2px solid var(--markline)}.seg-tgt{}.seg-oth{color:var(--muted)}
26
- .verdict{display:inline-block;padding:1px 8px;border-radius:10px;font-size:12px;font-weight:600;border:1px solid var(--axis)}
27
- .pass::before{content:"✓ "}.fail::before{content:"✗ "}code{font-size:13px}a{color:var(--s1)}
28
- .bars{display:grid;grid-template-columns:120px 1fr;gap:6px 10px;align-items:center;font-size:13px}
29
- .bar{height:14px;border-radius:0 4px 4px 0;margin:2px 0}.muted{color:var(--muted)}.scroll{overflow-x:auto}
30
- </style>
31
- </head>
32
- <body><main>
33
- <h1>Benak</h1>
34
- <p class="sub"><i>benak</i> (Indonesian): the mind, where thinking happens. <b>Does Beam think in your language?</b>
35
- We measured the language of Beam-1.0-501B-A23B's <code>reasoning_content</code> across 19 languages and 4 prompting conditions, re-analysed 4,139 Beam traces other entrants published, and checked whether a grammar book placed in Beam's 1M-token context lets it work in Buginese.</p>
36
- <div class="tiles" id="tiles"></div>
37
-
38
- <h2>1 · The atlas: share of Beam's reasoning written in English</h2>
39
- <p class="sub">Belebele reading comprehension, 40 parallel items per language, effort=low. Every cell is the mean share of reasoning text that GlotLID labels English. Darker means more English. Hover a cell for accuracy and n.</p>
40
- <div class="card scroll"><div class="legend">0% English <span class="ramp"></span> 100% English</div><table id="heat"></table>
41
- <p class="muted" style="font-size:12px">C0 no language instruction · C1 "reason in X" in the user turn · C2 the same instruction as a system prompt · C3 C2 plus one earlier turn whose <code>reasoning_content</code> is written in X.</p></div>
42
-
43
- <h3>Read the traces</h3>
44
- <div class="card"><select id="tl"></select> <select id="tc"></select> <button id="tn">another trace</button>
45
- <span class="legend" style="display:inline-flex;margin-left:12px"><span class="seg-eng">&nbsp;English&nbsp;</span> <span>target language</span> <span class="seg-oth">other</span></span>
46
- <div id="tmeta" class="muted" style="font-size:13px;margin-top:8px"></div><div class="trace" id="trace"></div></div>
47
-
48
- <h2>2 · Math in 9 languages: does forcing the language cost accuracy?</h2>
49
- <p class="sub">58 word problems, each answer checked against a Python solver that we ran, translated with every number preserved. C0 has no language instruction; C2 puts "reason in X" in the system prompt.</p>
50
- <div class="card scroll"><table id="mathtab"></table></div>
51
-
52
- <h2>3 · Other entrants' traces, re-read for free</h2>
53
- <p class="sub">We applied the same detector to Beam traces that neulketing (Korean) and esscor (TamilBench) had already published. These results cost $0 of inference.</p>
54
- <div class="card scroll"><table id="reptab"></table></div>
55
-
56
- <h2 id="g-h">4 · Can a grammar book in context teach Beam Buginese?</h2>
57
- <p class="sub" id="g-sub"></p>
58
- <div class="card" id="gram"></div>
59
-
60
- <h2>Pre-registered hypotheses</h2>
61
- <div class="card" id="verd"></div>
62
- <p class="muted">Data, code, pre-registration and the Southeast-Asian thinking dataset: <a href="https://huggingface.co/datasets/PugarHuda/benak">huggingface.co/datasets/PugarHuda/benak</a></p>
63
- </main><div id="tip"></div>
64
- <script>
65
- const $=s=>document.querySelector(s), tip=$('#tip');
66
- const LN={eng_Latn:'English',ind_Latn:'Indonesian',jav_Latn:'Javanese',sun_Latn:'Sundanese',zsm_Latn:'Malay',tgl_Latn:'Tagalog',ceb_Latn:'Cebuano',vie_Latn:'Vietnamese',tha_Thai:'Thai',khm_Khmr:'Khmer',lao_Laoo:'Lao',mya_Mymr:'Burmese',hin_Deva:'Hindi',tam_Taml:'Tamil',kor_Hang:'Korean',jpn_Jpan:'Japanese',arb_Arab:'Arabic',yor_Latn:'Yoruba',plt_Latn:'Malagasy'};
67
- const pct=x=>x==null?'–':Math.round(100*x)+'%';
68
- function lerp(a,b,t){const p=h=>[1,3,5].map(i=>parseInt(h.slice(i,i+2),16));const A=p(a),B=p(b);return `rgb(${A.map((v,i)=>Math.round(v+(B[i]-v)*t)).join(',')})`}
69
- const css=v=>getComputedStyle(document.documentElement).getPropertyValue(v).trim();
70
- function showTip(e,html){tip.innerHTML=html;tip.style.display='block';tip.style.left=Math.min(e.clientX+14,innerWidth-300)+'px';tip.style.top=(e.clientY+14)+'px'}
71
- document.addEventListener('mouseover',e=>{if(!e.target.dataset.tip)tip.style.display='none'});
72
- Promise.all([fetch('results.json').then(r=>r.json()),fetch('samples.json').then(r=>r.json())]).then(([R,S])=>{
73
- const A=R.atlas, conds=['C0','C1','C2','C3'];
74
- const langs=Object.keys(LN).filter(l=>A[`${l}|C0|low`]);
75
- // tiles
76
- const non=langs.filter(l=>l!=='eng_Latn'), c0=non.map(l=>A[`${l}|C0|low`].eng_share).filter(x=>x!=null);
77
- const best=non.map(l=>Math.max(...conds.map(c=>(A[`${l}|${c}|low`]||{}).target_share||0)));
78
- const ko=(R.replication||[]).find(s=>s.cond==='belebele_ko');
79
- $('#tiles').innerHTML=[[pct(c0.reduce((a,b)=>a+b,0)/c0.length),'of Beam\'s reasoning is English when the prompt is in another language (C0 mean, 18 languages)'],
80
- [pct(best.reduce((a,b)=>a+b,0)/best.length),'mean in-language share in the best condition per language'],
81
- [ko?pct(ko.mean_eng_share):'–','English share in 900 published Korean Belebele traces (neulketing)'],
82
- ['$'+(R.cost_usd??'?'),'total inference spend for this project']].map(([v,l])=>`<div class="card tile"><div class="v">${v}</div><div class="l">${l}</div></div>`).join('');
83
- // heatmap
84
- const s0=css('--seq0'),s1=css('--seq1');
85
- let h='<tr><th>language</th>'+conds.map(c=>`<th>${c}</th>`).join('')+'<th>acc C0</th></tr>';
86
- for(const l of langs){h+=`<tr><th>${LN[l]}</th>`;for(const c of conds){const d=A[`${l}|${c}|low`];if(!d){h+='<td class="c">–</td>';continue}
87
- const v=d.eng_share??0, bg=lerp(s0,s1,v), ink=(v>.55)===(!matchMedia('(prefers-color-scheme:dark)').matches)?'#fff':'#0b0b0b';
88
- h+=`<td class="c" style="background:${bg};color:${ink}" data-tip="1" data-l="${l}" data-c="${c}">${pct(d.eng_share)}</td>`}
89
- h+=`<td class="muted" style="padding:0 8px">${A[`${l}|C0|low`].acc}%</td></tr>`}
90
- $('#heat').innerHTML=h;
91
- $('#heat').addEventListener('mousemove',e=>{const t=e.target;if(!t.dataset.l)return;const d=A[`${t.dataset.l}|${t.dataset.c}|low`];
92
- showTip(e,`<b>${LN[t.dataset.l]} · ${t.dataset.c}</b><br>English: ${pct(d.eng_share)} · in-language: ${pct(d.target_share)}<br>mostly in-language traces: ${d.pct_mostly_target??'–'}%<br>accuracy ${d.acc}% · n=${d.n}<br>reasoning tokens ≈ ${Math.round(d.reasoning_tokens||0)}`)});
93
- // trace viewer
94
- $('#tl').innerHTML=langs.filter(l=>S[l]).map(l=>`<option value="${l}">${LN[l]}</option>`).join('');
95
- $('#tc').innerHTML=conds.map(c=>`<option>${c}</option>`).join('');let k=0;
96
- const draw=()=>{const L=$('#tl').value,C=$('#tc').value,arr=(S[L]||{})[C]||[];if(!arr.length){$('#trace').textContent='(no sample)';return}
97
- const t=arr[k%arr.length];$('#tmeta').textContent=`answer ${t.correct?'correct':'wrong'} · English ${pct(t.eng_share)} · in-language ${pct(t.target_share)}`;
98
- $('#trace').innerHTML=t.segs.map(([s,kind])=>`<span class="seg-${kind}">${s.replace(/&/g,'&amp;').replace(/</g,'&lt;')}</span>`).join(' ')};
99
- $('#tl').onchange=$('#tc').onchange=()=>{k=0;draw()};$('#tn').onclick=()=>{k++;draw()};$('#tl').value='ind_Latn';draw();
100
- // math table
101
- const M=R.math, ml=Object.keys(LN).filter(l=>M[`${l}|C0|low`]);
102
- $('#mathtab').innerHTML='<tr><th>language</th><th>acc C0</th><th>acc C2</th><th>English share C0</th><th>English share C2</th><th>reasoning tokens C0</th></tr>'+
103
- ml.map(l=>{const a=M[`${l}|C0|low`],b=M[`${l}|C2|low`]||{};return `<tr><th>${LN[l]}</th><td>${a.acc}%</td><td>${b.acc??'–'}%</td><td>${pct(a.eng_share)}</td><td>${pct(b.eng_share)}</td><td>${Math.round(a.reasoning_tokens||0)}</td></tr>`}).join('');
104
- // replication
105
- $('#reptab').innerHTML='<tr><th>source</th><th>language</th><th>condition</th><th>effort</th><th>n</th><th>English share</th><th>mostly in-language</th></tr>'+
106
- (R.replication||[]).filter(s=>s.n>=15).map(s=>`<tr><td>${s.source}</td><td>${LN[s.lang]||s.lang}</td><td>${s.cond}</td><td>${s.effort}</td><td>${s.n}</td><td>${pct(s.mean_eng_share)}</td><td>${s.pct_mostly_target??'–'}%</td></tr>`).join('');
107
- // grammar
108
- const G=R.grammar;if(G){const ks=['K0','K1','K2','K3'],lab={K0:'no context',K1:'dictionary',K2:'grammar',K3:'grammar + dictionary + examples'};
109
- const mx=Math.max(...ks.flatMap(c=>['x2ind','ind2x'].map(d=>(G[`${c}/${d}`]||{})['chrf++']||0)),1);
110
- $('#g-sub').textContent='chrF++ on 100 held-out NusaX sentences (higher is better). Each context condition is shown in both directions; the context grows from nothing to a whole book.';
111
- $('#gram').innerHTML='<div class="legend"><span style="width:12px;height:12px;background:var(--s1);border-radius:3px"></span>Buginese → Indonesian <span style="width:12px;height:12px;background:var(--s2);border-radius:3px;margin-left:10px"></span>Indonesian → Buginese</div><div class="bars">'+
112
- ks.map(c=>`<div>${lab[c]}<br><span class="muted">${((G[c+'/prompt_tokens']||0)/1000).toFixed(0)}k tokens</span></div><div>`+['x2ind','ind2x'].map((d,i)=>{const v=(G[`${c}/${d}`]||{})['chrf++'];
113
- return `<div class="bar" data-tip="1" style="width:${(v||0)/mx*100}%;background:var(--s${i+1})" title="${v}"></div>`}).join('')+
114
- `<span class="muted">${['x2ind','ind2x'].map(d=>(G[`${c}/${d}`]||{})['chrf++']??'–').join(' / ')} · thinks in Buginese: ${pct((G[c+'/think']||{}).target_share)}</span></div>`).join('')+'</div>'}
115
- else $('#g-h').style.display=$('#gram').style.display='none';
116
- // verdicts
117
- $('#verd').innerHTML=Object.entries(R.verdicts||{}).map(([k,v])=>`<p><b>${k}</b> <span class="verdict ${v.pass?'pass':'fail'}">${v.pass?'pass':'fail'}</span> ${v.detail}</p>`).join('');
118
- });
119
- </script>
120
- </body></html>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>Benak — what language does Beam think in?</title>
7
+ <style>
8
+ :root{--surface:#fcfcfb;--page:#f9f9f7;--ink:#0b0b0b;--ink2:#52514e;--muted:#898781;--grid:#e1e0d9;--axis:#c3c2b7;
9
+ --s1:#2a78d6;--s2:#eb6834;--seq0:#cde2fb;--seq1:#0d366b;--mark:#fde7dc;--markline:#eb6834}
10
+ @media (prefers-color-scheme:dark){:root{--surface:#1a1a19;--page:#0d0d0d;--ink:#f0efec;--ink2:#c3c2b7;--muted:#898781;
11
+ --grid:#2c2c2a;--axis:#383835;--s1:#3987e5;--s2:#d95926;--seq0:#184f95;--seq1:#cde2fb;--mark:#3a2418;--markline:#d95926}}
12
+ *{box-sizing:border-box}body{margin:0;background:var(--page);color:var(--ink);font:15px/1.55 system-ui,-apple-system,Segoe UI,sans-serif}
13
+ main{max-width:1080px;margin:0 auto;padding:32px 20px 80px}
14
+ h1{font-size:30px;margin:0 0 4px}h2{font-size:21px;margin:44px 0 6px}h3{font-size:16px;margin:20px 0 6px}
15
+ .sub{color:var(--ink2);margin:0 0 18px}.card{background:var(--surface);border:1px solid var(--grid);border-radius:10px;padding:18px;margin:14px 0}
16
+ .tiles{display:grid;grid-template-columns:repeat(auto-fit,minmax(200px,1fr));gap:12px}.tile .v{font-size:34px;font-weight:650}.tile .l{color:var(--ink2);font-size:13px}
17
+ table{border-collapse:separate;border-spacing:2px;font-size:13px}th{font-weight:600;color:var(--ink2);text-align:left;padding:4px 8px}
18
+ td.c{width:86px;height:34px;text-align:center;border-radius:4px;cursor:default;font-variant-numeric:tabular-nums}
19
+ .legend{display:flex;align-items:center;gap:8px;color:var(--ink2);font-size:13px;margin:8px 0}
20
+ .ramp{width:180px;height:10px;border-radius:5px;background:linear-gradient(90deg,var(--seq0),var(--seq1))}
21
+ #tip{position:fixed;pointer-events:none;background:var(--surface);color:var(--ink);border:1px solid var(--axis);border-radius:8px;
22
+ padding:8px 10px;font-size:13px;box-shadow:0 4px 14px rgba(0,0,0,.15);display:none;z-index:9;max-width:280px}
23
+ select,button{font:inherit;padding:5px 8px;border-radius:6px;border:1px solid var(--axis);background:var(--surface);color:var(--ink)}
24
+ .trace{white-space:pre-wrap;font-size:14px;max-height:420px;overflow:auto;border-top:1px solid var(--grid);padding-top:10px}
25
+ .seg-eng{background:var(--mark);border-bottom:2px solid var(--markline)}.seg-tgt{}.seg-oth{color:var(--muted)}
26
+ .verdict{display:inline-block;padding:1px 8px;border-radius:10px;font-size:12px;font-weight:600;border:1px solid var(--axis)}
27
+ .pass::before{content:"✓ "}.fail::before{content:"✗ "}code{font-size:13px}a{color:var(--s1)}
28
+ .bars{display:grid;grid-template-columns:120px 1fr;gap:6px 10px;align-items:center;font-size:13px}
29
+ .bar{height:14px;border-radius:0 4px 4px 0;margin:2px 0}.muted{color:var(--muted)}.scroll{overflow-x:auto}
30
+ </style>
31
+ </head>
32
+ <body><main>
33
+ <h1>Benak</h1>
34
+ <p class="sub"><i>benak</i> (Indonesian): the mind, where thinking happens. <b>Does Beam think in your language?</b>
35
+ We measured the language of Beam-1.0-501B-A23B's <code>reasoning_content</code> across 19 languages and 4 prompting conditions, re-analysed 4,139 Beam traces other entrants published, and checked whether a grammar book placed in Beam's 1M-token context lets it work in Buginese.</p>
36
+ <div class="tiles" id="tiles"></div>
37
+
38
+ <h2>1 · The atlas: share of Beam's reasoning written in English</h2>
39
+ <p class="sub">Belebele reading comprehension, 40 parallel items per language, effort=low. Every cell is the mean share of reasoning text that GlotLID labels English. Darker means more English. Hover a cell for accuracy and n.</p>
40
+ <div class="card scroll"><div class="legend">0% English <span class="ramp"></span> 100% English</div><table id="heat"></table>
41
+ <p class="muted" style="font-size:12px">C0 no language instruction · C1 "reason in X" in the user turn · C2 the same instruction as a system prompt · C3 C2 plus one earlier turn whose <code>reasoning_content</code> is written in X.</p></div>
42
+
43
+ <h3>Read the traces</h3>
44
+ <div class="card"><select id="tl"></select> <select id="tc"></select> <button id="tn">another trace</button>
45
+ <span class="legend" style="display:inline-flex;margin-left:12px"><span class="seg-eng">&nbsp;English&nbsp;</span> <span>target language</span> <span class="seg-oth">other</span></span>
46
+ <div id="tmeta" class="muted" style="font-size:13px;margin-top:8px"></div><div class="trace" id="trace"></div></div>
47
+
48
+ <h2>2 · Math in 9 languages: does forcing the language cost accuracy?</h2>
49
+ <p class="sub">58 word problems, each answer checked against a Python solver that we ran, translated with every number preserved. C0 has no language instruction; C2 puts "reason in X" in the system prompt.</p>
50
+ <div class="card scroll"><table id="mathtab"></table></div>
51
+
52
+ <h2>3 · Other entrants' traces, re-read for free</h2>
53
+ <p class="sub">We applied the same detector to Beam traces that neulketing (Korean) and esscor (TamilBench) had already published. These results cost $0 of inference.</p>
54
+ <div class="card scroll"><table id="reptab"></table></div>
55
+
56
+ <h2 id="g-h">4 · Can a grammar book in context teach Beam Buginese?</h2>
57
+ <p class="sub" id="g-sub"></p>
58
+ <div class="card" id="gram"></div>
59
+
60
+ <h2>Robustness: is the 60% real?</h2>
61
+ <p class="sub">The pre-registered metric counts every sentence in the reasoning. When Beam quotes the passage or the options, the sentence is in the target language even though Beam is not thinking in it. So we also removed quoted sentences (char 5-gram overlap with the prompt ≥ 60%) and re-measured everything with a second, independent detector (Lingua).</p>
62
+ <div class="card scroll"><table id="robtab"></table><p class="muted" id="agree" style="font-size:13px"></p></div>
63
+ <div id="highsec"><h2>Does more thinking change the language?</h2><p class="sub">Effort=high (32k-token budget) vs effort=low on the same 40 items, C0 and C2.</p><div class="card scroll"><table id="hightab"></table></div></div>
64
+ <h2>Use the fix</h2>
65
+ <div class="card"><p>We ship a 40-line wrapper, <code>benak_lang.think_in</code>, with templates for all 19 languages. It works with any OpenAI-compatible client. The default is <b>C2</b> (system prompt), which is robust across tasks. <code>primer=True</code> gives C3, which is best on reading comprehension, but the math table above shows that its reading-item primer does <b>not</b> transfer to math.</p>
66
+ <pre style="overflow:auto;font-size:13px;background:var(--page);padding:12px;border-radius:8px">from benak_lang import think_in
67
+ msgs = think_in([{"role": "user", "content": "Berapa 17 x 23?"}], "ind_Latn")
68
+ client.chat.completions.create(model="Beam-1.0-501B-A23B", messages=msgs,
69
+ extra_body={"reasoning_effort": "low"})</pre>
70
+ <p>For training, the <code>sea_thinking</code> config holds <b id="nthink"></b> correct traces whose reasoning is ≥ 80% in the target language, in the Multilingual-Thinking schema that Reflection's own TRL recipe uses.</p></div>
71
+ <h2>Pre-registered hypotheses</h2>
72
+ <div class="card" id="verd"></div>
73
+ <p class="muted">Data, code, pre-registration and the Southeast-Asian thinking dataset: <a href="https://huggingface.co/datasets/PugarHuda/benak">huggingface.co/datasets/PugarHuda/benak</a></p>
74
+ </main><div id="tip"></div>
75
+ <script>
76
+ const $=s=>document.querySelector(s), tip=$('#tip');
77
+ const LN={eng_Latn:'English',ind_Latn:'Indonesian',jav_Latn:'Javanese',sun_Latn:'Sundanese',zsm_Latn:'Malay',tgl_Latn:'Tagalog',ceb_Latn:'Cebuano',vie_Latn:'Vietnamese',tha_Thai:'Thai',khm_Khmr:'Khmer',lao_Laoo:'Lao',mya_Mymr:'Burmese',hin_Deva:'Hindi',tam_Taml:'Tamil',kor_Hang:'Korean',jpn_Jpan:'Japanese',arb_Arab:'Arabic',yor_Latn:'Yoruba',plt_Latn:'Malagasy',tam_Latn:'Tanglish (romanized Tamil)'};
78
+ const pct=x=>x==null?'–':Math.round(100*x)+'%';
79
+ function lerp(a,b,t){const p=h=>[1,3,5].map(i=>parseInt(h.slice(i,i+2),16));const A=p(a),B=p(b);return `rgb(${A.map((v,i)=>Math.round(v+(B[i]-v)*t)).join(',')})`}
80
+ const css=v=>getComputedStyle(document.documentElement).getPropertyValue(v).trim();
81
+ function showTip(e,html){tip.innerHTML=html;tip.style.display='block';tip.style.left=Math.min(e.clientX+14,innerWidth-300)+'px';tip.style.top=(e.clientY+14)+'px'}
82
+ document.addEventListener('mouseover',e=>{if(!e.target.dataset.tip)tip.style.display='none'});
83
+ Promise.all([fetch('results.json').then(r=>r.json()),fetch('samples.json').then(r=>r.json())]).then(([R,S])=>{
84
+ const A=R.atlas, conds=['C0','C1','C2','C3'];
85
+ const langs=Object.keys(LN).filter(l=>A[`${l}|C0|low`]);
86
+ // tiles
87
+ const non=langs.filter(l=>l!=='eng_Latn'), c0=non.map(l=>A[`${l}|C0|low`].eng_share).filter(x=>x!=null);
88
+ const best=non.map(l=>Math.max(...conds.map(c=>(A[`${l}|${c}|low`]||{}).target_share||0)));
89
+ const ko=(R.replication||[]).find(s=>s.cond==='belebele_ko');
90
+ $('#tiles').innerHTML=[[pct(c0.reduce((a,b)=>a+b,0)/c0.length),'of Beam\'s reasoning is English when the prompt is in another language (C0 mean, 18 languages)'],
91
+ [pct(best.reduce((a,b)=>a+b,0)/best.length),'mean in-language share in the best condition per language'],
92
+ [ko?pct(ko.mean_eng_share):'–','English share in 900 published Korean Belebele traces (neulketing)'],
93
+ ['$'+(R.cost_usd!=null?R.cost_usd.toFixed(2):'?'),'paid inference for the whole project (the rest ran on the Reflection free daily allowance)']].map(([v,l])=>`<div class="card tile"><div class="v">${v}</div><div class="l">${l}</div></div>`).join('');
94
+ // heatmap
95
+ const s0=css('--seq0'),s1=css('--seq1');
96
+ let h='<tr><th>language</th>'+conds.map(c=>`<th>${c}</th>`).join('')+'<th>acc C0</th></tr>';
97
+ for(const l of langs){h+=`<tr><th>${LN[l]}</th>`;for(const c of conds){const d=A[`${l}|${c}|low`];if(!d){h+='<td class="c">–</td>';continue}
98
+ const v=d.eng_share??0, bg=lerp(s0,s1,v), ink=(v>.55)===(!matchMedia('(prefers-color-scheme:dark)').matches)?'#fff':'#0b0b0b';
99
+ h+=`<td class="c" style="background:${bg};color:${ink}" data-tip="1" data-l="${l}" data-c="${c}">${pct(d.eng_share)}</td>`}
100
+ h+=`<td class="muted" style="padding:0 8px">${A[`${l}|C0|low`].acc}%</td></tr>`}
101
+ $('#heat').innerHTML=h;
102
+ $('#heat').addEventListener('mousemove',e=>{const t=e.target;if(!t.dataset.l)return;const d=A[`${t.dataset.l}|${t.dataset.c}|low`];
103
+ showTip(e,`<b>${LN[t.dataset.l]} · ${t.dataset.c}</b><br>English: ${pct(d.eng_share)} · in-language: ${pct(d.target_share)}<br>mostly in-language traces: ${d.pct_mostly_target??'–'}%<br>accuracy ${d.acc}% · n=${d.n}<br>reasoning tokens ≈ ${Math.round(d.reasoning_tokens||0)}`)});
104
+ // trace viewer
105
+ $('#tl').innerHTML=langs.filter(l=>S[l]).map(l=>`<option value="${l}">${LN[l]}</option>`).join('');
106
+ $('#tc').innerHTML=conds.map(c=>`<option>${c}</option>`).join('');let k=0;
107
+ const draw=()=>{const L=$('#tl').value,C=$('#tc').value,arr=(S[L]||{})[C]||[];if(!arr.length){$('#trace').textContent='(no sample)';return}
108
+ const t=arr[k%arr.length];$('#tmeta').textContent=`answer ${t.correct?'correct':'wrong'} · English ${pct(t.eng_share)} · in-language ${pct(t.target_share)}`;
109
+ $('#trace').innerHTML=t.segs.map(([s,kind])=>`<span class="seg-${kind}">${s.replace(/&/g,'&amp;').replace(/</g,'&lt;')}</span>`).join(' ')};
110
+ $('#tl').onchange=$('#tc').onchange=()=>{k=0;draw()};$('#tn').onclick=()=>{k++;draw()};$('#tl').value='ind_Latn';draw();
111
+ // math table
112
+ const M=R.math, ml=Object.keys(LN).filter(l=>M[`${l}|C0|low`]);
113
+ $('#mathtab').innerHTML='<tr><th>language</th><th>acc C0</th><th>acc C2</th><th>acc C3</th><th>English C0</th><th>English C2</th><th>English C3</th><th>reasoning tokens C0</th></tr>'+
114
+ ml.map(l=>{const a=M[`${l}|C0|low`],b=M[`${l}|C2|low`]||{},c=M[`${l}|C3|low`]||{};return `<tr><th>${LN[l]}</th><td>${a.acc}%</td><td>${b.acc??'–'}%</td><td>${c.acc!=null?c.acc+'%':'–'}</td><td>${pct(a.eng_share)}</td><td>${pct(b.eng_share)}</td><td>${pct(c.eng_share)}</td><td>${Math.round(a.reasoning_tokens||0)}</td></tr>`}).join('')+
115
+ (R.math_mcnemar_C0_vs_C2?`<tr><td colspan="8" class="muted" style="padding-top:8px">Paired C0 vs C2 on the same problem: ${R.math_mcnemar_C0_vs_C2.C0_only_correct} right only without the instruction, ${R.math_mcnemar_C0_vs_C2.C2_only_correct} right only with it (exact McNemar p = ${R.math_mcnemar_C0_vs_C2.p_exact}).</td></tr>`:'');
116
+ // replication
117
+ $('#reptab').innerHTML='<tr><th>source</th><th>language</th><th>condition</th><th>effort</th><th>n</th><th>English share</th><th>mostly in-language</th></tr>'+
118
+ (R.replication||[]).filter(s=>s.n>=15).map(s=>`<tr><td>${s.source}</td><td>${LN[s.lang]||s.lang}</td><td>${s.cond}</td><td>${s.effort}</td><td>${s.n}</td><td>${pct(s.mean_eng_share)}</td><td>${s.pct_mostly_target??'–'}%</td></tr>`).join('');
119
+ // grammar
120
+ const G=R.grammar;if(G){const ks=['K0','K1','K2','K3'],lab={K0:'no context',K1:'dictionary',K2:'grammar',K3:'grammar + dictionary + examples'};
121
+ const mx=Math.max(...ks.flatMap(c=>['x2ind','ind2x'].map(d=>(G[`${c}/${d}`]||{})['chrf++']||0)),1);
122
+ $('#g-sub').textContent='chrF++ on 100 held-out NusaX sentences (higher is better). Each context condition is shown in both directions; the context grows from nothing to a whole book.';
123
+ $('#gram').innerHTML='<div class="legend"><span style="width:12px;height:12px;background:var(--s1);border-radius:3px"></span>Buginese → Indonesian <span style="width:12px;height:12px;background:var(--s2);border-radius:3px;margin-left:10px"></span>Indonesian → Buginese</div><div class="bars">'+
124
+ ks.map(c=>`<div>${lab[c]}<br><span class="muted">${((G[c+'/prompt_tokens']||0)/1000).toFixed(0)}k tokens</span></div><div>`+['x2ind','ind2x'].map((d,i)=>{const v=(G[`${c}/${d}`]||{})['chrf++'];
125
+ return `<div class="bar" data-tip="1" style="width:${(v||0)/mx*100}%;background:var(--s${i+1})" title="${v}"></div>`}).join('')+
126
+ `<span class="muted">${['x2ind','ind2x'].map(d=>(G[`${c}/${d}`]||{})['chrf++']??'–').join(' / ')} · reasoning in Buginese: ${pct((G[c+'/think']||{}).target_share)} raw, <b>${pct((G[c+'/think_noquote']||{}).target_share)}</b> after removing quoted input (rest: ${pct((G[c+'/think_noquote']||{}).other_share)} mostly Indonesian, ${pct((G[c+'/think_noquote']||{}).eng_share)} English)</span></div>`).join('')+'</div>'}
127
+ else $('#g-h').style.display=$('#gram').style.display='none';
128
+ // robustness
129
+ const RB=R.robustness||{}, rl=c=>{const ks=Object.keys(RB).filter(k=>k.endsWith('|'+c)&&!k.startsWith('eng'));const m=f=>{const v=ks.map(k=>RB[k][f]).filter(x=>x!=null);return v.length?v.reduce((a,b)=>a+b,0)/v.length:null};return m};
130
+ $('#robtab').innerHTML='<tr><th>condition</th><th>English share (pre-registered, GlotLID)</th><th>quote-stripped (GlotLID)</th><th>Lingua</th><th>Lingua, quote-stripped</th></tr>'+conds.map(c=>{const m=rl(c);return `<tr><th>${c}</th><td>${pct(m('eng_share_prereg'))}</td><td>${pct(m('eng_share_noquote'))}</td><td>${pct(m('lingua_eng'))}</td><td>${pct(m('lingua_eng_noquote'))}</td></tr>`}).join('');
131
+ if(R.detector_agreement)$('#agree').textContent=`GlotLID and Lingua agree on whether a trace is mostly English in ${pct(R.detector_agreement.same_majority)} of ${R.detector_agreement.n} traces (mean absolute difference ${R.detector_agreement.mean_abs_diff}). Removing quotes raises the English share: Beam quotes in the target language and reasons in English.`;
132
+ const hl=Object.keys(LN).filter(l=>A[`${l}|C0|high`]);
133
+ if(hl.length){$('#hightab').innerHTML='<tr><th>language</th><th>English C0 low → high</th><th>English C2 low → high</th><th>acc C0 low → high</th><th>reasoning tokens C0 low → high</th></tr>'+hl.map(l=>{const g=(c,e,f)=>(A[`${l}|${c}|${e}`]||{})[f];return `<tr><th>${LN[l]}</th><td>${pct(g('C0','low','eng_share'))} → ${pct(g('C0','high','eng_share'))}</td><td>${pct(g('C2','low','eng_share'))} → ${pct(g('C2','high','eng_share'))}</td><td>${g('C0','low','acc')}% → ${g('C0','high','acc')}%</td><td>${Math.round(g('C0','low','reasoning_tokens')||0)} → ${Math.round(g('C0','high','reasoning_tokens')||0)}</td></tr>`}).join('')}else $('#highsec').style.display='none';
134
+ $('#nthink').textContent=(R.sea_thinking_rows??'?').toLocaleString();
135
+ // verdicts
136
+ $('#verd').innerHTML=Object.entries(R.verdicts||{}).map(([k,v])=>`<p><b>${k}</b> <span class="verdict ${v.pass?'pass':'fail'}">${v.pass?'pass':'fail'}</span> ${v.detail}</p>`).join('');
137
+ });
138
+ </script>
139
+ </body></html>
results.json CHANGED
@@ -1 +1 @@
1
- {"atlas": {"eng_Latn|C1|low": {"n": 40, "acc": 97.5, "target_share": 0.972, "eng_share": 0.0, "pct_mostly_target": 100.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 140.85, "completion_tokens": 152.175}, "eng_Latn|C2|low": {"n": 40, "acc": 97.5, "target_share": 0.961, "eng_share": 0.0, "pct_mostly_target": 100.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 155.95, "completion_tokens": 167.55}, "eng_Latn|C3|low": {"n": 40, "acc": 97.5, "target_share": 0.999, "eng_share": 0.0, "pct_mostly_target": 100.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 109.325, "completion_tokens": 120.875}, "jav_Latn|C3|low": {"n": 40, "acc": 92.5, "target_share": 0.777, "eng_share": 0.203, "pct_mostly_target": 82.5, "no_reasoning_signal": 0, "empty_or_truncated": 2, "reasoning_tokens": 394.075, "completion_tokens": 405.15}, "eng_Latn|C0|low": {"n": 40, "acc": 97.5, "target_share": 0.989, "eng_share": 0.0, "pct_mostly_target": 100.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 140, "completion_tokens": 151.45}, "sun_Latn|C3|low": {"n": 40, "acc": 90.0, "target_share": 0.649, "eng_share": 0.272, "pct_mostly_target": 62.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 252.775, "completion_tokens": 263.525}, "sun_Latn|C2|low": {"n": 40, "acc": 92.5, "target_share": 0.34, "eng_share": 0.616, "pct_mostly_target": 30.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 304.075, "completion_tokens": 357.325}, "jav_Latn|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.496, "eng_share": 0.469, "pct_mostly_target": 42.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 373.975, "completion_tokens": 412}, "tgl_Latn|C2|low": {"n": 40, "acc": 95.0, "target_share": 0.477, "eng_share": 0.501, "pct_mostly_target": 50.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 282.8, "completion_tokens": 346.7}, "ceb_Latn|C0|low": {"n": 40, "acc": 95.0, "target_share": 0.438, "eng_share": 0.509, "pct_mostly_target": 52.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 319.325, "completion_tokens": 329.9}, "ceb_Latn|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.445, "eng_share": 0.51, "pct_mostly_target": 47.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 341.3, "completion_tokens": 378.55}, "ceb_Latn|C2|low": {"n": 40, "acc": 95.0, "target_share": 0.457, "eng_share": 0.499, "pct_mostly_target": 47.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 306.275, "completion_tokens": 322.05}, "ind_Latn|C3|low": {"n": 40, "acc": 95.0, "target_share": 0.909, "eng_share": 0.084, "pct_mostly_target": 95.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 231.95, "completion_tokens": 242.975}, "ind_Latn|C2|low": {"n": 40, "acc": 95.0, "target_share": 0.879, "eng_share": 0.11, "pct_mostly_target": 92.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 232.375, "completion_tokens": 248.05}, "ind_Latn|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.508, "eng_share": 0.472, "pct_mostly_target": 47.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 271.1, "completion_tokens": 296.975}, "tgl_Latn|C3|low": {"n": 40, "acc": 95.0, "target_share": 0.738, "eng_share": 0.244, "pct_mostly_target": 75.0, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 360.075, "completion_tokens": 370.675}, "ceb_Latn|C3|low": {"n": 40, "acc": 92.5, "target_share": 0.633, "eng_share": 0.336, "pct_mostly_target": 67.5, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 349.2, "completion_tokens": 359.9}, "vie_Latn|C2|low": {"n": 40, "acc": 92.5, "target_share": 0.767, "eng_share": 0.22, "pct_mostly_target": 80.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 262.2, "completion_tokens": 273.35}, "vie_Latn|C0|low": {"n": 40, "acc": 97.5, "target_share": 0.443, "eng_share": 0.537, "pct_mostly_target": 47.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 265.25, "completion_tokens": 277.025}, "vie_Latn|C1|low": {"n": 40, "acc": 97.5, "target_share": 0.606, "eng_share": 0.379, "pct_mostly_target": 67.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 278.6, "completion_tokens": 307.375}, "tha_Thai|C0|low": {"n": 40, "acc": 85.0, "target_share": 0.312, "eng_share": 0.659, "pct_mostly_target": 27.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 357.5, "completion_tokens": 369.6}, "tha_Thai|C3|low": {"n": 40, "acc": 82.5, "target_share": 0.683, "eng_share": 0.294, "pct_mostly_target": 65.0, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 438.675, "completion_tokens": 450.675}, "tha_Thai|C1|low": {"n": 40, "acc": 85.0, "target_share": 0.467, "eng_share": 0.512, "pct_mostly_target": 42.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 405.775, "completion_tokens": 462.2}, "tha_Thai|C2|low": {"n": 40, "acc": 87.5, "target_share": 0.693, "eng_share": 0.3, "pct_mostly_target": 67.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 470.95, "completion_tokens": 505.425}, "khm_Khmr|C1|low": {"n": 40, "acc": 87.5, "target_share": 0.302, "eng_share": 0.681, "pct_mostly_target": 25.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 597.55, "completion_tokens": 779.225}, "khm_Khmr|C3|low": {"n": 40, "acc": 85.0, "target_share": 0.842, "eng_share": 0.155, "pct_mostly_target": 87.5, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 472.8, "completion_tokens": 484.175}, "vie_Latn|C3|low": {"n": 40, "acc": 97.5, "target_share": 0.687, "eng_share": 0.305, "pct_mostly_target": 67.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 201.55, "completion_tokens": 212.8}, "sun_Latn|C0|low": {"n": 40, "acc": 92.5, "target_share": 0.469, "eng_share": 0.511, "pct_mostly_target": 47.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 402.8, "completion_tokens": 413.825}, "jav_Latn|C0|low": {"n": 40, "acc": 90.0, "target_share": 0.401, "eng_share": 0.555, "pct_mostly_target": 35.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 361.3, "completion_tokens": 372.7}, "jav_Latn|C2|low": {"n": 40, "acc": 92.5, "target_share": 0.494, "eng_share": 0.415, "pct_mostly_target": 52.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 364.025, "completion_tokens": 409.375}, "zsm_Latn|C0|low": {"n": 40, "acc": 97.5, "target_share": 0.419, "eng_share": 0.558, "pct_mostly_target": 45.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 218.375, "completion_tokens": 229.525}, "lao_Laoo|C3|low": {"n": 40, "acc": 87.5, "target_share": 0.37, "eng_share": 0.548, "pct_mostly_target": 32.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 460.7, "completion_tokens": 471.65}, "mya_Mymr|C2|low": {"n": 40, "acc": 90.0, "target_share": 0.469, "eng_share": 0.51, "pct_mostly_target": 47.5, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 806.025, "completion_tokens": 979.9}, "mya_Mymr|C3|low": {"n": 40, "acc": 92.5, "target_share": 0.736, "eng_share": 0.257, "pct_mostly_target": 72.5, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 855.85, "completion_tokens": 875.55}, "mya_Mymr|C0|low": {"n": 40, "acc": 92.5, "target_share": 0.33, "eng_share": 0.631, "pct_mostly_target": 25.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 663.725, "completion_tokens": 676.1}, "mya_Mymr|C1|low": {"n": 40, "acc": 92.5, "target_share": 0.306, "eng_share": 0.654, "pct_mostly_target": 30.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 570.35, "completion_tokens": 973.725}, "hin_Deva|C1|low": {"n": 40, "acc": 90.0, "target_share": 0.26, "eng_share": 0.719, "pct_mostly_target": 17.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 469.975, "completion_tokens": 608.45}, "hin_Deva|C0|low": {"n": 40, "acc": 95.0, "target_share": 0.281, "eng_share": 0.688, "pct_mostly_target": 17.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 407.775, "completion_tokens": 418.8}, "tam_Taml|C1|low": {"n": 40, "acc": 90.0, "target_share": 0.344, "eng_share": 0.645, "pct_mostly_target": 27.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 595.675, "completion_tokens": 864}, "tam_Taml|C2|low": {"n": 40, "acc": 92.5, "target_share": 0.399, "eng_share": 0.588, "pct_mostly_target": 27.5, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 656.775, "completion_tokens": 742.925}, "hin_Deva|C3|low": {"n": 40, "acc": 97.5, "target_share": 0.679, "eng_share": 0.318, "pct_mostly_target": 65.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 504.575, "completion_tokens": 515.925}, "hin_Deva|C2|low": {"n": 40, "acc": 90.0, "target_share": 0.476, "eng_share": 0.505, "pct_mostly_target": 42.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 523.925, "completion_tokens": 579.125}, "kor_Hang|C0|low": {"n": 40, "acc": 95.0, "target_share": 0.267, "eng_share": 0.71, "pct_mostly_target": 22.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 279.875, "completion_tokens": 290.4}, "tam_Taml|C3|low": {"n": 40, "acc": 92.5, "target_share": 0.562, "eng_share": 0.43, "pct_mostly_target": 60.0, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 604, "completion_tokens": 614.475}, "khm_Khmr|C0|low": {"n": 40, "acc": 82.5, "target_share": 0.21, "eng_share": 0.772, "pct_mostly_target": 17.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 449.925, "completion_tokens": 460.95}, "khm_Khmr|C2|low": {"n": 40, "acc": 80.0, "target_share": 0.655, "eng_share": 0.337, "pct_mostly_target": 65.0, "no_reasoning_signal": 0, "empty_or_truncated": 3, "reasoning_tokens": 1000.875, "completion_tokens": 1122.25}, "kor_Hang|C3|low": {"n": 40, "acc": 95.0, "target_share": 0.529, "eng_share": 0.461, "pct_mostly_target": 55.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 270.5, "completion_tokens": 281.625}, "jpn_Jpan|C0|low": {"n": 40, "acc": 95.0, "target_share": 0.278, "eng_share": 0.699, "pct_mostly_target": 17.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 257.675, "completion_tokens": 269.05}, "kor_Hang|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.294, "eng_share": 0.691, "pct_mostly_target": 25.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 265.975, "completion_tokens": 322.275}, "lao_Laoo|C0|low": {"n": 40, "acc": 87.5, "target_share": 0.199, "eng_share": 0.779, "pct_mostly_target": 10.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 509.225, "completion_tokens": 520.25}, "arb_Arab|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.336, "eng_share": 0.643, "pct_mostly_target": 32.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 266.425, "completion_tokens": 314.55}, "jpn_Jpan|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.29, "eng_share": 0.693, "pct_mostly_target": 22.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 239.925, "completion_tokens": 294.175}, "jpn_Jpan|C3|low": {"n": 40, "acc": 95.0, "target_share": 0.228, "eng_share": 0.763, "pct_mostly_target": 15.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 253.3, "completion_tokens": 265.175}, "arb_Arab|C2|low": {"n": 40, "acc": 95.0, "target_share": 0.533, "eng_share": 0.432, "pct_mostly_target": 55.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 242.6, "completion_tokens": 267.175}, "kor_Hang|C2|low": {"n": 40, "acc": 95.0, "target_share": 0.434, "eng_share": 0.547, "pct_mostly_target": 45.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 325.825, "completion_tokens": 345.225}, "arb_Arab|C3|low": {"n": 40, "acc": 92.5, "target_share": 0.578, "eng_share": 0.402, "pct_mostly_target": 55.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 215.8, "completion_tokens": 227.075}, "yor_Latn|C1|low": {"n": 40, "acc": 77.5, "target_share": 0.534, "eng_share": 0.446, "pct_mostly_target": 67.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 762.425, "completion_tokens": 871.075}, "yor_Latn|C0|low": {"n": 40, "acc": 82.5, "target_share": 0.536, "eng_share": 0.438, "pct_mostly_target": 57.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 710.6, "completion_tokens": 722.575}, "jpn_Jpan|C2|low": {"n": 40, "acc": 95.0, "target_share": 0.67, "eng_share": 0.319, "pct_mostly_target": 60.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 236.125, "completion_tokens": 257.4}, "arb_Arab|C0|low": {"n": 40, "acc": 95.0, "target_share": 0.213, "eng_share": 0.741, "pct_mostly_target": 17.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 221, "completion_tokens": 232.3}, "yor_Latn|C2|low": {"n": 40, "acc": 77.5, "target_share": 0.456, "eng_share": 0.517, "pct_mostly_target": 40.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 616.35, "completion_tokens": 681.05}, "plt_Latn|C2|low": {"n": 40, "acc": 90.0, "target_share": 0.515, "eng_share": 0.453, "pct_mostly_target": 62.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 347.175, "completion_tokens": 359.85}, "plt_Latn|C1|low": {"n": 40, "acc": 87.5, "target_share": 0.5, "eng_share": 0.47, "pct_mostly_target": 55.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 545.35, "completion_tokens": 563.175}, "plt_Latn|C3|low": {"n": 40, "acc": 90.0, "target_share": 0.607, "eng_share": 0.377, "pct_mostly_target": 55.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 327.05, "completion_tokens": 338.05}, "ind_Latn|C0|low": {"n": 40, "acc": 95.0, "target_share": 0.472, "eng_share": 0.511, "pct_mostly_target": 52.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 271.25, "completion_tokens": 282.25}, "yor_Latn|C3|low": {"n": 40, "acc": 85.0, "target_share": 0.431, "eng_share": 0.548, "pct_mostly_target": 32.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 578.225, "completion_tokens": 594.15}, "sun_Latn|C1|low": {"n": 40, "acc": 90.0, "target_share": 0.406, "eng_share": 0.56, "pct_mostly_target": 40.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 393.575, "completion_tokens": 477.275}, "zsm_Latn|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.5, "eng_share": 0.483, "pct_mostly_target": 55.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 263.15, "completion_tokens": 338.95}, "zsm_Latn|C2|low": {"n": 40, "acc": 97.5, "target_share": 0.783, "eng_share": 0.212, "pct_mostly_target": 80.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 209.5, "completion_tokens": 240.85}, "zsm_Latn|C3|low": {"n": 40, "acc": 95.0, "target_share": 0.854, "eng_share": 0.138, "pct_mostly_target": 87.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 211.725, "completion_tokens": 222.775}, "tgl_Latn|C0|low": {"n": 40, "acc": 92.5, "target_share": 0.476, "eng_share": 0.49, "pct_mostly_target": 62.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 258.875, "completion_tokens": 269.65}, "tgl_Latn|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.498, "eng_share": 0.48, "pct_mostly_target": 65.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 279.7, "completion_tokens": 340}, "lao_Laoo|C1|low": {"n": 40, "acc": 87.5, "target_share": 0.198, "eng_share": 0.769, "pct_mostly_target": 12.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 580.05, "completion_tokens": 777.2}, "lao_Laoo|C2|low": {"n": 40, "acc": 82.5, "target_share": 0.411, "eng_share": 0.563, "pct_mostly_target": 37.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 543.975, "completion_tokens": 638.55}, "tam_Taml|C0|low": {"n": 40, "acc": 92.5, "target_share": 0.377, "eng_share": 0.608, "pct_mostly_target": 30.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 544.975, "completion_tokens": 555.85}, "plt_Latn|C0|low": {"n": 40, "acc": 90.0, "target_share": 0.499, "eng_share": 0.466, "pct_mostly_target": 55.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 471.4, "completion_tokens": 482.7}}, "math": {"ceb_Latn|C0|low": {"n": 51, "acc": 98.0, "target_share": 0.175, "eng_share": 0.742, "pct_mostly_target": 5.9, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 584.588, "completion_tokens": 664.294}, "eng_Latn|C0|low": {"n": 52, "acc": 94.2, "target_share": 0.923, "eng_share": 0.0, "pct_mostly_target": 100.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 529.962, "completion_tokens": 579.423}, "ind_Latn|C2|low": {"n": 49, "acc": 98.0, "target_share": 0.681, "eng_share": 0.22, "pct_mostly_target": 69.4, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 507.755, "completion_tokens": 581.878}, "zsm_Latn|C0|low": {"n": 51, "acc": 90.2, "target_share": 0.221, "eng_share": 0.733, "pct_mostly_target": 11.8, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 685.863, "completion_tokens": 740.549}, "vie_Latn|C2|low": {"n": 49, "acc": 87.8, "target_share": 0.512, "eng_share": 0.432, "pct_mostly_target": 46.9, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 802.714, "completion_tokens": 876.367}, "ceb_Latn|C2|low": {"n": 48, "acc": 93.8, "target_share": 0.228, "eng_share": 0.683, "pct_mostly_target": 10.4, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 572.667, "completion_tokens": 711.812}, "jav_Latn|C2|low": {"n": 50, "acc": 96.0, "target_share": 0.145, "eng_share": 0.554, "pct_mostly_target": 0.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 582.74, "completion_tokens": 761.28}, "ind_Latn|C0|low": {"n": 49, "acc": 98.0, "target_share": 0.217, "eng_share": 0.718, "pct_mostly_target": 12.2, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 517.878, "completion_tokens": 558.612}, "zsm_Latn|C2|low": {"n": 50, "acc": 96.0, "target_share": 0.579, "eng_share": 0.372, "pct_mostly_target": 52.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 683.08, "completion_tokens": 827.5}, "tgl_Latn|C0|low": {"n": 51, "acc": 98.0, "target_share": 0.182, "eng_share": 0.74, "pct_mostly_target": 2.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 610.49, "completion_tokens": 684.51}, "eng_Latn|C2|low": {"n": 52, "acc": 94.2, "target_share": 0.92, "eng_share": 0.0, "pct_mostly_target": 100.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 551.577, "completion_tokens": 605.135}, "sun_Latn|C2|low": {"n": 51, "acc": 98.0, "target_share": 0.167, "eng_share": 0.558, "pct_mostly_target": 2.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 499.353, "completion_tokens": 706.51}, "jav_Latn|C0|low": {"n": 46, "acc": 95.7, "target_share": 0.172, "eng_share": 0.715, "pct_mostly_target": 2.2, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 570.609, "completion_tokens": 647.674}, "tgl_Latn|C2|low": {"n": 49, "acc": 95.9, "target_share": 0.249, "eng_share": 0.679, "pct_mostly_target": 16.3, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 608, "completion_tokens": 834.122}, "vie_Latn|C0|low": {"n": 51, "acc": 88.2, "target_share": 0.211, "eng_share": 0.735, "pct_mostly_target": 11.8, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 827.392, "completion_tokens": 860.137}, "sun_Latn|C0|low": {"n": 48, "acc": 97.9, "target_share": 0.184, "eng_share": 0.706, "pct_mostly_target": 4.2, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 547.312, "completion_tokens": 581.562}, "tha_Thai|C0|low": {"n": 54, "acc": 92.6, "target_share": 0.131, "eng_share": 0.808, "pct_mostly_target": 1.9, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 740.704, "completion_tokens": 762.019}, "tha_Thai|C2|low": {"n": 52, "acc": 94.2, "target_share": 0.416, "eng_share": 0.523, "pct_mostly_target": 34.6, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 835.231, "completion_tokens": 913.731}}, "replication": [{"source": "esscor", "lang": "eng_Latn", "cond": "cat=everyday", "effort": "default", "n": 14, "mean_target_share": 0.864, "mean_eng_share": 0.0, "pct_mostly_target": 92.9}, {"source": "esscor", "lang": "eng_Latn", "cond": "cat=everyday", "effort": "minimal", "n": 7, "mean_target_share": 0.797, "mean_eng_share": 0.0, "pct_mostly_target": 85.7}, {"source": "esscor", "lang": "eng_Latn", "cond": "cat=knowledge", "effort": "default", "n": 30, "mean_target_share": 0.869, "mean_eng_share": 0.0, "pct_mostly_target": 96.7}, {"source": "esscor", "lang": "eng_Latn", "cond": "cat=knowledge", "effort": "minimal", "n": 15, "mean_target_share": 0.931, "mean_eng_share": 0.0, "pct_mostly_target": 100.0}, {"source": "esscor", "lang": "eng_Latn", "cond": "cat=logic", "effort": "default", "n": 16, "mean_target_share": 0.884, "mean_eng_share": 0.0, "pct_mostly_target": 100.0}, {"source": "esscor", "lang": "eng_Latn", "cond": "cat=logic", "effort": "minimal", "n": 8, "mean_target_share": 0.959, "mean_eng_share": 0.0, "pct_mostly_target": 100.0}, {"source": "esscor", "lang": "eng_Latn", "cond": "cat=math", "effort": "default", "n": 40, "mean_target_share": 0.883, "mean_eng_share": 0.0, "pct_mostly_target": 97.3}, {"source": "esscor", "lang": "eng_Latn", "cond": "cat=math", "effort": "minimal", "n": 20, "mean_target_share": 0.761, "mean_eng_share": 0.0, "pct_mostly_target": 84.2}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=everyday", "effort": "default", "n": 14, "mean_target_share": 0.034, "mean_eng_share": 0.821, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=everyday", "effort": "minimal", "n": 7, "mean_target_share": 0.0, "mean_eng_share": 0.758, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=knowledge", "effort": "default", "n": 30, "mean_target_share": 0.044, "mean_eng_share": 0.855, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=knowledge", "effort": "minimal", "n": 15, "mean_target_share": 0.075, "mean_eng_share": 0.836, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=logic", "effort": "default", "n": 16, "mean_target_share": 0.134, "mean_eng_share": 0.73, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=logic", "effort": "minimal", "n": 8, "mean_target_share": 0.045, "mean_eng_share": 0.778, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=math", "effort": "default", "n": 40, "mean_target_share": 0.058, "mean_eng_share": 0.814, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=math", "effort": "minimal", "n": 20, "mean_target_share": 0.068, "mean_eng_share": 0.851, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=calendar", "effort": "default", "n": 6, "mean_target_share": 0.328, "mean_eng_share": 0.398, "pct_mostly_target": 33.3}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=everyday", "effort": "default", "n": 14, "mean_target_share": 0.132, "mean_eng_share": 0.743, "pct_mostly_target": 7.1}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=everyday", "effort": "minimal", "n": 7, "mean_target_share": 0.212, "mean_eng_share": 0.655, "pct_mostly_target": 28.6}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=grammar", "effort": "default", "n": 6, "mean_target_share": 0.497, "mean_eng_share": 0.45, "pct_mostly_target": 50.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=knowledge", "effort": "default", "n": 30, "mean_target_share": 0.222, "mean_eng_share": 0.721, "pct_mostly_target": 20.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=knowledge", "effort": "minimal", "n": 15, "mean_target_share": 0.204, "mean_eng_share": 0.746, "pct_mostly_target": 13.3}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=literature", "effort": "default", "n": 18, "mean_target_share": 0.334, "mean_eng_share": 0.607, "pct_mostly_target": 27.8}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=logic", "effort": "default", "n": 16, "mean_target_share": 0.246, "mean_eng_share": 0.72, "pct_mostly_target": 18.8}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=logic", "effort": "minimal", "n": 8, "mean_target_share": 0.135, "mean_eng_share": 0.775, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=math", "effort": "default", "n": 40, "mean_target_share": 0.149, "mean_eng_share": 0.74, "pct_mostly_target": 10.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=math", "effort": "minimal", "n": 20, "mean_target_share": 0.011, "mean_eng_share": 0.879, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=numerals", "effort": "default", "n": 9, "mean_target_share": 0.168, "mean_eng_share": 0.775, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=numwords", "effort": "default", "n": 12, "mean_target_share": 0.461, "mean_eng_share": 0.461, "pct_mostly_target": 50.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=script", "effort": "default", "n": 15, "mean_target_share": 0.251, "mean_eng_share": 0.65, "pct_mostly_target": 0.0}, {"source": "neulketing", "lang": "eng_Latn", "cond": "belebele_en", "effort": "low", "n": 900, "mean_target_share": 0.976, "mean_eng_share": 0.0, "pct_mostly_target": 99.9}, {"source": "neulketing", "lang": "kor_Hang", "cond": "belebele_ko", "effort": "low", "n": 900, "mean_target_share": 0.291, "mean_eng_share": 0.674, "pct_mostly_target": 22.7}, {"source": "neulketing", "lang": "kor_Hang", "cond": "belebele_mixed", "effort": "low", "n": 900, "mean_target_share": 0.185, "mean_eng_share": 0.792, "pct_mostly_target": 4.3}, {"source": "neulketing", "lang": "kor_Hang", "cond": "items", "effort": "low", "n": 577, "mean_target_share": 0.193, "mean_eng_share": 0.752, "pct_mostly_target": 3.5}, {"source": "neulketing", "lang": "kor_Hang", "cond": "items_hint", "effort": "low", "n": 145, "mean_target_share": 0.208, "mean_eng_share": 0.742, "pct_mostly_target": 2.8}, {"source": "neulketing", "lang": "kor_Hang", "cond": "items_particles", "effort": "high", "n": 201, "mean_target_share": 0.119, "mean_eng_share": 0.824, "pct_mostly_target": 0.5}], "grammar": {"K0/x2ind": {"chrf++": 33.64, "n": 100, "empty": 0}, "K0/ind2x": {"chrf++": 23.88, "n": 100, "empty": 0}, "K0/think": {"n_seg": 62, "target_share": 0.318, "eng_share": 0.19, "other_share": 0.492, "first_eng_pos": 0.0, "top_other": "ind_Latn"}, "K0/prompt_tokens": 3034, "K1/x2ind": {"chrf++": 26.99, "n": 100, "empty": 0}, "K1/ind2x": {"chrf++": 24.21, "n": 100, "empty": 0}, "K1/think": {"n_seg": 254, "target_share": 0.793, "eng_share": 0.039, "other_share": 0.168, "first_eng_pos": 0.0, "top_other": "ind_Latn"}, "K1/prompt_tokens": 248587, "K2/x2ind": {"chrf++": 33.71, "n": 100, "empty": 0}, "K2/ind2x": {"chrf++": 23.7, "n": 100, "empty": 0}, "K2/think": {"n_seg": 208, "target_share": 0.575, "eng_share": 0.057, "other_share": 0.369, "first_eng_pos": 0.0, "top_other": "ind_Latn"}, "K2/prompt_tokens": 15400, "K3/x2ind": {"chrf++": 49.12, "n": 100, "empty": 0}, "K3/ind2x": {"chrf++": 32.73, "n": 100, "empty": 0}, "K3/think": {"n_seg": 267, "target_share": 0.438, "eng_share": 0.032, "other_share": 0.53, "first_eng_pos": 0.0, "top_other": "ind_Latn"}, "K3/prompt_tokens": 402849}, "lid_calibration": {"eng_Latn": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 1.0, "confusions": [["eng_Latn", 300]]}, "ind_Latn": {"n": 300, "acc_with_alias": 0.983, "pred_eng": 0.0, "confusions": [["ind_Latn", 291], ["zsm_Latn", 4], ["bjn_Latn", 2]]}, "jav_Latn": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["jav_Latn", 300]]}, "sun_Latn": {"n": 300, "acc_with_alias": 0.997, "pred_eng": 0.0, "confusions": [["sun_Latn", 299], ["jav_Latn", 1]]}, "zsm_Latn": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["zsm_Latn", 256], ["ind_Latn", 44]]}, "tgl_Latn": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["fil_Latn", 300]]}, "ceb_Latn": {"n": 300, "acc_with_alias": 0.977, "pred_eng": 0.003, "confusions": [["ceb_Latn", 293], ["hil_Latn", 4], ["eng_Latn", 1]]}, "vie_Latn": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["vie_Latn", 300]]}, "tha_Thai": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["tha_Thai", 300]]}, "khm_Khmr": {"n": 300, "acc_with_alias": 0.997, "pred_eng": 0.0, "confusions": [["khm_Khmr", 299], ["ayr_Latn", 1]]}, "lao_Laoo": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["lao_Laoo", 300]]}, "mya_Mymr": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["mya_Mymr", 300]]}, "hin_Deva": {"n": 300, "acc_with_alias": 0.997, "pred_eng": 0.0, "confusions": [["hin_Deva", 299], ["anp_Deva", 1]]}, "tam_Taml": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["tam_Taml", 300]]}, "kor_Hang": {"n": 300, "acc_with_alias": 0.997, "pred_eng": 0.0, "confusions": [["kor_Hang", 299], ["zxx_Latn", 1]]}, "jpn_Jpan": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["jpn_Jpan", 300]]}, "arb_Arab": {"n": 300, "acc_with_alias": 0.823, "pred_eng": 0.0, "confusions": [["arb_Arab", 247], ["ars_Arab", 23], ["ary_Arab", 13]]}, "yor_Latn": {"n": 300, "acc_with_alias": 0.993, "pred_eng": 0.0, "confusions": [["yor_Latn", 298], ["ife_Latn", 1], ["kyu_Latn", 1]]}, "plt_Latn": {"n": 300, "acc_with_alias": 0.993, "pred_eng": 0.0, "confusions": [["plt_Latn", 298], ["xmv_Latn", 1], ["skg_Latn", 1]]}}, "tiers": {"ind_Latn": 3, "jav_Latn": 1, "sun_Latn": 1, "zsm_Latn": 3, "tgl_Latn": 3, "ceb_Latn": 3, "vie_Latn": 4, "tha_Thai": 3, "khm_Khmr": 1, "lao_Laoo": 2, "mya_Mymr": 1, "hin_Deva": 4, "tam_Taml": 3, "kor_Hang": 4, "jpn_Jpan": 5, "arb_Arab": 5, "yor_Latn": 2, "plt_Latn": 1, "eng_Latn": 5, "bug_Latn": 1}, "verdicts": {"H1": {"pass": true, "detail": "15/18 languages with C0 eng_share >= 0.5", "langs": ["arb_Arab", "ceb_Latn", "hin_Deva", "ind_Latn", "jav_Latn", "jpn_Jpan", "khm_Khmr", "kor_Hang", "lao_Laoo", "mya_Mymr", "sun_Latn", "tam_Taml", "tha_Thai", "vie_Latn", "zsm_Latn"]}, "H2": {"pass": true, "detail": "C2 > C0 in 16/18; of those 15 still < 0.8"}, "H3": {"pass": true, "detail": "C3 > C2 in 13/18"}, "H5": {"pass": true, "detail": "mean acc(C2)-acc(C0) = 0.12 pts over 9 langs"}, "H6": {"pass": false, "detail": "x2ind: [33.64, 26.99, 33.71, 49.12]; ind2x: [23.88, 24.21, 23.7, 32.73]"}, "H7": {"pass": true, "detail": "K3 think target_share = 0.438"}, "H4": {"pass": true, "detail": "Spearman(Joshi tier, C2 target_share) = 0.281 over 18 languages (pre-registration phrased it as a negative correlation with 'low-resource-ness'; tier is ordered high=5)"}}, "atlas_eng_share_ci95": {"arb_Arab|C0": [0.663, 0.814], "arb_Arab|C1": [0.561, 0.725], "arb_Arab|C2": [0.323, 0.552], "arb_Arab|C3": [0.292, 0.518], "ceb_Latn|C0": [0.422, 0.599], "ceb_Latn|C1": [0.431, 0.593], "ceb_Latn|C2": [0.407, 0.595], "ceb_Latn|C3": [0.227, 0.451], "eng_Latn|C0": [0.0, 0.0], "eng_Latn|C1": [0.0, 0.0], "eng_Latn|C2": [0.0, 0.0], "eng_Latn|C3": [0.0, 0.0], "hin_Deva|C0": [0.608, 0.766], "hin_Deva|C1": [0.651, 0.786], "hin_Deva|C2": [0.397, 0.617], "hin_Deva|C3": [0.19, 0.453], "ind_Latn|C0": [0.423, 0.593], "ind_Latn|C1": [0.381, 0.565], "ind_Latn|C2": [0.05, 0.183], "ind_Latn|C3": [0.031, 0.156], "jav_Latn|C0": [0.478, 0.638], "jav_Latn|C1": [0.374, 0.555], "jav_Latn|C2": [0.32, 0.51], "jav_Latn|C3": [0.113, 0.306], "jpn_Jpan|C0": [0.623, 0.774], "jpn_Jpan|C1": [0.588, 0.787], "jpn_Jpan|C2": [0.203, 0.442], "jpn_Jpan|C3": [0.668, 0.85], "khm_Khmr|C0": [0.695, 0.845], "khm_Khmr|C1": [0.591, 0.769], "khm_Khmr|C2": [0.214, 0.466], "khm_Khmr|C3": [0.061, 0.267], "kor_Hang|C0": [0.629, 0.788], "kor_Hang|C1": [0.617, 0.762], "kor_Hang|C2": [0.445, 0.647], "kor_Hang|C3": [0.343, 0.582], "lao_Laoo|C0": [0.717, 0.843], "lao_Laoo|C1": [0.697, 0.837], "lao_Laoo|C2": [0.443, 0.678], "lao_Laoo|C3": [0.409, 0.681], "mya_Mymr|C0": [0.552, 0.711], "mya_Mymr|C1": [0.575, 0.73], "mya_Mymr|C2": [0.409, 0.613], "mya_Mymr|C3": [0.147, 0.368], "plt_Latn|C0": [0.388, 0.545], "plt_Latn|C1": [0.404, 0.541], "plt_Latn|C2": [0.351, 0.553], "plt_Latn|C3": [0.262, 0.491], "sun_Latn|C0": [0.431, 0.593], "sun_Latn|C1": [0.486, 0.634], "sun_Latn|C2": [0.513, 0.72], "sun_Latn|C3": [0.181, 0.368], "tam_Taml|C0": [0.536, 0.678], "tam_Taml|C1": [0.576, 0.713], "tam_Taml|C2": [0.505, 0.671], "tam_Taml|C3": [0.315, 0.55], "tgl_Latn|C0": [0.402, 0.589], "tgl_Latn|C1": [0.4, 0.566], "tgl_Latn|C2": [0.425, 0.582], "tgl_Latn|C3": [0.141, 0.359], "tha_Thai|C0": [0.578, 0.739], "tha_Thai|C1": [0.398, 0.624], "tha_Thai|C2": [0.189, 0.412], "tha_Thai|C3": [0.185, 0.417], "vie_Latn|C0": [0.451, 0.621], "vie_Latn|C1": [0.279, 0.484], "vie_Latn|C2": [0.133, 0.317], "vie_Latn|C3": [0.19, 0.428], "yor_Latn|C0": [0.386, 0.493], "yor_Latn|C1": [0.394, 0.505], "yor_Latn|C2": [0.469, 0.568], "yor_Latn|C3": [0.455, 0.638], "zsm_Latn|C0": [0.475, 0.643], "zsm_Latn|C1": [0.383, 0.591], "zsm_Latn|C2": [0.114, 0.318], "zsm_Latn|C3": [0.074, 0.204]}, "robustness": {"arb_Arab|C0": {"eng_share_prereg": 0.741, "eng_share_noquote": 0.893, "target_share_noquote": 0.049, "lingua_eng": 0.642, "lingua_eng_noquote": 0.793}, "arb_Arab|C1": {"eng_share_prereg": 0.643, "eng_share_noquote": 0.832, "target_share_noquote": 0.138, "lingua_eng": 0.52, "lingua_eng_noquote": 0.676}, "arb_Arab|C2": {"eng_share_prereg": 0.432, "eng_share_noquote": 0.611, "target_share_noquote": 0.329, "lingua_eng": 0.373, "lingua_eng_noquote": 0.539}, "arb_Arab|C3": {"eng_share_prereg": 0.402, "eng_share_noquote": 0.553, "target_share_noquote": 0.424, "lingua_eng": 0.294, "lingua_eng_noquote": 0.405}, "ceb_Latn|C0": {"eng_share_prereg": 0.509, "eng_share_noquote": 0.77, "target_share_noquote": 0.16, "lingua_eng": 0.457, "lingua_eng_noquote": 0.679}, "ceb_Latn|C1": {"eng_share_prereg": 0.51, "eng_share_noquote": 0.748, "target_share_noquote": 0.194, "lingua_eng": 0.425, "lingua_eng_noquote": 0.595}, "ceb_Latn|C2": {"eng_share_prereg": 0.499, "eng_share_noquote": 0.701, "target_share_noquote": 0.233, "lingua_eng": 0.432, "lingua_eng_noquote": 0.613}, "ceb_Latn|C3": {"eng_share_prereg": 0.336, "eng_share_noquote": 0.461, "target_share_noquote": 0.523, "lingua_eng": 0.326, "lingua_eng_noquote": 0.437}, "eng_Latn|C0": {"eng_share_prereg": 0.0, "eng_share_noquote": 0.0, "target_share_noquote": 0.959, "lingua_eng": 0.93, "lingua_eng_noquote": 0.908}, "eng_Latn|C1": {"eng_share_prereg": 0.0, "eng_share_noquote": 0.0, "target_share_noquote": 0.941, "lingua_eng": 0.943, "lingua_eng_noquote": 0.963}, "eng_Latn|C2": {"eng_share_prereg": 0.0, "eng_share_noquote": 0.0, "target_share_noquote": 0.933, "lingua_eng": 0.926, "lingua_eng_noquote": 0.948}, "eng_Latn|C3": {"eng_share_prereg": 0.0, "eng_share_noquote": 0.0, "target_share_noquote": 0.979, "lingua_eng": 0.977, "lingua_eng_noquote": 0.973}, "hin_Deva|C0": {"eng_share_prereg": 0.688, "eng_share_noquote": 0.868, "target_share_noquote": 0.098, "lingua_eng": 0.71, "lingua_eng_noquote": 0.879}, "hin_Deva|C1": {"eng_share_prereg": 0.719, "eng_share_noquote": 0.857, "target_share_noquote": 0.117, "lingua_eng": 0.715, "lingua_eng_noquote": 0.853}, "hin_Deva|C2": {"eng_share_prereg": 0.505, "eng_share_noquote": 0.643, "target_share_noquote": 0.332, "lingua_eng": 0.516, "lingua_eng_noquote": 0.653}, "hin_Deva|C3": {"eng_share_prereg": 0.318, "eng_share_noquote": 0.457, "target_share_noquote": 0.539, "lingua_eng": 0.319, "lingua_eng_noquote": 0.458}, "ind_Latn|C0": {"eng_share_prereg": 0.511, "eng_share_noquote": 0.806, "target_share_noquote": 0.159, "lingua_eng": 0.49, "lingua_eng_noquote": 0.774}, "ind_Latn|C1": {"eng_share_prereg": 0.472, "eng_share_noquote": 0.712, "target_share_noquote": 0.261, "lingua_eng": 0.451, "lingua_eng_noquote": 0.667}, "ind_Latn|C2": {"eng_share_prereg": 0.11, "eng_share_noquote": 0.387, "target_share_noquote": 0.573, "lingua_eng": 0.106, "lingua_eng_noquote": 0.299}, "ind_Latn|C3": {"eng_share_prereg": 0.084, "eng_share_noquote": 0.174, "target_share_noquote": 0.821, "lingua_eng": 0.09, "lingua_eng_noquote": 0.174}, "jav_Latn|C0": {"eng_share_prereg": 0.555, "eng_share_noquote": 0.798, "target_share_noquote": 0.16, "lingua_eng": 0.571, "lingua_eng_noquote": 0.828}, "jav_Latn|C1": {"eng_share_prereg": 0.469, "eng_share_noquote": 0.659, "target_share_noquote": 0.253, "lingua_eng": 0.486, "lingua_eng_noquote": 0.64}, "jav_Latn|C2": {"eng_share_prereg": 0.415, "eng_share_noquote": 0.616, "target_share_noquote": 0.249, "lingua_eng": 0.444, "lingua_eng_noquote": 0.615}, "jav_Latn|C3": {"eng_share_prereg": 0.203, "eng_share_noquote": 0.287, "target_share_noquote": 0.691, "lingua_eng": 0.269, "lingua_eng_noquote": 0.343}, "jpn_Jpan|C0": {"eng_share_prereg": 0.699, "eng_share_noquote": 0.872, "target_share_noquote": 0.096, "lingua_eng": 0.595, "lingua_eng_noquote": 0.756}, "jpn_Jpan|C1": {"eng_share_prereg": 0.693, "eng_share_noquote": 0.786, "target_share_noquote": 0.195, "lingua_eng": 0.607, "lingua_eng_noquote": 0.693}, "jpn_Jpan|C2": {"eng_share_prereg": 0.319, "eng_share_noquote": 0.405, "target_share_noquote": 0.581, "lingua_eng": 0.245, "lingua_eng_noquote": 0.316}, "jpn_Jpan|C3": {"eng_share_prereg": 0.763, "eng_share_noquote": 0.844, "target_share_noquote": 0.147, "lingua_eng": 0.564, "lingua_eng_noquote": 0.636}, "khm_Khmr|C0": {"eng_share_prereg": 0.772, "eng_share_noquote": 0.884, "target_share_noquote": 0.087, "lingua_eng": 0.9, "lingua_eng_noquote": 0.933}, "khm_Khmr|C1": {"eng_share_prereg": 0.681, "eng_share_noquote": 0.853, "target_share_noquote": 0.127, "lingua_eng": 0.785, "lingua_eng_noquote": 0.887}, "khm_Khmr|C2": {"eng_share_prereg": 0.337, "eng_share_noquote": 0.372, "target_share_noquote": 0.62, "lingua_eng": 0.432, "lingua_eng_noquote": 0.499}, "khm_Khmr|C3": {"eng_share_prereg": 0.155, "eng_share_noquote": 0.191, "target_share_noquote": 0.804, "lingua_eng": 0.301, "lingua_eng_noquote": 0.35}, "kor_Hang|C0": {"eng_share_prereg": 0.71, "eng_share_noquote": 0.849, "target_share_noquote": 0.119, "lingua_eng": 0.62, "lingua_eng_noquote": 0.748}, "kor_Hang|C1": {"eng_share_prereg": 0.691, "eng_share_noquote": 0.858, "target_share_noquote": 0.123, "lingua_eng": 0.607, "lingua_eng_noquote": 0.759}, "kor_Hang|C2": {"eng_share_prereg": 0.547, "eng_share_noquote": 0.736, "target_share_noquote": 0.233, "lingua_eng": 0.455, "lingua_eng_noquote": 0.631}, "kor_Hang|C3": {"eng_share_prereg": 0.461, "eng_share_noquote": 0.576, "target_share_noquote": 0.412, "lingua_eng": 0.361, "lingua_eng_noquote": 0.46}, "lao_Laoo|C0": {"eng_share_prereg": 0.779, "eng_share_noquote": 0.899, "target_share_noquote": 0.074, "lingua_eng": 0.861, "lingua_eng_noquote": 0.914}, "lao_Laoo|C1": {"eng_share_prereg": 0.769, "eng_share_noquote": 0.881, "target_share_noquote": 0.083, "lingua_eng": 0.808, "lingua_eng_noquote": 0.85}, "lao_Laoo|C2": {"eng_share_prereg": 0.563, "eng_share_noquote": 0.682, "target_share_noquote": 0.295, "lingua_eng": 0.59, "lingua_eng_noquote": 0.653}, "lao_Laoo|C3": {"eng_share_prereg": 0.548, "eng_share_noquote": 0.625, "target_share_noquote": 0.286, "lingua_eng": 0.636, "lingua_eng_noquote": 0.67}, "mya_Mymr|C0": {"eng_share_prereg": 0.631, "eng_share_noquote": 0.797, "target_share_noquote": 0.157, "lingua_eng": 0.832, "lingua_eng_noquote": 0.917}, "mya_Mymr|C1": {"eng_share_prereg": 0.654, "eng_share_noquote": 0.824, "target_share_noquote": 0.119, "lingua_eng": 0.815, "lingua_eng_noquote": 0.915}, "mya_Mymr|C2": {"eng_share_prereg": 0.51, "eng_share_noquote": 0.676, "target_share_noquote": 0.294, "lingua_eng": 0.647, "lingua_eng_noquote": 0.756}, "mya_Mymr|C3": {"eng_share_prereg": 0.257, "eng_share_noquote": 0.452, "target_share_noquote": 0.536, "lingua_eng": 0.413, "lingua_eng_noquote": 0.547}, "plt_Latn|C0": {"eng_share_prereg": 0.466, "eng_share_noquote": 0.751, "target_share_noquote": 0.188, "lingua_eng": 0.439, "lingua_eng_noquote": 0.715}, "plt_Latn|C1": {"eng_share_prereg": 0.47, "eng_share_noquote": 0.715, "target_share_noquote": 0.246, "lingua_eng": 0.485, "lingua_eng_noquote": 0.71}, "plt_Latn|C2": {"eng_share_prereg": 0.453, "eng_share_noquote": 0.71, "target_share_noquote": 0.228, "lingua_eng": 0.399, "lingua_eng_noquote": 0.587}, "plt_Latn|C3": {"eng_share_prereg": 0.377, "eng_share_noquote": 0.517, "target_share_noquote": 0.468, "lingua_eng": 0.335, "lingua_eng_noquote": 0.421}, "sun_Latn|C0": {"eng_share_prereg": 0.511, "eng_share_noquote": 0.787, "target_share_noquote": 0.201, "lingua_eng": 0.522, "lingua_eng_noquote": 0.757}, "sun_Latn|C1": {"eng_share_prereg": 0.56, "eng_share_noquote": 0.762, "target_share_noquote": 0.198, "lingua_eng": 0.545, "lingua_eng_noquote": 0.724}, "sun_Latn|C2": {"eng_share_prereg": 0.616, "eng_share_noquote": 0.712, "target_share_noquote": 0.248, "lingua_eng": 0.618, "lingua_eng_noquote": 0.681}, "sun_Latn|C3": {"eng_share_prereg": 0.272, "eng_share_noquote": 0.402, "target_share_noquote": 0.461, "lingua_eng": 0.315, "lingua_eng_noquote": 0.422}, "tam_Taml|C0": {"eng_share_prereg": 0.608, "eng_share_noquote": 0.758, "target_share_noquote": 0.219, "lingua_eng": 0.56, "lingua_eng_noquote": 0.704}, "tam_Taml|C1": {"eng_share_prereg": 0.645, "eng_share_noquote": 0.831, "target_share_noquote": 0.154, "lingua_eng": 0.585, "lingua_eng_noquote": 0.759}, "tam_Taml|C2": {"eng_share_prereg": 0.588, "eng_share_noquote": 0.723, "target_share_noquote": 0.258, "lingua_eng": 0.527, "lingua_eng_noquote": 0.653}, "tam_Taml|C3": {"eng_share_prereg": 0.43, "eng_share_noquote": 0.538, "target_share_noquote": 0.453, "lingua_eng": 0.373, "lingua_eng_noquote": 0.468}, "tgl_Latn|C0": {"eng_share_prereg": 0.49, "eng_share_noquote": 0.79, "target_share_noquote": 0.158, "lingua_eng": 0.425, "lingua_eng_noquote": 0.687}, "tgl_Latn|C1": {"eng_share_prereg": 0.48, "eng_share_noquote": 0.782, "target_share_noquote": 0.19, "lingua_eng": 0.461, "lingua_eng_noquote": 0.721}, "tgl_Latn|C2": {"eng_share_prereg": 0.501, "eng_share_noquote": 0.755, "target_share_noquote": 0.222, "lingua_eng": 0.459, "lingua_eng_noquote": 0.694}, "tgl_Latn|C3": {"eng_share_prereg": 0.244, "eng_share_noquote": 0.389, "target_share_noquote": 0.527, "lingua_eng": 0.229, "lingua_eng_noquote": 0.34}, "tha_Thai|C0": {"eng_share_prereg": 0.659, "eng_share_noquote": 0.855, "target_share_noquote": 0.095, "lingua_eng": 0.585, "lingua_eng_noquote": 0.772}, "tha_Thai|C1": {"eng_share_prereg": 0.512, "eng_share_noquote": 0.637, "target_share_noquote": 0.342, "lingua_eng": 0.445, "lingua_eng_noquote": 0.543}, "tha_Thai|C2": {"eng_share_prereg": 0.3, "eng_share_noquote": 0.416, "target_share_noquote": 0.575, "lingua_eng": 0.249, "lingua_eng_noquote": 0.321}, "tha_Thai|C3": {"eng_share_prereg": 0.294, "eng_share_noquote": 0.407, "target_share_noquote": 0.562, "lingua_eng": 0.239, "lingua_eng_noquote": 0.282}, "vie_Latn|C0": {"eng_share_prereg": 0.537, "eng_share_noquote": 0.77, "target_share_noquote": 0.201, "lingua_eng": 0.512, "lingua_eng_noquote": 0.732}, "vie_Latn|C1": {"eng_share_prereg": 0.379, "eng_share_noquote": 0.654, "target_share_noquote": 0.321, "lingua_eng": 0.352, "lingua_eng_noquote": 0.603}, "vie_Latn|C2": {"eng_share_prereg": 0.22, "eng_share_noquote": 0.471, "target_share_noquote": 0.486, "lingua_eng": 0.209, "lingua_eng_noquote": 0.457}, "vie_Latn|C3": {"eng_share_prereg": 0.305, "eng_share_noquote": 0.44, "target_share_noquote": 0.552, "lingua_eng": 0.262, "lingua_eng_noquote": 0.376}, "yor_Latn|C0": {"eng_share_prereg": 0.438, "eng_share_noquote": 0.643, "target_share_noquote": 0.326, "lingua_eng": 0.378, "lingua_eng_noquote": 0.55}, "yor_Latn|C1": {"eng_share_prereg": 0.446, "eng_share_noquote": 0.614, "target_share_noquote": 0.365, "lingua_eng": 0.389, "lingua_eng_noquote": 0.535}, "yor_Latn|C2": {"eng_share_prereg": 0.517, "eng_share_noquote": 0.623, "target_share_noquote": 0.347, "lingua_eng": 0.426, "lingua_eng_noquote": 0.519}, "yor_Latn|C3": {"eng_share_prereg": 0.548, "eng_share_noquote": 0.594, "target_share_noquote": 0.384, "lingua_eng": 0.466, "lingua_eng_noquote": 0.504}, "zsm_Latn|C0": {"eng_share_prereg": 0.558, "eng_share_noquote": 0.819, "target_share_noquote": 0.147, "lingua_eng": 0.499, "lingua_eng_noquote": 0.729}, "zsm_Latn|C1": {"eng_share_prereg": 0.483, "eng_share_noquote": 0.726, "target_share_noquote": 0.247, "lingua_eng": 0.451, "lingua_eng_noquote": 0.666}, "zsm_Latn|C2": {"eng_share_prereg": 0.212, "eng_share_noquote": 0.49, "target_share_noquote": 0.503, "lingua_eng": 0.194, "lingua_eng_noquote": 0.429}, "zsm_Latn|C3": {"eng_share_prereg": 0.138, "eng_share_noquote": 0.319, "target_share_noquote": 0.671, "lingua_eng": 0.129, "lingua_eng_noquote": 0.271}}, "detector_agreement": {"n": 3040, "mean_abs_diff": 0.134, "same_majority": 0.838}, "condition_means": {"C0": {"eng_share": 0.603, "target_share": 0.368, "acc": 91.806}, "C1": {"eng_share": 0.571, "target_share": 0.405, "acc": 91.389}, "C2": {"eng_share": 0.425, "target_share": 0.55, "acc": 90.833}, "C3": {"eng_share": 0.341, "target_share": 0.638, "acc": 91.806}}, "math_mcnemar_C0_vs_C2": {"C0_only_correct": 4, "C2_only_correct": 6, "p_exact": 0.7539}, "routes": {"reflection.ai": 1987, "huggingface": 1053}, "cost_usd": 2.6405, "sea_thinking_rows": 174}
 
1
+ {"atlas": {"eng_Latn|C1|low": {"n": 40, "acc": 97.5, "target_share": 0.972, "eng_share": 0.0, "pct_mostly_target": 100.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 140.85, "completion_tokens": 152.175}, "eng_Latn|C2|low": {"n": 40, "acc": 97.5, "target_share": 0.961, "eng_share": 0.0, "pct_mostly_target": 100.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 155.95, "completion_tokens": 167.55}, "eng_Latn|C3|low": {"n": 40, "acc": 97.5, "target_share": 0.999, "eng_share": 0.0, "pct_mostly_target": 100.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 109.325, "completion_tokens": 120.875}, "jav_Latn|C3|low": {"n": 40, "acc": 92.5, "target_share": 0.777, "eng_share": 0.203, "pct_mostly_target": 82.5, "no_reasoning_signal": 0, "empty_or_truncated": 2, "reasoning_tokens": 394.075, "completion_tokens": 405.15}, "eng_Latn|C0|low": {"n": 40, "acc": 97.5, "target_share": 0.989, "eng_share": 0.0, "pct_mostly_target": 100.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 140, "completion_tokens": 151.45}, "sun_Latn|C3|low": {"n": 40, "acc": 90.0, "target_share": 0.649, "eng_share": 0.272, "pct_mostly_target": 62.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 252.775, "completion_tokens": 263.525}, "sun_Latn|C2|low": {"n": 40, "acc": 92.5, "target_share": 0.34, "eng_share": 0.616, "pct_mostly_target": 30.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 304.075, "completion_tokens": 357.325}, "jav_Latn|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.496, "eng_share": 0.469, "pct_mostly_target": 42.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 373.975, "completion_tokens": 412}, "tgl_Latn|C2|low": {"n": 40, "acc": 95.0, "target_share": 0.477, "eng_share": 0.501, "pct_mostly_target": 50.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 282.8, "completion_tokens": 346.7}, "ceb_Latn|C0|low": {"n": 40, "acc": 95.0, "target_share": 0.438, "eng_share": 0.509, "pct_mostly_target": 52.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 319.325, "completion_tokens": 329.9}, "ceb_Latn|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.445, "eng_share": 0.51, "pct_mostly_target": 47.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 341.3, "completion_tokens": 378.55}, "ceb_Latn|C2|low": {"n": 40, "acc": 95.0, "target_share": 0.457, "eng_share": 0.499, "pct_mostly_target": 47.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 306.275, "completion_tokens": 322.05}, "ind_Latn|C3|low": {"n": 40, "acc": 95.0, "target_share": 0.909, "eng_share": 0.084, "pct_mostly_target": 95.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 231.95, "completion_tokens": 242.975}, "ind_Latn|C2|low": {"n": 40, "acc": 95.0, "target_share": 0.879, "eng_share": 0.11, "pct_mostly_target": 92.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 232.375, "completion_tokens": 248.05}, "ind_Latn|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.508, "eng_share": 0.472, "pct_mostly_target": 47.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 271.1, "completion_tokens": 296.975}, "tgl_Latn|C3|low": {"n": 40, "acc": 95.0, "target_share": 0.738, "eng_share": 0.244, "pct_mostly_target": 75.0, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 360.075, "completion_tokens": 370.675}, "ceb_Latn|C3|low": {"n": 40, "acc": 92.5, "target_share": 0.633, "eng_share": 0.336, "pct_mostly_target": 67.5, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 349.2, "completion_tokens": 359.9}, "vie_Latn|C2|low": {"n": 40, "acc": 92.5, "target_share": 0.767, "eng_share": 0.22, "pct_mostly_target": 80.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 262.2, "completion_tokens": 273.35}, "vie_Latn|C0|low": {"n": 40, "acc": 97.5, "target_share": 0.443, "eng_share": 0.537, "pct_mostly_target": 47.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 265.25, "completion_tokens": 277.025}, "vie_Latn|C1|low": {"n": 40, "acc": 97.5, "target_share": 0.606, "eng_share": 0.379, "pct_mostly_target": 67.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 278.6, "completion_tokens": 307.375}, "tha_Thai|C0|low": {"n": 40, "acc": 85.0, "target_share": 0.312, "eng_share": 0.659, "pct_mostly_target": 27.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 357.5, "completion_tokens": 369.6}, "tha_Thai|C3|low": {"n": 40, "acc": 82.5, "target_share": 0.683, "eng_share": 0.294, "pct_mostly_target": 65.0, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 438.675, "completion_tokens": 450.675}, "tha_Thai|C1|low": {"n": 40, "acc": 85.0, "target_share": 0.467, "eng_share": 0.512, "pct_mostly_target": 42.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 405.775, "completion_tokens": 462.2}, "tha_Thai|C2|low": {"n": 40, "acc": 87.5, "target_share": 0.693, "eng_share": 0.3, "pct_mostly_target": 67.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 470.95, "completion_tokens": 505.425}, "khm_Khmr|C1|low": {"n": 40, "acc": 87.5, "target_share": 0.302, "eng_share": 0.681, "pct_mostly_target": 25.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 597.55, "completion_tokens": 779.225}, "khm_Khmr|C3|low": {"n": 40, "acc": 85.0, "target_share": 0.842, "eng_share": 0.155, "pct_mostly_target": 87.5, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 472.8, "completion_tokens": 484.175}, "vie_Latn|C3|low": {"n": 40, "acc": 97.5, "target_share": 0.687, "eng_share": 0.305, "pct_mostly_target": 67.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 201.55, "completion_tokens": 212.8}, "sun_Latn|C0|low": {"n": 40, "acc": 92.5, "target_share": 0.469, "eng_share": 0.511, "pct_mostly_target": 47.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 402.8, "completion_tokens": 413.825}, "jav_Latn|C0|low": {"n": 40, "acc": 90.0, "target_share": 0.401, "eng_share": 0.555, "pct_mostly_target": 35.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 361.3, "completion_tokens": 372.7}, "jav_Latn|C2|low": {"n": 40, "acc": 92.5, "target_share": 0.494, "eng_share": 0.415, "pct_mostly_target": 52.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 364.025, "completion_tokens": 409.375}, "zsm_Latn|C0|low": {"n": 40, "acc": 97.5, "target_share": 0.419, "eng_share": 0.558, "pct_mostly_target": 45.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 218.375, "completion_tokens": 229.525}, "lao_Laoo|C3|low": {"n": 40, "acc": 87.5, "target_share": 0.37, "eng_share": 0.548, "pct_mostly_target": 32.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 460.7, "completion_tokens": 471.65}, "mya_Mymr|C2|low": {"n": 40, "acc": 90.0, "target_share": 0.469, "eng_share": 0.51, "pct_mostly_target": 47.5, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 806.025, "completion_tokens": 979.9}, "mya_Mymr|C3|low": {"n": 40, "acc": 92.5, "target_share": 0.736, "eng_share": 0.257, "pct_mostly_target": 72.5, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 855.85, "completion_tokens": 875.55}, "mya_Mymr|C0|low": {"n": 40, "acc": 92.5, "target_share": 0.33, "eng_share": 0.631, "pct_mostly_target": 25.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 663.725, "completion_tokens": 676.1}, "mya_Mymr|C1|low": {"n": 40, "acc": 92.5, "target_share": 0.306, "eng_share": 0.654, "pct_mostly_target": 30.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 570.35, "completion_tokens": 973.725}, "hin_Deva|C1|low": {"n": 40, "acc": 90.0, "target_share": 0.26, "eng_share": 0.719, "pct_mostly_target": 17.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 469.975, "completion_tokens": 608.45}, "hin_Deva|C0|low": {"n": 40, "acc": 95.0, "target_share": 0.281, "eng_share": 0.688, "pct_mostly_target": 17.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 407.775, "completion_tokens": 418.8}, "tam_Taml|C1|low": {"n": 40, "acc": 90.0, "target_share": 0.344, "eng_share": 0.645, "pct_mostly_target": 27.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 595.675, "completion_tokens": 864}, "tam_Taml|C2|low": {"n": 40, "acc": 92.5, "target_share": 0.399, "eng_share": 0.588, "pct_mostly_target": 27.5, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 656.775, "completion_tokens": 742.925}, "hin_Deva|C3|low": {"n": 40, "acc": 97.5, "target_share": 0.679, "eng_share": 0.318, "pct_mostly_target": 65.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 504.575, "completion_tokens": 515.925}, "hin_Deva|C2|low": {"n": 40, "acc": 90.0, "target_share": 0.476, "eng_share": 0.505, "pct_mostly_target": 42.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 523.925, "completion_tokens": 579.125}, "kor_Hang|C0|low": {"n": 40, "acc": 95.0, "target_share": 0.267, "eng_share": 0.71, "pct_mostly_target": 22.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 279.875, "completion_tokens": 290.4}, "tam_Taml|C3|low": {"n": 40, "acc": 92.5, "target_share": 0.562, "eng_share": 0.43, "pct_mostly_target": 60.0, "no_reasoning_signal": 0, "empty_or_truncated": 1, "reasoning_tokens": 604, "completion_tokens": 614.475}, "khm_Khmr|C0|low": {"n": 40, "acc": 82.5, "target_share": 0.21, "eng_share": 0.772, "pct_mostly_target": 17.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 449.925, "completion_tokens": 460.95}, "khm_Khmr|C2|low": {"n": 40, "acc": 80.0, "target_share": 0.655, "eng_share": 0.337, "pct_mostly_target": 65.0, "no_reasoning_signal": 0, "empty_or_truncated": 3, "reasoning_tokens": 1000.875, "completion_tokens": 1122.25}, "kor_Hang|C3|low": {"n": 40, "acc": 95.0, "target_share": 0.529, "eng_share": 0.461, "pct_mostly_target": 55.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 270.5, "completion_tokens": 281.625}, "jpn_Jpan|C0|low": {"n": 40, "acc": 95.0, "target_share": 0.278, "eng_share": 0.699, "pct_mostly_target": 17.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 257.675, "completion_tokens": 269.05}, "kor_Hang|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.294, "eng_share": 0.691, "pct_mostly_target": 25.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 265.975, "completion_tokens": 322.275}, "lao_Laoo|C0|low": {"n": 40, "acc": 87.5, "target_share": 0.199, "eng_share": 0.779, "pct_mostly_target": 10.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 509.225, "completion_tokens": 520.25}, "arb_Arab|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.336, "eng_share": 0.643, "pct_mostly_target": 32.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 266.425, "completion_tokens": 314.55}, "jpn_Jpan|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.29, "eng_share": 0.693, "pct_mostly_target": 22.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 239.925, "completion_tokens": 294.175}, "jpn_Jpan|C3|low": {"n": 40, "acc": 95.0, "target_share": 0.228, "eng_share": 0.763, "pct_mostly_target": 15.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 253.3, "completion_tokens": 265.175}, "arb_Arab|C2|low": {"n": 40, "acc": 95.0, "target_share": 0.533, "eng_share": 0.432, "pct_mostly_target": 55.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 242.6, "completion_tokens": 267.175}, "kor_Hang|C2|low": {"n": 40, "acc": 95.0, "target_share": 0.434, "eng_share": 0.547, "pct_mostly_target": 45.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 325.825, "completion_tokens": 345.225}, "arb_Arab|C3|low": {"n": 40, "acc": 92.5, "target_share": 0.578, "eng_share": 0.402, "pct_mostly_target": 55.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 215.8, "completion_tokens": 227.075}, "yor_Latn|C1|low": {"n": 40, "acc": 77.5, "target_share": 0.534, "eng_share": 0.446, "pct_mostly_target": 67.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 762.425, "completion_tokens": 871.075}, "yor_Latn|C0|low": {"n": 40, "acc": 82.5, "target_share": 0.536, "eng_share": 0.438, "pct_mostly_target": 57.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 710.6, "completion_tokens": 722.575}, "jpn_Jpan|C2|low": {"n": 40, "acc": 95.0, "target_share": 0.67, "eng_share": 0.319, "pct_mostly_target": 60.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 236.125, "completion_tokens": 257.4}, "arb_Arab|C0|low": {"n": 40, "acc": 95.0, "target_share": 0.213, "eng_share": 0.741, "pct_mostly_target": 17.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 221, "completion_tokens": 232.3}, "yor_Latn|C2|low": {"n": 40, "acc": 77.5, "target_share": 0.456, "eng_share": 0.517, "pct_mostly_target": 40.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 616.35, "completion_tokens": 681.05}, "plt_Latn|C2|low": {"n": 40, "acc": 90.0, "target_share": 0.515, "eng_share": 0.453, "pct_mostly_target": 62.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 347.175, "completion_tokens": 359.85}, "plt_Latn|C1|low": {"n": 40, "acc": 87.5, "target_share": 0.5, "eng_share": 0.47, "pct_mostly_target": 55.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 545.35, "completion_tokens": 563.175}, "plt_Latn|C3|low": {"n": 40, "acc": 90.0, "target_share": 0.607, "eng_share": 0.377, "pct_mostly_target": 55.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 327.05, "completion_tokens": 338.05}, "ind_Latn|C0|low": {"n": 40, "acc": 95.0, "target_share": 0.472, "eng_share": 0.511, "pct_mostly_target": 52.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 271.25, "completion_tokens": 282.25}, "yor_Latn|C3|low": {"n": 40, "acc": 85.0, "target_share": 0.431, "eng_share": 0.548, "pct_mostly_target": 32.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 578.225, "completion_tokens": 594.15}, "sun_Latn|C1|low": {"n": 40, "acc": 90.0, "target_share": 0.406, "eng_share": 0.56, "pct_mostly_target": 40.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 393.575, "completion_tokens": 477.275}, "zsm_Latn|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.5, "eng_share": 0.483, "pct_mostly_target": 55.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 263.15, "completion_tokens": 338.95}, "zsm_Latn|C2|low": {"n": 40, "acc": 97.5, "target_share": 0.783, "eng_share": 0.212, "pct_mostly_target": 80.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 209.5, "completion_tokens": 240.85}, "zsm_Latn|C3|low": {"n": 40, "acc": 95.0, "target_share": 0.854, "eng_share": 0.138, "pct_mostly_target": 87.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 211.725, "completion_tokens": 222.775}, "tgl_Latn|C0|low": {"n": 40, "acc": 92.5, "target_share": 0.476, "eng_share": 0.49, "pct_mostly_target": 62.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 258.875, "completion_tokens": 269.65}, "tgl_Latn|C1|low": {"n": 40, "acc": 95.0, "target_share": 0.498, "eng_share": 0.48, "pct_mostly_target": 65.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 279.7, "completion_tokens": 340}, "lao_Laoo|C1|low": {"n": 40, "acc": 87.5, "target_share": 0.198, "eng_share": 0.769, "pct_mostly_target": 12.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 580.05, "completion_tokens": 777.2}, "lao_Laoo|C2|low": {"n": 40, "acc": 82.5, "target_share": 0.411, "eng_share": 0.563, "pct_mostly_target": 37.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 543.975, "completion_tokens": 638.55}, "tam_Taml|C0|low": {"n": 40, "acc": 92.5, "target_share": 0.377, "eng_share": 0.608, "pct_mostly_target": 30.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 544.975, "completion_tokens": 555.85}, "plt_Latn|C0|low": {"n": 40, "acc": 90.0, "target_share": 0.499, "eng_share": 0.466, "pct_mostly_target": 55.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 471.4, "completion_tokens": 482.7}, "ind_Latn|C0|high": {"n": 40, "acc": 95.0, "target_share": 0.417, "eng_share": 0.565, "pct_mostly_target": 37.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 487.975, "completion_tokens": 503.9}, "ind_Latn|C2|high": {"n": 40, "acc": 95.0, "target_share": 0.784, "eng_share": 0.202, "pct_mostly_target": 82.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 366.575, "completion_tokens": 384.475}, "jav_Latn|C0|high": {"n": 40, "acc": 90.0, "target_share": 0.49, "eng_share": 0.459, "pct_mostly_target": 45.0, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 750.975, "completion_tokens": 765.925}, "jav_Latn|C2|high": {"n": 40, "acc": 95.0, "target_share": 0.423, "eng_share": 0.531, "pct_mostly_target": 32.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 656.05, "completion_tokens": 726.025}}, "math": {"ind_Latn|C0|low": {"n": 58, "acc": 96.6, "target_share": 0.233, "eng_share": 0.697, "pct_mostly_target": 15.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 485.224, "completion_tokens": 522.931}, "ind_Latn|C2|low": {"n": 58, "acc": 96.6, "target_share": 0.713, "eng_share": 0.197, "pct_mostly_target": 73.7, "no_reasoning_signal": 1, "empty_or_truncated": 0, "reasoning_tokens": 467.914, "completion_tokens": 535.81}, "eng_Latn|C0|low": {"n": 58, "acc": 94.8, "target_share": 0.926, "eng_share": 0.0, "pct_mostly_target": 100.0, "no_reasoning_signal": 1, "empty_or_truncated": 0, "reasoning_tokens": 497.81, "completion_tokens": 543.362}, "eng_Latn|C2|low": {"n": 58, "acc": 94.8, "target_share": 0.923, "eng_share": 0.0, "pct_mostly_target": 100.0, "no_reasoning_signal": 1, "empty_or_truncated": 0, "reasoning_tokens": 531.948, "completion_tokens": 581.103}, "jav_Latn|C0|low": {"n": 58, "acc": 96.6, "target_share": 0.147, "eng_share": 0.702, "pct_mostly_target": 1.7, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 531.862, "completion_tokens": 595.586}, "jav_Latn|C2|low": {"n": 58, "acc": 96.6, "target_share": 0.146, "eng_share": 0.549, "pct_mostly_target": 1.7, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 555.741, "completion_tokens": 722.793}, "sun_Latn|C0|low": {"n": 58, "acc": 96.6, "target_share": 0.184, "eng_share": 0.688, "pct_mostly_target": 6.9, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 526.207, "completion_tokens": 556.586}, "sun_Latn|C2|low": {"n": 58, "acc": 94.8, "target_share": 0.165, "eng_share": 0.556, "pct_mostly_target": 1.7, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 513.983, "completion_tokens": 701.707}, "zsm_Latn|C0|low": {"n": 58, "acc": 91.4, "target_share": 0.206, "eng_share": 0.726, "pct_mostly_target": 12.1, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 653.483, "completion_tokens": 703.017}, "zsm_Latn|C2|low": {"n": 58, "acc": 94.8, "target_share": 0.603, "eng_share": 0.345, "pct_mostly_target": 55.2, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 623.155, "completion_tokens": 757.845}, "tgl_Latn|C0|low": {"n": 58, "acc": 98.3, "target_share": 0.167, "eng_share": 0.756, "pct_mostly_target": 1.7, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 579.879, "completion_tokens": 646.397}, "tgl_Latn|C2|low": {"n": 58, "acc": 96.6, "target_share": 0.228, "eng_share": 0.691, "pct_mostly_target": 15.5, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 561.707, "completion_tokens": 776.224}, "ceb_Latn|C0|low": {"n": 58, "acc": 94.8, "target_share": 0.181, "eng_share": 0.732, "pct_mostly_target": 8.6, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 587.241, "completion_tokens": 659.828}, "ceb_Latn|C2|low": {"n": 58, "acc": 91.4, "target_share": 0.216, "eng_share": 0.69, "pct_mostly_target": 12.1, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 560.5, "completion_tokens": 684.603}, "vie_Latn|C0|low": {"n": 58, "acc": 87.9, "target_share": 0.213, "eng_share": 0.736, "pct_mostly_target": 12.5, "no_reasoning_signal": 2, "empty_or_truncated": 0, "reasoning_tokens": 755.207, "completion_tokens": 785.448}, "vie_Latn|C2|low": {"n": 58, "acc": 87.9, "target_share": 0.561, "eng_share": 0.388, "pct_mostly_target": 53.4, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 732.362, "completion_tokens": 796.397}, "tha_Thai|C0|low": {"n": 58, "acc": 93.1, "target_share": 0.129, "eng_share": 0.811, "pct_mostly_target": 1.7, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 714.224, "completion_tokens": 734.655}, "tha_Thai|C2|low": {"n": 58, "acc": 94.8, "target_share": 0.438, "eng_share": 0.506, "pct_mostly_target": 37.9, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 792.776, "completion_tokens": 864.655}, "ind_Latn|C3|low": {"n": 58, "acc": 94.8, "target_share": 0.666, "eng_share": 0.245, "pct_mostly_target": 68.5, "no_reasoning_signal": 4, "empty_or_truncated": 0, "reasoning_tokens": 358.172, "completion_tokens": 370.948}, "jav_Latn|C3|low": {"n": 58, "acc": 93.1, "target_share": 0.287, "eng_share": 0.507, "pct_mostly_target": 26.3, "no_reasoning_signal": 1, "empty_or_truncated": 0, "reasoning_tokens": 419.897, "completion_tokens": 435.345}, "sun_Latn|C3|low": {"n": 58, "acc": 94.8, "target_share": 0.131, "eng_share": 0.688, "pct_mostly_target": 7.0, "no_reasoning_signal": 1, "empty_or_truncated": 0, "reasoning_tokens": 421.552, "completion_tokens": 440.724}, "tha_Thai|C3|low": {"n": 58, "acc": 93.1, "target_share": 0.262, "eng_share": 0.667, "pct_mostly_target": 24.1, "no_reasoning_signal": 0, "empty_or_truncated": 0, "reasoning_tokens": 539, "completion_tokens": 551.862}}, "replication": [{"source": "esscor", "lang": "eng_Latn", "cond": "cat=everyday", "effort": "default", "n": 14, "mean_target_share": 0.864, "mean_eng_share": 0.0, "pct_mostly_target": 92.9}, {"source": "esscor", "lang": "eng_Latn", "cond": "cat=everyday", "effort": "minimal", "n": 7, "mean_target_share": 0.797, "mean_eng_share": 0.0, "pct_mostly_target": 85.7}, {"source": "esscor", "lang": "eng_Latn", "cond": "cat=knowledge", "effort": "default", "n": 30, "mean_target_share": 0.869, "mean_eng_share": 0.0, "pct_mostly_target": 96.7}, {"source": "esscor", "lang": "eng_Latn", "cond": "cat=knowledge", "effort": "minimal", "n": 15, "mean_target_share": 0.931, "mean_eng_share": 0.0, "pct_mostly_target": 100.0}, {"source": "esscor", "lang": "eng_Latn", "cond": "cat=logic", "effort": "default", "n": 16, "mean_target_share": 0.884, "mean_eng_share": 0.0, "pct_mostly_target": 100.0}, {"source": "esscor", "lang": "eng_Latn", "cond": "cat=logic", "effort": "minimal", "n": 8, "mean_target_share": 0.959, "mean_eng_share": 0.0, "pct_mostly_target": 100.0}, {"source": "esscor", "lang": "eng_Latn", "cond": "cat=math", "effort": "default", "n": 40, "mean_target_share": 0.883, "mean_eng_share": 0.0, "pct_mostly_target": 97.3}, {"source": "esscor", "lang": "eng_Latn", "cond": "cat=math", "effort": "minimal", "n": 20, "mean_target_share": 0.761, "mean_eng_share": 0.0, "pct_mostly_target": 84.2}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=everyday", "effort": "default", "n": 14, "mean_target_share": 0.034, "mean_eng_share": 0.821, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=everyday", "effort": "minimal", "n": 7, "mean_target_share": 0.0, "mean_eng_share": 0.758, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=knowledge", "effort": "default", "n": 30, "mean_target_share": 0.044, "mean_eng_share": 0.855, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=knowledge", "effort": "minimal", "n": 15, "mean_target_share": 0.075, "mean_eng_share": 0.836, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=logic", "effort": "default", "n": 16, "mean_target_share": 0.134, "mean_eng_share": 0.73, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=logic", "effort": "minimal", "n": 8, "mean_target_share": 0.045, "mean_eng_share": 0.778, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=math", "effort": "default", "n": 40, "mean_target_share": 0.058, "mean_eng_share": 0.814, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Latn", "cond": "cat=math", "effort": "minimal", "n": 20, "mean_target_share": 0.068, "mean_eng_share": 0.851, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=calendar", "effort": "default", "n": 6, "mean_target_share": 0.328, "mean_eng_share": 0.398, "pct_mostly_target": 33.3}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=everyday", "effort": "default", "n": 14, "mean_target_share": 0.132, "mean_eng_share": 0.743, "pct_mostly_target": 7.1}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=everyday", "effort": "minimal", "n": 7, "mean_target_share": 0.212, "mean_eng_share": 0.655, "pct_mostly_target": 28.6}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=grammar", "effort": "default", "n": 6, "mean_target_share": 0.497, "mean_eng_share": 0.45, "pct_mostly_target": 50.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=knowledge", "effort": "default", "n": 30, "mean_target_share": 0.222, "mean_eng_share": 0.721, "pct_mostly_target": 20.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=knowledge", "effort": "minimal", "n": 15, "mean_target_share": 0.204, "mean_eng_share": 0.746, "pct_mostly_target": 13.3}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=literature", "effort": "default", "n": 18, "mean_target_share": 0.334, "mean_eng_share": 0.607, "pct_mostly_target": 27.8}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=logic", "effort": "default", "n": 16, "mean_target_share": 0.246, "mean_eng_share": 0.72, "pct_mostly_target": 18.8}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=logic", "effort": "minimal", "n": 8, "mean_target_share": 0.135, "mean_eng_share": 0.775, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=math", "effort": "default", "n": 40, "mean_target_share": 0.149, "mean_eng_share": 0.74, "pct_mostly_target": 10.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=math", "effort": "minimal", "n": 20, "mean_target_share": 0.011, "mean_eng_share": 0.879, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=numerals", "effort": "default", "n": 9, "mean_target_share": 0.168, "mean_eng_share": 0.775, "pct_mostly_target": 0.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=numwords", "effort": "default", "n": 12, "mean_target_share": 0.461, "mean_eng_share": 0.461, "pct_mostly_target": 50.0}, {"source": "esscor", "lang": "tam_Taml", "cond": "cat=script", "effort": "default", "n": 15, "mean_target_share": 0.251, "mean_eng_share": 0.65, "pct_mostly_target": 0.0}, {"source": "neulketing", "lang": "eng_Latn", "cond": "belebele_en", "effort": "low", "n": 900, "mean_target_share": 0.976, "mean_eng_share": 0.0, "pct_mostly_target": 99.9}, {"source": "neulketing", "lang": "kor_Hang", "cond": "belebele_ko", "effort": "low", "n": 900, "mean_target_share": 0.291, "mean_eng_share": 0.674, "pct_mostly_target": 22.7}, {"source": "neulketing", "lang": "kor_Hang", "cond": "belebele_mixed", "effort": "low", "n": 900, "mean_target_share": 0.185, "mean_eng_share": 0.792, "pct_mostly_target": 4.3}, {"source": "neulketing", "lang": "kor_Hang", "cond": "items", "effort": "low", "n": 577, "mean_target_share": 0.193, "mean_eng_share": 0.752, "pct_mostly_target": 3.5}, {"source": "neulketing", "lang": "kor_Hang", "cond": "items_hint", "effort": "low", "n": 145, "mean_target_share": 0.208, "mean_eng_share": 0.742, "pct_mostly_target": 2.8}, {"source": "neulketing", "lang": "kor_Hang", "cond": "items_particles", "effort": "high", "n": 201, "mean_target_share": 0.119, "mean_eng_share": 0.824, "pct_mostly_target": 0.5}], "grammar": {"K0/x2ind": {"chrf++": 33.64, "n": 100, "empty": 0}, "K0/ind2x": {"chrf++": 23.88, "n": 100, "empty": 0}, "K0/think": {"n_seg": 62, "target_share": 0.318, "eng_share": 0.19, "other_share": 0.492, "first_eng_pos": 0.0, "top_other": "ind_Latn"}, "K0/prompt_tokens": 3034, "K1/x2ind": {"chrf++": 26.99, "n": 100, "empty": 0}, "K1/ind2x": {"chrf++": 24.21, "n": 100, "empty": 0}, "K1/think": {"n_seg": 254, "target_share": 0.793, "eng_share": 0.039, "other_share": 0.168, "first_eng_pos": 0.0, "top_other": "ind_Latn"}, "K1/prompt_tokens": 248587, "K2/x2ind": {"chrf++": 33.71, "n": 100, "empty": 0}, "K2/ind2x": {"chrf++": 23.7, "n": 100, "empty": 0}, "K2/think": {"n_seg": 208, "target_share": 0.575, "eng_share": 0.057, "other_share": 0.369, "first_eng_pos": 0.0, "top_other": "ind_Latn"}, "K2/prompt_tokens": 15400, "K3/x2ind": {"chrf++": 49.12, "n": 100, "empty": 0}, "K3/ind2x": {"chrf++": 32.73, "n": 100, "empty": 0}, "K3/think": {"n_seg": 267, "target_share": 0.438, "eng_share": 0.032, "other_share": 0.53, "first_eng_pos": 0.0, "top_other": "ind_Latn"}, "K3/prompt_tokens": 402849, "K0/think_noquote": {"n_seg": 40, "target_share": 0.0, "eng_share": 0.313, "other_share": 0.687, "first_eng_pos": 0.0, "top_other": "ind_Latn"}, "K3/think_noquote": {"n_seg": 151, "target_share": 0.067, "eng_share": 0.052, "other_share": 0.881, "first_eng_pos": 0.0, "top_other": "ind_Latn"}, "K2/think_noquote": {"n_seg": 115, "target_share": 0.253, "eng_share": 0.127, "other_share": 0.62, "first_eng_pos": 0.0, "top_other": "ind_Latn"}, "K1/think_noquote": {"n_seg": 76, "target_share": 0.221, "eng_share": 0.188, "other_share": 0.591, "first_eng_pos": 0.0, "top_other": "ind_Latn"}}, "lid_calibration": {"eng_Latn": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 1.0, "confusions": [["eng_Latn", 300]]}, "ind_Latn": {"n": 300, "acc_with_alias": 0.983, "pred_eng": 0.0, "confusions": [["ind_Latn", 291], ["zsm_Latn", 4], ["bjn_Latn", 2]]}, "jav_Latn": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["jav_Latn", 300]]}, "sun_Latn": {"n": 300, "acc_with_alias": 0.997, "pred_eng": 0.0, "confusions": [["sun_Latn", 299], ["jav_Latn", 1]]}, "zsm_Latn": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["zsm_Latn", 256], ["ind_Latn", 44]]}, "tgl_Latn": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["fil_Latn", 300]]}, "ceb_Latn": {"n": 300, "acc_with_alias": 0.977, "pred_eng": 0.003, "confusions": [["ceb_Latn", 293], ["hil_Latn", 4], ["eng_Latn", 1]]}, "vie_Latn": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["vie_Latn", 300]]}, "tha_Thai": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["tha_Thai", 300]]}, "khm_Khmr": {"n": 300, "acc_with_alias": 0.997, "pred_eng": 0.0, "confusions": [["khm_Khmr", 299], ["ayr_Latn", 1]]}, "lao_Laoo": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["lao_Laoo", 300]]}, "mya_Mymr": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["mya_Mymr", 300]]}, "hin_Deva": {"n": 300, "acc_with_alias": 0.997, "pred_eng": 0.0, "confusions": [["hin_Deva", 299], ["anp_Deva", 1]]}, "tam_Taml": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["tam_Taml", 300]]}, "kor_Hang": {"n": 300, "acc_with_alias": 0.997, "pred_eng": 0.0, "confusions": [["kor_Hang", 299], ["zxx_Latn", 1]]}, "jpn_Jpan": {"n": 300, "acc_with_alias": 1.0, "pred_eng": 0.0, "confusions": [["jpn_Jpan", 300]]}, "arb_Arab": {"n": 300, "acc_with_alias": 0.823, "pred_eng": 0.0, "confusions": [["arb_Arab", 247], ["ars_Arab", 23], ["ary_Arab", 13]]}, "yor_Latn": {"n": 300, "acc_with_alias": 0.993, "pred_eng": 0.0, "confusions": [["yor_Latn", 298], ["ife_Latn", 1], ["kyu_Latn", 1]]}, "plt_Latn": {"n": 300, "acc_with_alias": 0.993, "pred_eng": 0.0, "confusions": [["plt_Latn", 298], ["xmv_Latn", 1], ["skg_Latn", 1]]}}, "tiers": {"ind_Latn": 3, "jav_Latn": 1, "sun_Latn": 1, "zsm_Latn": 3, "tgl_Latn": 3, "ceb_Latn": 3, "vie_Latn": 4, "tha_Thai": 3, "khm_Khmr": 1, "lao_Laoo": 2, "mya_Mymr": 1, "hin_Deva": 4, "tam_Taml": 3, "kor_Hang": 4, "jpn_Jpan": 5, "arb_Arab": 5, "yor_Latn": 2, "plt_Latn": 1, "eng_Latn": 5, "bug_Latn": 1}, "verdicts": {"H1": {"pass": true, "detail": "15/18 languages with C0 eng_share >= 0.5", "langs": ["arb_Arab", "ceb_Latn", "hin_Deva", "ind_Latn", "jav_Latn", "jpn_Jpan", "khm_Khmr", "kor_Hang", "lao_Laoo", "mya_Mymr", "sun_Latn", "tam_Taml", "tha_Thai", "vie_Latn", "zsm_Latn"]}, "H2": {"pass": true, "detail": "C2 > C0 in 16/18; of those 15 still < 0.8"}, "H3": {"pass": true, "detail": "C3 > C2 in 13/18"}, "H4": {"pass": true, "detail": "Spearman(Joshi tier, C2 target_share) = 0.281 over 18 languages (pre-registration phrased it as a negative correlation with 'low-resource-ness'; tier is ordered high=5); permutation p = 0.269 (direction as predicted, not significant)"}, "H5": {"pass": true, "detail": "mean acc(C2)-acc(C0) = -0.2 pts over 9 langs"}, "H6": {"pass": false, "detail": "x2ind: [33.64, 26.99, 33.71, 49.12]; ind2x: [23.88, 24.21, 23.7, 32.73]"}, "H7": {"pass": true, "detail": "K3 think target_share = 0.438"}}, "atlas_eng_share_ci95": {"arb_Arab|C0": [0.663, 0.814], "arb_Arab|C1": [0.561, 0.725], "arb_Arab|C2": [0.323, 0.552], "arb_Arab|C3": [0.292, 0.518], "ceb_Latn|C0": [0.422, 0.599], "ceb_Latn|C1": [0.431, 0.593], "ceb_Latn|C2": [0.407, 0.595], "ceb_Latn|C3": [0.227, 0.451], "eng_Latn|C0": [0.0, 0.0], "eng_Latn|C1": [0.0, 0.0], "eng_Latn|C2": [0.0, 0.0], "eng_Latn|C3": [0.0, 0.0], "hin_Deva|C0": [0.608, 0.766], "hin_Deva|C1": [0.651, 0.786], "hin_Deva|C2": [0.397, 0.617], "hin_Deva|C3": [0.19, 0.453], "ind_Latn|C0": [0.423, 0.593], "ind_Latn|C1": [0.381, 0.565], "ind_Latn|C2": [0.05, 0.183], "ind_Latn|C3": [0.031, 0.156], "jav_Latn|C0": [0.478, 0.638], "jav_Latn|C1": [0.374, 0.555], "jav_Latn|C2": [0.32, 0.51], "jav_Latn|C3": [0.113, 0.306], "jpn_Jpan|C0": [0.623, 0.774], "jpn_Jpan|C1": [0.588, 0.787], "jpn_Jpan|C2": [0.203, 0.442], "jpn_Jpan|C3": [0.668, 0.85], "khm_Khmr|C0": [0.695, 0.845], "khm_Khmr|C1": [0.591, 0.769], "khm_Khmr|C2": [0.214, 0.466], "khm_Khmr|C3": [0.061, 0.267], "kor_Hang|C0": [0.629, 0.788], "kor_Hang|C1": [0.617, 0.762], "kor_Hang|C2": [0.445, 0.647], "kor_Hang|C3": [0.343, 0.582], "lao_Laoo|C0": [0.717, 0.843], "lao_Laoo|C1": [0.697, 0.837], "lao_Laoo|C2": [0.443, 0.678], "lao_Laoo|C3": [0.409, 0.681], "mya_Mymr|C0": [0.552, 0.711], "mya_Mymr|C1": [0.575, 0.73], "mya_Mymr|C2": [0.409, 0.613], "mya_Mymr|C3": [0.147, 0.368], "plt_Latn|C0": [0.388, 0.545], "plt_Latn|C1": [0.404, 0.541], "plt_Latn|C2": [0.351, 0.553], "plt_Latn|C3": [0.262, 0.491], "sun_Latn|C0": [0.431, 0.593], "sun_Latn|C1": [0.486, 0.634], "sun_Latn|C2": [0.513, 0.72], "sun_Latn|C3": [0.181, 0.368], "tam_Taml|C0": [0.536, 0.678], "tam_Taml|C1": [0.576, 0.713], "tam_Taml|C2": [0.505, 0.671], "tam_Taml|C3": [0.315, 0.55], "tgl_Latn|C0": [0.402, 0.589], "tgl_Latn|C1": [0.4, 0.566], "tgl_Latn|C2": [0.425, 0.582], "tgl_Latn|C3": [0.141, 0.359], "tha_Thai|C0": [0.578, 0.739], "tha_Thai|C1": [0.398, 0.624], "tha_Thai|C2": [0.189, 0.412], "tha_Thai|C3": [0.185, 0.417], "vie_Latn|C0": [0.451, 0.621], "vie_Latn|C1": [0.279, 0.484], "vie_Latn|C2": [0.133, 0.317], "vie_Latn|C3": [0.19, 0.428], "yor_Latn|C0": [0.386, 0.493], "yor_Latn|C1": [0.394, 0.505], "yor_Latn|C2": [0.469, 0.568], "yor_Latn|C3": [0.455, 0.638], "zsm_Latn|C0": [0.475, 0.643], "zsm_Latn|C1": [0.383, 0.591], "zsm_Latn|C2": [0.114, 0.318], "zsm_Latn|C3": [0.074, 0.204]}, "robustness": {"arb_Arab|C0": {"eng_share_prereg": 0.741, "eng_share_noquote": 0.893, "target_share_noquote": 0.049, "lingua_eng": 0.642, "lingua_eng_noquote": 0.793}, "arb_Arab|C1": {"eng_share_prereg": 0.643, "eng_share_noquote": 0.832, "target_share_noquote": 0.138, "lingua_eng": 0.52, "lingua_eng_noquote": 0.676}, "arb_Arab|C2": {"eng_share_prereg": 0.432, "eng_share_noquote": 0.611, "target_share_noquote": 0.329, "lingua_eng": 0.373, "lingua_eng_noquote": 0.539}, "arb_Arab|C3": {"eng_share_prereg": 0.402, "eng_share_noquote": 0.553, "target_share_noquote": 0.424, "lingua_eng": 0.294, "lingua_eng_noquote": 0.405}, "ceb_Latn|C0": {"eng_share_prereg": 0.509, "eng_share_noquote": 0.77, "target_share_noquote": 0.16, "lingua_eng": 0.457, "lingua_eng_noquote": 0.679}, "ceb_Latn|C1": {"eng_share_prereg": 0.51, "eng_share_noquote": 0.748, "target_share_noquote": 0.194, "lingua_eng": 0.425, "lingua_eng_noquote": 0.595}, "ceb_Latn|C2": {"eng_share_prereg": 0.499, "eng_share_noquote": 0.701, "target_share_noquote": 0.233, "lingua_eng": 0.432, "lingua_eng_noquote": 0.613}, "ceb_Latn|C3": {"eng_share_prereg": 0.336, "eng_share_noquote": 0.461, "target_share_noquote": 0.523, "lingua_eng": 0.326, "lingua_eng_noquote": 0.437}, "eng_Latn|C0": {"eng_share_prereg": 0.0, "eng_share_noquote": 0.0, "target_share_noquote": 0.959, "lingua_eng": 0.93, "lingua_eng_noquote": 0.908}, "eng_Latn|C1": {"eng_share_prereg": 0.0, "eng_share_noquote": 0.0, "target_share_noquote": 0.941, "lingua_eng": 0.943, "lingua_eng_noquote": 0.963}, "eng_Latn|C2": {"eng_share_prereg": 0.0, "eng_share_noquote": 0.0, "target_share_noquote": 0.933, "lingua_eng": 0.926, "lingua_eng_noquote": 0.948}, "eng_Latn|C3": {"eng_share_prereg": 0.0, "eng_share_noquote": 0.0, "target_share_noquote": 0.979, "lingua_eng": 0.977, "lingua_eng_noquote": 0.973}, "hin_Deva|C0": {"eng_share_prereg": 0.688, "eng_share_noquote": 0.868, "target_share_noquote": 0.098, "lingua_eng": 0.71, "lingua_eng_noquote": 0.879}, "hin_Deva|C1": {"eng_share_prereg": 0.719, "eng_share_noquote": 0.857, "target_share_noquote": 0.117, "lingua_eng": 0.715, "lingua_eng_noquote": 0.853}, "hin_Deva|C2": {"eng_share_prereg": 0.505, "eng_share_noquote": 0.643, "target_share_noquote": 0.332, "lingua_eng": 0.516, "lingua_eng_noquote": 0.653}, "hin_Deva|C3": {"eng_share_prereg": 0.318, "eng_share_noquote": 0.457, "target_share_noquote": 0.539, "lingua_eng": 0.319, "lingua_eng_noquote": 0.458}, "ind_Latn|C0": {"eng_share_prereg": 0.511, "eng_share_noquote": 0.806, "target_share_noquote": 0.159, "lingua_eng": 0.49, "lingua_eng_noquote": 0.774}, "ind_Latn|C1": {"eng_share_prereg": 0.472, "eng_share_noquote": 0.712, "target_share_noquote": 0.261, "lingua_eng": 0.451, "lingua_eng_noquote": 0.667}, "ind_Latn|C2": {"eng_share_prereg": 0.11, "eng_share_noquote": 0.387, "target_share_noquote": 0.573, "lingua_eng": 0.106, "lingua_eng_noquote": 0.299}, "ind_Latn|C3": {"eng_share_prereg": 0.084, "eng_share_noquote": 0.174, "target_share_noquote": 0.821, "lingua_eng": 0.09, "lingua_eng_noquote": 0.174}, "jav_Latn|C0": {"eng_share_prereg": 0.555, "eng_share_noquote": 0.798, "target_share_noquote": 0.16, "lingua_eng": 0.571, "lingua_eng_noquote": 0.828}, "jav_Latn|C1": {"eng_share_prereg": 0.469, "eng_share_noquote": 0.659, "target_share_noquote": 0.253, "lingua_eng": 0.486, "lingua_eng_noquote": 0.64}, "jav_Latn|C2": {"eng_share_prereg": 0.415, "eng_share_noquote": 0.616, "target_share_noquote": 0.249, "lingua_eng": 0.444, "lingua_eng_noquote": 0.615}, "jav_Latn|C3": {"eng_share_prereg": 0.203, "eng_share_noquote": 0.287, "target_share_noquote": 0.691, "lingua_eng": 0.269, "lingua_eng_noquote": 0.343}, "jpn_Jpan|C0": {"eng_share_prereg": 0.699, "eng_share_noquote": 0.872, "target_share_noquote": 0.096, "lingua_eng": 0.595, "lingua_eng_noquote": 0.756}, "jpn_Jpan|C1": {"eng_share_prereg": 0.693, "eng_share_noquote": 0.786, "target_share_noquote": 0.195, "lingua_eng": 0.607, "lingua_eng_noquote": 0.693}, "jpn_Jpan|C2": {"eng_share_prereg": 0.319, "eng_share_noquote": 0.405, "target_share_noquote": 0.581, "lingua_eng": 0.245, "lingua_eng_noquote": 0.316}, "jpn_Jpan|C3": {"eng_share_prereg": 0.763, "eng_share_noquote": 0.844, "target_share_noquote": 0.147, "lingua_eng": 0.564, "lingua_eng_noquote": 0.636}, "khm_Khmr|C0": {"eng_share_prereg": 0.772, "eng_share_noquote": 0.884, "target_share_noquote": 0.087, "lingua_eng": 0.9, "lingua_eng_noquote": 0.933}, "khm_Khmr|C1": {"eng_share_prereg": 0.681, "eng_share_noquote": 0.853, "target_share_noquote": 0.127, "lingua_eng": 0.785, "lingua_eng_noquote": 0.887}, "khm_Khmr|C2": {"eng_share_prereg": 0.337, "eng_share_noquote": 0.372, "target_share_noquote": 0.62, "lingua_eng": 0.432, "lingua_eng_noquote": 0.499}, "khm_Khmr|C3": {"eng_share_prereg": 0.155, "eng_share_noquote": 0.191, "target_share_noquote": 0.804, "lingua_eng": 0.301, "lingua_eng_noquote": 0.35}, "kor_Hang|C0": {"eng_share_prereg": 0.71, "eng_share_noquote": 0.849, "target_share_noquote": 0.119, "lingua_eng": 0.62, "lingua_eng_noquote": 0.748}, "kor_Hang|C1": {"eng_share_prereg": 0.691, "eng_share_noquote": 0.858, "target_share_noquote": 0.123, "lingua_eng": 0.607, "lingua_eng_noquote": 0.759}, "kor_Hang|C2": {"eng_share_prereg": 0.547, "eng_share_noquote": 0.736, "target_share_noquote": 0.233, "lingua_eng": 0.455, "lingua_eng_noquote": 0.631}, "kor_Hang|C3": {"eng_share_prereg": 0.461, "eng_share_noquote": 0.576, "target_share_noquote": 0.412, "lingua_eng": 0.361, "lingua_eng_noquote": 0.46}, "lao_Laoo|C0": {"eng_share_prereg": 0.779, "eng_share_noquote": 0.899, "target_share_noquote": 0.074, "lingua_eng": 0.861, "lingua_eng_noquote": 0.914}, "lao_Laoo|C1": {"eng_share_prereg": 0.769, "eng_share_noquote": 0.881, "target_share_noquote": 0.083, "lingua_eng": 0.808, "lingua_eng_noquote": 0.85}, "lao_Laoo|C2": {"eng_share_prereg": 0.563, "eng_share_noquote": 0.682, "target_share_noquote": 0.295, "lingua_eng": 0.59, "lingua_eng_noquote": 0.653}, "lao_Laoo|C3": {"eng_share_prereg": 0.548, "eng_share_noquote": 0.625, "target_share_noquote": 0.286, "lingua_eng": 0.636, "lingua_eng_noquote": 0.67}, "mya_Mymr|C0": {"eng_share_prereg": 0.631, "eng_share_noquote": 0.797, "target_share_noquote": 0.157, "lingua_eng": 0.832, "lingua_eng_noquote": 0.917}, "mya_Mymr|C1": {"eng_share_prereg": 0.654, "eng_share_noquote": 0.824, "target_share_noquote": 0.119, "lingua_eng": 0.815, "lingua_eng_noquote": 0.915}, "mya_Mymr|C2": {"eng_share_prereg": 0.51, "eng_share_noquote": 0.676, "target_share_noquote": 0.294, "lingua_eng": 0.647, "lingua_eng_noquote": 0.756}, "mya_Mymr|C3": {"eng_share_prereg": 0.257, "eng_share_noquote": 0.452, "target_share_noquote": 0.536, "lingua_eng": 0.413, "lingua_eng_noquote": 0.547}, "plt_Latn|C0": {"eng_share_prereg": 0.466, "eng_share_noquote": 0.751, "target_share_noquote": 0.188, "lingua_eng": 0.439, "lingua_eng_noquote": 0.715}, "plt_Latn|C1": {"eng_share_prereg": 0.47, "eng_share_noquote": 0.715, "target_share_noquote": 0.246, "lingua_eng": 0.485, "lingua_eng_noquote": 0.71}, "plt_Latn|C2": {"eng_share_prereg": 0.453, "eng_share_noquote": 0.71, "target_share_noquote": 0.228, "lingua_eng": 0.399, "lingua_eng_noquote": 0.587}, "plt_Latn|C3": {"eng_share_prereg": 0.377, "eng_share_noquote": 0.517, "target_share_noquote": 0.468, "lingua_eng": 0.335, "lingua_eng_noquote": 0.421}, "sun_Latn|C0": {"eng_share_prereg": 0.511, "eng_share_noquote": 0.787, "target_share_noquote": 0.201, "lingua_eng": 0.522, "lingua_eng_noquote": 0.757}, "sun_Latn|C1": {"eng_share_prereg": 0.56, "eng_share_noquote": 0.762, "target_share_noquote": 0.198, "lingua_eng": 0.545, "lingua_eng_noquote": 0.724}, "sun_Latn|C2": {"eng_share_prereg": 0.616, "eng_share_noquote": 0.712, "target_share_noquote": 0.248, "lingua_eng": 0.618, "lingua_eng_noquote": 0.681}, "sun_Latn|C3": {"eng_share_prereg": 0.272, "eng_share_noquote": 0.402, "target_share_noquote": 0.461, "lingua_eng": 0.315, "lingua_eng_noquote": 0.422}, "tam_Taml|C0": {"eng_share_prereg": 0.608, "eng_share_noquote": 0.758, "target_share_noquote": 0.219, "lingua_eng": 0.56, "lingua_eng_noquote": 0.704}, "tam_Taml|C1": {"eng_share_prereg": 0.645, "eng_share_noquote": 0.831, "target_share_noquote": 0.154, "lingua_eng": 0.585, "lingua_eng_noquote": 0.759}, "tam_Taml|C2": {"eng_share_prereg": 0.588, "eng_share_noquote": 0.723, "target_share_noquote": 0.258, "lingua_eng": 0.527, "lingua_eng_noquote": 0.653}, "tam_Taml|C3": {"eng_share_prereg": 0.43, "eng_share_noquote": 0.538, "target_share_noquote": 0.453, "lingua_eng": 0.373, "lingua_eng_noquote": 0.468}, "tgl_Latn|C0": {"eng_share_prereg": 0.49, "eng_share_noquote": 0.79, "target_share_noquote": 0.158, "lingua_eng": 0.425, "lingua_eng_noquote": 0.687}, "tgl_Latn|C1": {"eng_share_prereg": 0.48, "eng_share_noquote": 0.782, "target_share_noquote": 0.19, "lingua_eng": 0.461, "lingua_eng_noquote": 0.721}, "tgl_Latn|C2": {"eng_share_prereg": 0.501, "eng_share_noquote": 0.755, "target_share_noquote": 0.222, "lingua_eng": 0.459, "lingua_eng_noquote": 0.694}, "tgl_Latn|C3": {"eng_share_prereg": 0.244, "eng_share_noquote": 0.389, "target_share_noquote": 0.527, "lingua_eng": 0.229, "lingua_eng_noquote": 0.34}, "tha_Thai|C0": {"eng_share_prereg": 0.659, "eng_share_noquote": 0.855, "target_share_noquote": 0.095, "lingua_eng": 0.585, "lingua_eng_noquote": 0.772}, "tha_Thai|C1": {"eng_share_prereg": 0.512, "eng_share_noquote": 0.637, "target_share_noquote": 0.342, "lingua_eng": 0.445, "lingua_eng_noquote": 0.543}, "tha_Thai|C2": {"eng_share_prereg": 0.3, "eng_share_noquote": 0.416, "target_share_noquote": 0.575, "lingua_eng": 0.249, "lingua_eng_noquote": 0.321}, "tha_Thai|C3": {"eng_share_prereg": 0.294, "eng_share_noquote": 0.407, "target_share_noquote": 0.562, "lingua_eng": 0.239, "lingua_eng_noquote": 0.282}, "vie_Latn|C0": {"eng_share_prereg": 0.537, "eng_share_noquote": 0.77, "target_share_noquote": 0.201, "lingua_eng": 0.512, "lingua_eng_noquote": 0.732}, "vie_Latn|C1": {"eng_share_prereg": 0.379, "eng_share_noquote": 0.654, "target_share_noquote": 0.321, "lingua_eng": 0.352, "lingua_eng_noquote": 0.603}, "vie_Latn|C2": {"eng_share_prereg": 0.22, "eng_share_noquote": 0.471, "target_share_noquote": 0.486, "lingua_eng": 0.209, "lingua_eng_noquote": 0.457}, "vie_Latn|C3": {"eng_share_prereg": 0.305, "eng_share_noquote": 0.44, "target_share_noquote": 0.552, "lingua_eng": 0.262, "lingua_eng_noquote": 0.376}, "yor_Latn|C0": {"eng_share_prereg": 0.438, "eng_share_noquote": 0.643, "target_share_noquote": 0.326, "lingua_eng": 0.378, "lingua_eng_noquote": 0.55}, "yor_Latn|C1": {"eng_share_prereg": 0.446, "eng_share_noquote": 0.614, "target_share_noquote": 0.365, "lingua_eng": 0.389, "lingua_eng_noquote": 0.535}, "yor_Latn|C2": {"eng_share_prereg": 0.517, "eng_share_noquote": 0.623, "target_share_noquote": 0.347, "lingua_eng": 0.426, "lingua_eng_noquote": 0.519}, "yor_Latn|C3": {"eng_share_prereg": 0.548, "eng_share_noquote": 0.594, "target_share_noquote": 0.384, "lingua_eng": 0.466, "lingua_eng_noquote": 0.504}, "zsm_Latn|C0": {"eng_share_prereg": 0.558, "eng_share_noquote": 0.819, "target_share_noquote": 0.147, "lingua_eng": 0.499, "lingua_eng_noquote": 0.729}, "zsm_Latn|C1": {"eng_share_prereg": 0.483, "eng_share_noquote": 0.726, "target_share_noquote": 0.247, "lingua_eng": 0.451, "lingua_eng_noquote": 0.666}, "zsm_Latn|C2": {"eng_share_prereg": 0.212, "eng_share_noquote": 0.49, "target_share_noquote": 0.503, "lingua_eng": 0.194, "lingua_eng_noquote": 0.429}, "zsm_Latn|C3": {"eng_share_prereg": 0.138, "eng_share_noquote": 0.319, "target_share_noquote": 0.671, "lingua_eng": 0.129, "lingua_eng_noquote": 0.271}}, "detector_agreement": {"n": 3040, "mean_abs_diff": 0.134, "same_majority": 0.838}, "condition_means": {"C0": {"eng_share": 0.603, "target_share": 0.368, "acc": 91.806}, "C1": {"eng_share": 0.571, "target_share": 0.405, "acc": 91.389}, "C2": {"eng_share": 0.425, "target_share": 0.55, "acc": 90.833}, "C3": {"eng_share": 0.341, "target_share": 0.638, "acc": 91.806}}, "math_mcnemar_C0_vs_C2": {"C0_only_correct": 7, "C2_only_correct": 6, "p_exact": 1.0}, "routes": {"reflection.ai": 2360, "huggingface": 2116}, "cost_usd": 2.8987, "sea_thinking_rows": 1453}