TamilBench / index.html
esscor's picture
v2: Belebele 900x4, tokenizer league, hard split, writing, Token Saver
2bc3590 verified
Raw History Blame Contribute Delete
25.9 kB
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>TamilBench · What does Beam charge to think in Tamil?</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;800&family=Noto+Sans+Tamil:wght@400;600&display=swap" rel="stylesheet">
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js"></script>
<script src="data.js"></script>
<script src="data2.js"></script>
<style>
:root{--bg:#0d0f14;--card:#161a22;--line:#262c38;--txt:#e8ebf1;--mut:#9aa3b2;--en:#6aa6ff;--ta:#ff8a3d;--tl:#3ddc97;--acc:#ffd166}
*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--txt);font-family:Inter,'Noto Sans Tamil',system-ui,sans-serif;line-height:1.55}
.wrap{max-width:1080px;margin:0 auto;padding:28px 20px 60px}
h1{font-size:clamp(28px,4.5vw,46px);line-height:1.1;margin:8px 0 10px;font-weight:800}
h1 .ta{color:var(--ta)}h2{font-size:24px;margin:44px 0 12px}h3{margin:0 0 6px;font-size:16px}
.kicker{color:var(--acc);font-weight:600;letter-spacing:.06em;text-transform:uppercase;font-size:13px}
.lead{color:var(--mut);font-size:18px;max-width:780px}
.grid{display:grid;gap:14px}.g4{grid-template-columns:repeat(auto-fit,minmax(220px,1fr))}.g2{grid-template-columns:repeat(auto-fit,minmax(420px,1fr))}
.card{background:var(--card);border:1px solid var(--line);border-radius:14px;padding:18px}
.big{font-size:38px;font-weight:800;line-height:1}.mut{color:var(--mut);font-size:14px}
.pill{display:inline-block;padding:2px 9px;border-radius:99px;font-size:12px;font-weight:600;margin-right:6px}
.en{background:#6aa6ff22;color:var(--en)}.ta{background:#ff8a3d22;color:var(--ta)}.tl{background:#3ddc9722;color:var(--tl)}
textarea{width:100%;min-height:90px;background:#0b0d12;color:var(--txt);border:1px solid var(--line);border-radius:10px;padding:12px;font:inherit;font-size:16px}
button,select{background:#222837;color:var(--txt);border:1px solid var(--line);border-radius:9px;padding:8px 12px;font:inherit;cursor:pointer}
button:hover{border-color:var(--acc)}
.toks{display:flex;flex-wrap:wrap;gap:3px;margin-top:10px;font-family:ui-monospace,monospace;font-size:13px}
.toks span{padding:2px 5px;border-radius:5px;background:#ffffff10}
table{width:100%;border-collapse:collapse;font-size:14px}th,td{text-align:left;padding:7px 8px;border-bottom:1px solid var(--line);vertical-align:top}
th{color:var(--mut);font-weight:600}tr.row{cursor:pointer}tr.row:hover{background:#ffffff08}
.ok{color:var(--tl)}.bad{color:#ff5d73}
.trace{white-space:pre-wrap;font-size:13px;color:var(--mut);background:#0b0d12;border-radius:8px;padding:10px;margin-top:6px}
ul.f li{margin:8px 0}a{color:var(--en)}
.foot{margin-top:50px;color:var(--mut);font-size:13px;border-top:1px solid var(--line);padding-top:18px}
canvas{max-height:320px}
</style>
</head>
<body><div class="wrap">
<div class="kicker">Build with Beam · TamilBench v2</div>
<h1>Beam reads <span class="ta">தமிழ்</span> well.<br>Its tokenizer makes Tamil pay 6× for it.</h1>
<p class="lead">80 million people speak Tamil, and most of them type it two ways: in Tamil script and in “Tanglish” (Tamil in Latin letters). I asked <b>Beam-1.0-501B-A23B</b> the same 50 questions in English, Tamil and Tanglish, plus 22 questions that only exist in Tamil, and measured accuracy, prompt tokens, reasoning tokens and dollars. <span id="calls"></span></p>
<div class="grid g4" style="margin-top:22px">
<div class="card"><div class="big" id="s-acc"></div><div class="mut">accuracy on the parallel set, in all three languages, at default <i>and</i> minimal reasoning effort</div></div>
<div class="card"><div class="big" style="color:var(--ta)" id="s-tok"></div><div class="mut">more input tokens for the same question in Tamil script (tokenizer: <span id="s-tpw"></span> tokens per Tamil word)</div></div>
<div class="card"><div class="big" style="color:var(--tl)" id="s-rsn"></div><div class="mut">more reasoning tokens when the question is in Tanglish; Beam quietly translates before it thinks</div></div>
<div class="card"><div class="big" style="color:var(--acc)" id="s-cost"></div><div class="mut">cost per query vs English (Tamil / Tanglish) at Baseten’s $0.40 in / $1.20 out per 1M</div></div>
</div>
<div class="card" style="margin-top:22px;border-color:#ffd16655"><b style="color:var(--acc)">New in v2:</b> <span class="mut">900 Belebele reading-comprehension questions in four input modes, a tokenizer league against 7 other open models, a 56-item hard Tamil-script split, a Tamil writing test, and the Token Saver.</span> <span id="v2calls" class="mut"></span></div>
<div class="grid g4" style="margin-top:14px">
<div class="card"><div class="big" style="color:var(--ta)" id="v-gap"></div><div class="mut">accuracy drop on Belebele when the same passage and question are in Tamil instead of English (<span id="v-accs"></span>)</div></div>
<div class="card"><div class="big" style="color:var(--ta)" id="v-ratio"></div><div class="mut">Tamil tokens per English token on 488 FLORES passages. Beam shares Qwen3's tokenizer; Gemma 3 needs only <span id="v-gem"></span></div></div>
<div class="card"><div class="big" style="color:var(--tl)" id="v-save"></div><div class="mut">fewer input tokens when the Tamil passage is auto-converted to Tanglish. The catch: accuracy falls to <span id="v-tracc"></span> and Beam thinks 2× longer, so the bill goes <i>up</i> 1.5×</div></div>
<div class="card"><div class="big" style="color:var(--acc)" id="v-cost"></div><div class="mut">cost of a Tamil Belebele query vs English, because Beam also thinks <span id="v-rsn"></span> longer in Tamil</div></div>
</div>
<h2>Belebele: same 900 questions, four ways to ask</h2>
<div class="grid g2">
<div class="card"><h3>Accuracy by input mode</h3><canvas id="b1"></canvas>
<p class="mut"><b>English</b>: all English. <b>Tamil</b>: all Tamil. <b>Mixed</b>: Tamil passage, English question and options (how most Indian apps actually prompt). <b>Auto-Tanglish</b>: the Tamil prompt machine-transliterated to Latin letters by the Token Saver below.</p></div>
<div class="card"><h3>Cost per 1,000 questions and what it buys</h3><canvas id="b2"></canvas>
<p class="mut" id="b2note"></p></div>
</div>
<h2>Tokenizer league: what Tamil costs on 8 open models</h2>
<div class="card"><canvas id="lg_"></canvas>
<p class="mut">Same 488 FLORES-200 passages (the Belebele source), tokenized with each model's own <code>tokenizer.json</code>. Bars are Tamil tokens ÷ English tokens: 1.0× would be no tax. Beam's counts are identical to Qwen3's on every passage, so Beam inherits Qwen's weak Tamil vocabulary (about 0.9 Tamil characters per token, so a Tamil letter often costs more than one token).</p></div>
<h2>Hard Tamil: script-level tasks</h2>
<div class="grid g2">
<div class="card"><canvas id="hd"></canvas></div>
<div class="card"><div class="big" id="h-acc"></div><div class="mut" style="margin-bottom:10px">on <span id="h-n"></span> hard items × 3 samples. Answers are computed by code, not by hand.</div>
<div class="mut" id="h-note">Beam knows Tamil grammar, ancient numerals, Thirukkural and the calendar cold. It breaks when it has to <i>manipulate</i> letters, like reversing a word or counting letters across a sentence. That's what you'd expect when one Tamil letter is split across byte-level tokens.</div></div>
</div>
<h2>Does Beam <i>write</i> good Tamil?</h2>
<div class="grid g4">
<div class="card"><div class="big" id="w-flu"></div><div class="mut">fluency / 5 on 15 open-ended prompts (rent advance, UPI fraud, dengue, a complaint letter…), judged by Gemma-3-27B and DeepSeek-V3.1</div></div>
<div class="card"><div class="big" id="w-cor"></div><div class="mut">correctness / 5, same judges</div></div>
<div class="card"><div class="big" style="color:var(--tl)" id="w-reg"></div><div class="mut">Tanglish questions answered in Tanglish. The others switched to Tamil script or a Tamil-English mix (checked by script share, not by a judge)</div></div>
<div class="card"><div class="big" style="color:var(--ta)" id="w-out"></div><div class="mut">more output tokens for a Tamil-script answer than the same answer in Tanglish. Output is billed at 3× input</div></div>
</div>
<h2>v1: the parallel set (English / Tamil / Tanglish)</h2>
<div class="grid g2">
<div class="card"><h3>Where the tokens go (avg per question, default effort)</h3><canvas id="c1"></canvas>
<p class="mut">Tamil script is expensive to <i>read</i>; Tanglish is cheap to read but expensive to <i>think</i> about. They land at almost the same price.</p></div>
<div class="card"><h3>Cost per 1,000 questions: default vs <code>reasoning_effort="minimal"</code></h3><canvas id="c2"></canvas>
<p class="mut">Minimal effort kept 100% accuracy in every language here and cut the bill by <span id="s-save"></span>.</p></div>
</div>
<div class="card" style="margin-top:14px"><h3>Reasoning tokens by task type</h3><canvas id="c3"></canvas>
<p class="mut">Logic puzzles in Tanglish make Beam think ~4× longer than the same puzzle in English, the biggest gap in the set.</p></div>
<h2>Token Saver: paste Tamil, see what Beam will charge</h2>
<div class="card">
<p class="mut" style="margin-top:0">Runs Beam’s real tokenizer in your browser, converts your Tamil to Tanglish with the same transliterator used in the Belebele run, and estimates what other models would charge for the same text.</p>
<textarea id="tin">வீட்டு வாடகை ஒப்பந்தத்தில் முன்பணம் எப்போது திரும்பக் கிடைக்கும்? எளிய தமிழில் விளக்கவும்.</textarea>
<div style="margin:10px 0;display:flex;gap:8px;flex-wrap:wrap">
<button data-s="யாதும் ஊரே யாவரும் கேளிர்">Tamil</button>
<button data-s="Yaadhum oore yaavarum kelir">Tanglish</button>
<button data-s="Every town is my town, everyone is my kin">English</button>
<button data-s="Chennai la oru 2BHK rent evlo?">Tanglish chat</button>
</div>
<div id="tout" class="mut">Loading tokenizer…</div>
<div class="grid g2" style="margin-top:12px"><div class="card" style="background:#0b0d12"><h3>Auto-Tanglish version</h3><div id="ttl" class="mut" style="white-space:pre-wrap;max-height:160px;overflow:auto"></div><div id="ttlc" style="margin-top:8px"></div><button id="cpy" style="margin-top:8px">Copy Tanglish</button></div>
<div class="card" style="background:#0b0d12"><h3>Same text on other models (estimated)</h3><div id="tleague" class="mut"></div></div></div>
<div class="toks" id="tlist"></div>
</div>
<h2>Native Tamil: questions with no English twin</h2>
<div class="grid g4">
<div class="card"><div class="big" id="n-acc"></div><div class="mut">accuracy on 22 native items × 3 samples: Tamil number words, ancient numerals (௲ ௱ ௰), letter counting by Tamil grammar, Sangam literature, grammar, the Tamil calendar</div></div>
<div class="card"><div class="big" id="n-rsn"></div><div class="mut">avg reasoning tokens on native items, about 2.6× the parallel Tamil set: counting எழுத்துகள் through byte-level tokens takes real work</div></div>
<div class="card"><h3>The one miss</h3><div class="mut">Asked for the last word of Kural 1 (அகர முதல… <b>உலகு</b>), one of three samples answered “என்று” (“that”), echoing the prompt’s phrasing instead of the couplet.</div></div>
</div>
<h2>What this means if you build for Tamil users</h2>
<div class="card"><ul class="f">
<li><b>v2: Beam is measurably weaker reading Tamil than English</b> on Belebele, and the Tamil prompt costs about 4× more. Keep instructions and options in English when your content is Tamil: the mixed mode gets most of the accuracy back.</li>
<li><b>v2: keep the Tamil passage, ask in English.</b> The Mixed mode scored 95.6% (vs 88.3% all-Tamil) at two-thirds of the all-Tamil cost. It is the best setup for Tamil RAG on Beam.</li>
<li><b>v2: don't auto-transliterate to save money.</b> Auto-Tanglish cut input tokens 56%, but accuracy dropped to 80.3%, 32 of 900 answers ran out of tokens, and total cost rose 1.5× over Tamil script. Cheap to read, expensive to think.</li>
<li><b>v2: pick the model by tokenizer for Tamil-heavy RAG.</b> On the same text, Gemma 3 uses about a quarter of Beam's tokens.</li>
<li><b>Budget ~1.7× English</b> per Tamil or Tanglish query on Beam. Script choice barely changes the price; it moves cost between input and reasoning.</li>
<li><b>Turn reasoning down.</b> <code>reasoning_effort="minimal"</code> lost nothing on everyday Tamil tasks and saved ~16–21%.</li>
<li><b>Long Tamil documents hurt most.</b> At ~0.9 characters per token, Tamil RAG context costs ~3.7× English: summarise or translate retrieved chunks before stuffing a 1M context.</li>
<li><b>Tanglish is the cheaper input</b> for chat UIs (2.5 chars/token) if you can tolerate the extra thinking; for short-answer apps Tamil script is slightly cheaper overall.</li>
<li><b>Beam is ready for Tamil.</b> 100% on 300 parallel runs and 98.5% on native tasks means accuracy isn't the problem. Cost is.</li>
</ul></div>
<h2>Every answer, with Beam's reasoning</h2>
<div class="card">
<div style="display:flex;gap:8px;flex-wrap:wrap;margin-bottom:10px">
<select id="fl"><option value="">All languages</option><option value="en">English</option><option value="ta">Tamil</option><option value="tl">Tanglish</option></select>
<select id="fs"><option value="">All splits</option><option value="parallel|default">Parallel · default</option><option value="parallel|minimal">Parallel · minimal</option><option value="native|default">Native Tamil</option></select>
<select id="fc"><option value="">All results</option><option value="0">Wrong only</option></select>
</div>
<div style="max-height:520px;overflow:auto"><table><thead><tr><th>ID</th><th>Lang</th><th>Question</th><th>Gold</th><th>Beam</th><th>In</th><th>Think</th></tr></thead><tbody id="tb"></tbody></table></div>
<p class="mut">Tap a row to read Beam's reasoning trace (first 700 characters).</p>
</div>
<h2>Method</h2>
<div class="card mut">
<p>50 parallel items (20 math word problems with Indian context such as auto fares, rent splits and cents of land, 8 logic, 15 Tamil culture and knowledge, 7 everyday), each written in English, Tamil script and colloquial Tanglish, plus 22 native-Tamil items. Every prompt ends with “answer only” in its own language. Model: <code>reflection/Beam-1.0-501B-A23B</code> via Hugging Face Inference Providers (Baseten), max_tokens 6000. Parallel set: 2 samples per language at default effort and 1 at minimal effort; native set: 3 samples. Scoring: last number for numeric answers, alias match for words (Tamil or Latin script, Tamil digits normalised). Token counts come from the API's usage field; tokenizer stats come from Beam's own <code>tokenizer.json</code>. Cost uses Baseten list price ($0.40/1M input, $1.20/1M output incl. reasoning) as of Oct 2026. Total spend for the whole study: <b id="s-total"></b>.</p>
<p><b>v2 method:</b> Belebele (facebook/belebele, tam_Taml + eng_Latn, 900 aligned questions) × 4 input modes, default effort, one sample, scored on the option number. Tokenizer league: 488 unique FLORES passages through each model's tokenizer.json (Beam, Qwen3-8B, Gemma-3-27B, Llama-3.3-70B, DeepSeek-V3, gpt-oss-120b, Sarvam-M, Mistral-Small-3.1). Hard split: 56 items with gold computed by code (letter counts by Tamil grammar, grapheme reversal, uyirmei split, traditional numerals, Thirukkural endings, calendar), 3 samples. Writing: 15 prompts, judged 1–5 by two non-Beam models; register measured by script share.</p><p><b>Limits:</b> one sample per Belebele question; judges are LLMs, not native-speaker raters; the transliterator is a simple rule-based scheme, not how every Tamil speaker types Tanglish.</p>
<p>Dataset, prompts, every raw response: <a href="https://huggingface.co/datasets/esscor/TamilBench" target="_blank">huggingface.co/datasets/esscor/TamilBench</a></p>
</div>
<div class="foot">Built by Mohamed Adib (SVCE Chennai) for the Hugging Face × Reflection “Build with Beam” challenge. Apache-2.0 model; data CC-BY-4.0.</div>
</div>
<script>
const {S,Q,R}=window.TB, L={en:'English',ta:'Tamil',tl:'Tanglish'}, C={en:'#6aa6ff',ta:'#ff8a3d',tl:'#3ddc97'};
const d=S.default,m=S.minimal,f=x=>x.toFixed(2);
calls.textContent=`${S.total_calls} API calls in total.`;
document.getElementById('s-acc').textContent='100%';
document.getElementById('s-tok').textContent=(S.tokenizer.ta.tokens/S.tokenizer.en.tokens).toFixed(1)+'×';
document.getElementById('s-tpw').textContent=S.tokenizer.ta.tok_per_word;
document.getElementById('s-rsn').textContent=(d.tl.reasoning/d.en.reasoning).toFixed(1)+'×';
document.getElementById('s-cost').textContent=(d.ta.cost_per_1k/d.en.cost_per_1k).toFixed(1)+'× / '+(d.tl.cost_per_1k/d.en.cost_per_1k).toFixed(1)+'×';
const sv=['en','ta','tl'].map(l=>1-m[l].cost_per_1k/d[l].cost_per_1k);
document.getElementById('s-save').textContent=Math.round(Math.min(...sv)*100)+'–'+Math.round(Math.max(...sv)*100)+'%';
document.getElementById('n-acc').textContent=S.native.acc+'%';
document.getElementById('n-rsn').textContent=Math.round(S.native.reasoning);
document.getElementById('s-total').textContent='$'+S.total_cost.toFixed(2);
Chart.defaults.color='#9aa3b2';Chart.defaults.borderColor='#262c38';Chart.defaults.font.family='Inter';
const langs=['en','ta','tl'];
new Chart(c1,{type:'bar',data:{labels:langs.map(l=>L[l]),datasets:[
{label:'Prompt tokens',data:langs.map(l=>d[l].prompt),backgroundColor:'#ffd166'},
{label:'Reasoning tokens',data:langs.map(l=>d[l].reasoning),backgroundColor:'#c77dff'},
{label:'Answer tokens',data:langs.map(l=>d[l].completion-d[l].reasoning),backgroundColor:'#3ddc97'}]},
options:{plugins:{legend:{position:'bottom'}},scales:{x:{stacked:true},y:{stacked:true}}}});
new Chart(c2,{type:'bar',data:{labels:langs.map(l=>L[l]),datasets:[
{label:'Default effort',data:langs.map(l=>d[l].cost_per_1k),backgroundColor:langs.map(l=>C[l])},
{label:'Minimal effort',data:langs.map(l=>m[l].cost_per_1k),backgroundColor:langs.map(l=>C[l]+'77')}]},
options:{plugins:{legend:{position:'bottom'},tooltip:{callbacks:{label:c=>' $'+c.raw.toFixed(3)+' per 1k'}}},scales:{y:{ticks:{callback:v=>'$'+v}}}}});
const cats=['math','logic','knowledge','everyday'];
new Chart(c3,{type:'bar',data:{labels:cats,datasets:langs.map(l=>({label:L[l],data:cats.map(c=>S.bycat[c][l].reasoning),backgroundColor:C[l]}))},options:{plugins:{legend:{position:'bottom'}}}});
// table
function render(){const fl=document.getElementById('fl').value,fs=document.getElementById('fs').value,fc=document.getElementById('fc').value;let h='';
R.forEach((r,i)=>{if(fl&&r.lang!==fl)return;if(fs&&(r.split+'|'+r.effort)!==fs)return;if(fc==='0'&&r.ok)return;const q=Q[r.id];
h+=`<tr class="row" data-i="${i}"><td>${r.id}${r.split==='parallel'&&r.effort==='minimal'?' <span class="mut">min</span>':''}</td><td><span class="pill ${r.lang}">${L[r.lang]}</span></td><td>${(q[r.lang]||'').slice(0,90)}</td><td>${q.answer}</td><td class="${r.ok?'ok':'bad'}">${r.out.replace(/</g,'&lt;')}</td><td>${r.pt}</td><td>${r.rt}</td></tr>`});
tb.innerHTML=h}
['fl','fs','fc'].forEach(x=>document.getElementById(x).onchange=render);render();
tb.onclick=e=>{const tr=e.target.closest('tr.row');if(!tr)return;const nx=tr.nextElementSibling;if(nx&&nx.classList.contains('tr'))return nx.remove();
const r=R[tr.dataset.i];const t=document.createElement('tr');t.className='tr';t.innerHTML=`<td colspan="7"><div class="trace">${(r.why||'(no trace)').replace(/</g,'&lt;')}</div></td>`;tr.after(t)};
</script>
<script>
(()=>{const S2=window.TB2,B=S2.belebele,lg=S2.tokenizer_league.rows,M={en:'English',ta:'Tamil',mix:'Mixed',translit:'Auto-Tanglish'},MC={en:'#6aa6ff',ta:'#ff8a3d',mix:'#ffd166',translit:'#3ddc97'};
const modes=['en','ta','mix','translit'].filter(m=>B[m]);const $=id=>document.getElementById(id);
$('v2calls').textContent=S2.calls+' more Beam calls.';
$('v-gap').textContent='−'+(B.en.acc-B.ta.acc).toFixed(1)+' pts';$('v-accs').textContent=B.en.acc+'% vs '+B.ta.acc+'%';
const be=lg.find(r=>r.model==='Beam'),ge=lg.find(r=>r.model==='Gemma3');$('v-ratio').textContent=be.ratio+'×';$('v-gem').textContent=ge.ratio+'×';
if(B.translit){$('v-save').textContent=Math.round(100*(1-B.translit.prompt/B.ta.prompt))+'%';$('v-tracc').textContent=B.translit.acc+'%'}
$('v-cost').textContent=(B.ta.cost_per_1k/B.en.cost_per_1k).toFixed(1)+'×';$('v-rsn').textContent=(B.ta.reasoning/B.en.reasoning).toFixed(1)+'×';
new Chart(b1,{type:'bar',data:{labels:modes.map(m=>M[m]),datasets:[{label:'Accuracy %',data:modes.map(m=>B[m].acc),backgroundColor:modes.map(m=>MC[m])}]},options:{plugins:{legend:{display:false},tooltip:{callbacks:{label:c=>' '+c.raw+'% of '+B[modes[c.dataIndex]].n}}},scales:{y:{min:70,max:100}}}});
new Chart(b2,{type:'bar',data:{labels:modes.map(m=>M[m]),datasets:[{label:'Prompt',data:modes.map(m=>+(B[m].prompt*0.4/1000).toFixed(3)),backgroundColor:'#ffd166'},{label:'Reasoning + answer',data:modes.map(m=>+(B[m].completion*1.2/1000).toFixed(3)),backgroundColor:'#c77dff'}]},options:{plugins:{legend:{position:'bottom'}},scales:{x:{stacked:true},y:{stacked:true,ticks:{callback:v=>'$'+v}}}}});
$('b2note').textContent='Average tokens per question: '+modes.map(m=>M[m]+' '+Math.round(B[m].prompt)+' in / '+Math.round(B[m].reasoning)+' thinking').join(' · ')+'.';
const L2=[...lg].sort((a,b)=>a.ratio-b.ratio);
new Chart(lg_,{type:'bar',data:{labels:L2.map(r=>r.model),datasets:[{label:'Tamil ÷ English tokens',data:L2.map(r=>r.ratio),backgroundColor:L2.map(r=>r.model==='Beam'?'#ff8a3d':'#6aa6ff')}]},options:{indexAxis:'y',plugins:{legend:{display:false},tooltip:{callbacks:{label:c=>' '+c.raw+'× · '+L2[c.dataIndex].tok_per_ta_word+' tokens per Tamil word'}}}}});
const H=S2.hard,hc=Object.keys(H);new Chart(hd,{type:'bar',data:{labels:hc.map(c=>c.replace(/_/g,' ')),datasets:[{label:'Accuracy %',data:hc.map(c=>H[c].acc),backgroundColor:hc.map(c=>H[c].acc<100?'#ff5d73':'#3ddc97')}]},options:{indexAxis:'y',plugins:{legend:{display:false}},scales:{x:{min:0,max:100}}}});
$('h-acc').textContent=(100*S2.hard_total.correct/S2.hard_total.n).toFixed(1)+'%';$('h-n').textContent=S2.hard_total.n/3;
const W=S2.writing;$('w-flu').textContent=W.fluency.toFixed(1);$('w-cor').textContent=W.correctness.toFixed(1);$('w-reg').textContent=W.tanglish_register_kept+'/'+W.tanglish_n;$('w-out').textContent=W.tamil_output_vs_tanglish_tokens+'×';
})();
</script>
<script type="module">
import {AutoTokenizer, env} from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.5.1';
env.allowRemoteModels=false; env.allowLocalModels=true; env.localModelPath=new URL('./models/',location.href).href;
const V={'அ':'a','ஆ':'aa','இ':'i','ஈ':'ee','உ':'u','ஊ':'oo','எ':'e','ஏ':'ae','ஐ':'ai','ஒ':'o','ஓ':'oa','ஔ':'au','ஃ':'h'},CC={'க':'k','ங':'ng','ச':'s','ஞ':'nj','ட':'t','ண':'n','த':'th','ந':'n','ப':'p','ம':'m','ய':'y','ர':'r','ல':'l','வ':'v','ழ':'zh','ள':'l','ற':'r','ன':'n','ஜ':'j','ஷ':'sh','ஸ':'s','ஹ':'h','ஶ':'sh'},SG={'ா':'aa','ி':'i','ீ':'ee','ு':'u','ூ':'oo','ெ':'e','ே':'ae','ை':'ai','ொ':'o','ோ':'oa','ௌ':'au','்':''};
function tanglish(s){let o='';const a=[...s];for(let i=0;i<a.length;i++){const ch=a[i];if(CC[ch]){const nx=a[i+1];if(nx in SG){o+=CC[ch]+SG[nx];i++;continue}o+=CC[ch]+'a'}else if(V[ch])o+=V[ch];else if(ch>='௦'&&ch<='௯')o+=String(ch.charCodeAt(0)-0x0BE6);else if(ch in SG)o+=SG[ch];else o+=ch}return o}
const out=document.getElementById('tout'),list=document.getElementById('tlist'),tin=document.getElementById('tin');
let tok;
try{tok=await AutoTokenizer.from_pretrained('beam');}catch(e){out.textContent='Could not load the tokenizer in this browser ('+e.message+').';}
function go(){if(!tok)return;const s=tin.value;const ids=tok.encode(s,{add_special_tokens:false});const words=s.trim().split(/\s+/).filter(Boolean).length||1;
const tam=[...s].filter(c=>c>='\u0b80'&&c<='\u0bff').length,tl=tanglish(s),tids=tok.encode(tl,{add_special_tokens:false});
const usd=n=>'$'+(n*0.4/1e6*1e6/1000).toFixed(2);
out.innerHTML=`<b style="color:#ffd166;font-size:22px">${ids.length}</b> Beam tokens · ${[...s].length} characters · ${(ids.length/words).toFixed(1)} tokens per word · ${usd(ids.length)} per 1,000 such prompts (input only)`;
document.getElementById('ttl').textContent=tam?tl:'(no Tamil script to convert)';
document.getElementById('ttlc').innerHTML=tam?`<b style="color:#3ddc97;font-size:20px">${tids.length}</b> Beam tokens · <b>${Math.round(100*(1-tids.length/ids.length))}% fewer</b>`:'';
window._tl=tl;
const lgr=window.TB2.tokenizer_league.rows,be=lgr.find(r=>r.model==='Beam');const frac=tam/Math.max(1,[...s].filter(c=>/\S/.test(c)).length);
document.getElementById('tleague').innerHTML=frac<0.3?'Paste Tamil script to compare models.':[...lgr].sort((a,b)=>a.ratio-b.ratio).map(r=>`<div style="display:flex;justify-content:space-between"><span>${r.model}</span><b style="color:${r.model==='Beam'?'#ff8a3d':'#e8ebf1'}">≈${Math.round(ids.length*r.flores_ta/be.flores_ta)}</b></div>`).join('')+'<div style="margin-top:6px;font-size:12px">Scaled by each tokenizer\'s Tamil token count on FLORES relative to Beam\'s.</div>';
list.innerHTML=ids.slice(0,200).map(i=>`<span>${tok.decode([i]).replace(/</g,'&lt;')||'·'}</span>`).join('')}
tin.oninput=go;document.getElementById('cpy').onclick=()=>navigator.clipboard.writeText(window._tl||'');document.querySelectorAll('[data-s]').forEach(b=>b.onclick=()=>{tin.value=b.dataset.s;go()});go();
</script>
</body></html>