TamilBench v1

README.md

@@ -1,10 +1,23 @@
 ---
-title: TamilBench
-emoji: 📊
-colorFrom: green
-colorTo: purple
+title: TamilBench - Beam's Tamil Token Tax
+emoji: 🪔
+colorFrom: yellow
+colorTo: red
 sdk: static
 pinned: false
+license: cc-by-4.0
+short_description: What does Beam charge to think in Tamil and Tanglish?
+models:
+  - reflection/Beam-1.0-501B-A23B
+datasets:
+  - esscor/TamilBench
+tags:
+  - beam
+  - build-with-beam
+  - tamil
+  - multilingual
+  - tokenizer
 ---
+TamilBench: the same 50 questions in English, Tamil script and Tanglish, plus 22 native-Tamil questions, run on **Beam-1.0-501B-A23B** (via HF Inference Providers / Baseten). Measures accuracy, prompt tokens, reasoning tokens and cost, with an in-browser Beam tokenizer playground and every raw answer with its reasoning trace.
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Data: https://huggingface.co/datasets/esscor/TamilBench

index.html

@@ -1,19 +1,153 @@
 <!doctype html>
-<html>
-	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
-		<title>My static Space</title>
-		<link rel="stylesheet" href="style.css" />
-	</head>
-	<body>
-		<div class="card">
-			<h1>Welcome to your static Space!</h1>
-			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
-			<p>
-				Also don't forget to check the
-				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
-			</p>
-		</div>
-	</body>
-</html>
+<html lang="en">
+<head>
+<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
+<title>TamilBench · What does Beam charge to think in Tamil?</title>
+<link rel="preconnect" href="https://fonts.googleapis.com">
+<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;800&family=Noto+Sans+Tamil:wght@400;600&display=swap" rel="stylesheet">
+<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js"></script>
+<script src="data.js"></script>
+<style>
+:root{--bg:#0d0f14;--card:#161a22;--line:#262c38;--txt:#e8ebf1;--mut:#9aa3b2;--en:#6aa6ff;--ta:#ff8a3d;--tl:#3ddc97;--acc:#ffd166}
+*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--txt);font-family:Inter,'Noto Sans Tamil',system-ui,sans-serif;line-height:1.55}
+.wrap{max-width:1080px;margin:0 auto;padding:28px 20px 60px}
+h1{font-size:clamp(28px,4.5vw,46px);line-height:1.1;margin:8px 0 10px;font-weight:800}
+h1 .ta{color:var(--ta)}h2{font-size:24px;margin:44px 0 12px}h3{margin:0 0 6px;font-size:16px}
+.kicker{color:var(--acc);font-weight:600;letter-spacing:.06em;text-transform:uppercase;font-size:13px}
+.lead{color:var(--mut);font-size:18px;max-width:780px}
+.grid{display:grid;gap:14px}.g4{grid-template-columns:repeat(auto-fit,minmax(220px,1fr))}.g2{grid-template-columns:repeat(auto-fit,minmax(420px,1fr))}
+.card{background:var(--card);border:1px solid var(--line);border-radius:14px;padding:18px}
+.big{font-size:38px;font-weight:800;line-height:1}.mut{color:var(--mut);font-size:14px}
+.pill{display:inline-block;padding:2px 9px;border-radius:99px;font-size:12px;font-weight:600;margin-right:6px}
+.en{background:#6aa6ff22;color:var(--en)}.ta{background:#ff8a3d22;color:var(--ta)}.tl{background:#3ddc9722;color:var(--tl)}
+textarea{width:100%;min-height:90px;background:#0b0d12;color:var(--txt);border:1px solid var(--line);border-radius:10px;padding:12px;font:inherit;font-size:16px}
+button,select{background:#222837;color:var(--txt);border:1px solid var(--line);border-radius:9px;padding:8px 12px;font:inherit;cursor:pointer}
+button:hover{border-color:var(--acc)}
+.toks{display:flex;flex-wrap:wrap;gap:3px;margin-top:10px;font-family:ui-monospace,monospace;font-size:13px}
+.toks span{padding:2px 5px;border-radius:5px;background:#ffffff10}
+table{width:100%;border-collapse:collapse;font-size:14px}th,td{text-align:left;padding:7px 8px;border-bottom:1px solid var(--line);vertical-align:top}
+th{color:var(--mut);font-weight:600}tr.row{cursor:pointer}tr.row:hover{background:#ffffff08}
+.ok{color:var(--tl)}.bad{color:#ff5d73}
+.trace{white-space:pre-wrap;font-size:13px;color:var(--mut);background:#0b0d12;border-radius:8px;padding:10px;margin-top:6px}
+ul.f li{margin:8px 0}a{color:var(--en)}
+.foot{margin-top:50px;color:var(--mut);font-size:13px;border-top:1px solid var(--line);padding-top:18px}
+canvas{max-height:320px}
+</style>
+</head>
+<body><div class="wrap">
+<div class="kicker">Build with Beam · TamilBench v1</div>
+<h1>Beam understands <span class="ta">தமிழ்</span> perfectly.<br>It just charges more to think in it.</h1>
+<p class="lead">80 million people speak Tamil, and most of them type it two ways: in Tamil script and in “Tanglish” (Tamil in Latin letters). I asked <b>Beam-1.0-501B-A23B</b> the same 50 questions in English, Tamil and Tanglish, plus 22 questions that only exist in Tamil, and measured accuracy, prompt tokens, reasoning tokens and dollars. <span id="calls"></span></p>
+
+<div class="grid g4" style="margin-top:22px">
+ <div class="card"><div class="big" id="s-acc"></div><div class="mut">accuracy on the parallel set, in all three languages, at default <i>and</i> minimal reasoning effort</div></div>
+ <div class="card"><div class="big" style="color:var(--ta)" id="s-tok"></div><div class="mut">more input tokens for the same question in Tamil script (tokenizer: <span id="s-tpw"></span> tokens per Tamil word)</div></div>
+ <div class="card"><div class="big" style="color:var(--tl)" id="s-rsn"></div><div class="mut">more reasoning tokens when the question is in Tanglish; Beam quietly translates before it thinks</div></div>
+ <div class="card"><div class="big" style="color:var(--acc)" id="s-cost"></div><div class="mut">cost per query vs English (Tamil / Tanglish) at Baseten’s $0.40 in / $1.20 out per 1M</div></div>
+</div>
+
+<h2>The Tamil token tax, measured</h2>
+<div class="grid g2">
+ <div class="card"><h3>Where the tokens go (avg per question, default effort)</h3><canvas id="c1"></canvas>
+  <p class="mut">Tamil script is expensive to <i>read</i>; Tanglish is cheap to read but expensive to <i>think</i> about. They land at almost the same price.</p></div>
+ <div class="card"><h3>Cost per 1,000 questions: default vs <code>reasoning_effort="minimal"</code></h3><canvas id="c2"></canvas>
+  <p class="mut">Minimal effort kept 100% accuracy in every language here and cut the bill by <span id="s-save"></span>.</p></div>
+</div>
+<div class="card" style="margin-top:14px"><h3>Reasoning tokens by task type</h3><canvas id="c3"></canvas>
+ <p class="mut">Logic puzzles in Tanglish make Beam think ~4× longer than the same puzzle in English, the biggest gap in the set.</p></div>
+
+<h2>Try it: how many Beam tokens is your sentence?</h2>
+<div class="card">
+ <p class="mut" style="margin-top:0">Runs Beam’s real tokenizer in your browser (loaded from the model repo). Paste anything in Tamil, Tanglish or English.</p>
+ <textarea id="tin">யாதும் ஊரே யாவரும் கேளிர்</textarea>
+ <div style="margin:10px 0;display:flex;gap:8px;flex-wrap:wrap">
+  <button data-s="யாதும் ஊரே யாவரும் கேளிர்">Tamil</button>
+  <button data-s="Yaadhum oore yaavarum kelir">Tanglish</button>
+  <button data-s="Every town is my town, everyone is my kin">English</button>
+  <button data-s="Chennai la oru 2BHK rent evlo?">Tanglish chat</button>
+ </div>
+ <div id="tout" class="mut">Loading tokenizer…</div><div class="toks" id="tlist"></div>
+</div>
+
+<h2>Native Tamil: questions with no English twin</h2>
+<div class="grid g4">
+ <div class="card"><div class="big" id="n-acc"></div><div class="mut">accuracy on 22 native items × 3 samples: Tamil number words, ancient numerals (௲ ௱ ௰), letter counting by Tamil grammar, Sangam literature, grammar, the Tamil calendar</div></div>
+ <div class="card"><div class="big" id="n-rsn"></div><div class="mut">avg reasoning tokens on native items, about 2.6× the parallel Tamil set: counting எழுத்துகள் through byte-level tokens takes real work</div></div>
+ <div class="card"><h3>The one miss</h3><div class="mut">Asked for the last word of Kural 1 (அகர முதல… <b>உலகு</b>), one of three samples answered “என்று” (“that”), echoing the prompt’s phrasing instead of the couplet.</div></div>
+</div>
+
+<h2>What this means if you build for Tamil users</h2>
+<div class="card"><ul class="f">
+ <li><b>Budget ~1.7× English</b> per Tamil or Tanglish query on Beam. Script choice barely changes the price; it moves cost between input and reasoning.</li>
+ <li><b>Turn reasoning down.</b> <code>reasoning_effort="minimal"</code> lost nothing on everyday Tamil tasks and saved ~16–21%.</li>
+ <li><b>Long Tamil documents hurt most.</b> At ~0.9 characters per token, Tamil RAG context costs ~3.7× English: summarise or translate retrieved chunks before stuffing a 1M context.</li>
+ <li><b>Tanglish is the cheaper input</b> for chat UIs (2.5 chars/token) if you can tolerate the extra thinking; for short-answer apps Tamil script is slightly cheaper overall.</li>
+ <li><b>Beam is ready for Tamil.</b> 100% on 300 parallel runs and 98.5% on native tasks means accuracy isn't the problem. Cost is.</li>
+</ul></div>
+
+<h2>Every answer, with Beam's reasoning</h2>
+<div class="card">
+ <div style="display:flex;gap:8px;flex-wrap:wrap;margin-bottom:10px">
+  <select id="fl"><option value="">All languages</option><option value="en">English</option><option value="ta">Tamil</option><option value="tl">Tanglish</option></select>
+  <select id="fs"><option value="">All splits</option><option value="parallel|default">Parallel · default</option><option value="parallel|minimal">Parallel · minimal</option><option value="native|default">Native Tamil</option></select>
+  <select id="fc"><option value="">All results</option><option value="0">Wrong only</option></select>
+ </div>
+ <div style="max-height:520px;overflow:auto"><table><thead><tr><th>ID</th><th>Lang</th><th>Question</th><th>Gold</th><th>Beam</th><th>In</th><th>Think</th></tr></thead><tbody id="tb"></tbody></table></div>
+ <p class="mut">Tap a row to read Beam's reasoning trace (first 700 characters).</p>
+</div>
+
+<h2>Method</h2>
+<div class="card mut">
+ <p>50 parallel items (20 math word problems with Indian context such as auto fares, rent splits and cents of land, 8 logic, 15 Tamil culture and knowledge, 7 everyday), each written in English, Tamil script and colloquial Tanglish, plus 22 native-Tamil items. Every prompt ends with “answer only” in its own language. Model: <code>reflection/Beam-1.0-501B-A23B</code> via Hugging Face Inference Providers (Baseten), max_tokens 6000. Parallel set: 2 samples per language at default effort and 1 at minimal effort; native set: 3 samples. Scoring: last number for numeric answers, alias match for words (Tamil or Latin script, Tamil digits normalised). Token counts come from the API's usage field; tokenizer stats come from Beam's own <code>tokenizer.json</code>. Cost uses Baseten list price ($0.40/1M input, $1.20/1M output incl. reasoning) as of Oct 2026. Total spend for the whole study: <b id="s-total"></b>.</p>
+ <p><b>Limits:</b> 72 items is small, and the parallel set is saturated (Beam aced it), so treat accuracy as a sanity floor; the token and cost findings are the point. Next: longer documents, Tamil-script answers, and a harder native split.</p>
+ <p>Dataset, prompts, every raw response: <a href="https://huggingface.co/datasets/esscor/TamilBench" target="_blank">huggingface.co/datasets/esscor/TamilBench</a></p>
+</div>
+<div class="foot">Built by Mohamed Adib (SVCE Chennai) for the Hugging Face × Reflection “Build with Beam” challenge. Apache-2.0 model; data CC-BY-4.0.</div>
+</div>
+<script>
+const {S,Q,R}=window.TB, L={en:'English',ta:'Tamil',tl:'Tanglish'}, C={en:'#6aa6ff',ta:'#ff8a3d',tl:'#3ddc97'};
+const d=S.default,m=S.minimal,f=x=>x.toFixed(2);
+calls.textContent=`${S.total_calls} API calls in total.`;
+document.getElementById('s-acc').textContent='100%';
+document.getElementById('s-tok').textContent=(S.tokenizer.ta.tokens/S.tokenizer.en.tokens).toFixed(1)+'×';
+document.getElementById('s-tpw').textContent=S.tokenizer.ta.tok_per_word;
+document.getElementById('s-rsn').textContent=(d.tl.reasoning/d.en.reasoning).toFixed(1)+'×';
+document.getElementById('s-cost').textContent=(d.ta.cost_per_1k/d.en.cost_per_1k).toFixed(1)+'× / '+(d.tl.cost_per_1k/d.en.cost_per_1k).toFixed(1)+'×';
+const sv=['en','ta','tl'].map(l=>1-m[l].cost_per_1k/d[l].cost_per_1k);
+document.getElementById('s-save').textContent=Math.round(Math.min(...sv)*100)+'–'+Math.round(Math.max(...sv)*100)+'%';
+document.getElementById('n-acc').textContent=S.native.acc+'%';
+document.getElementById('n-rsn').textContent=Math.round(S.native.reasoning);
+document.getElementById('s-total').textContent='$'+S.total_cost.toFixed(2);
+Chart.defaults.color='#9aa3b2';Chart.defaults.borderColor='#262c38';Chart.defaults.font.family='Inter';
+const langs=['en','ta','tl'];
+new Chart(c1,{type:'bar',data:{labels:langs.map(l=>L[l]),datasets:[
+ {label:'Prompt tokens',data:langs.map(l=>d[l].prompt),backgroundColor:'#ffd166'},
+ {label:'Reasoning tokens',data:langs.map(l=>d[l].reasoning),backgroundColor:'#c77dff'},
+ {label:'Answer tokens',data:langs.map(l=>d[l].completion-d[l].reasoning),backgroundColor:'#3ddc97'}]},
+ options:{plugins:{legend:{position:'bottom'}},scales:{x:{stacked:true},y:{stacked:true}}}});
+new Chart(c2,{type:'bar',data:{labels:langs.map(l=>L[l]),datasets:[
+ {label:'Default effort',data:langs.map(l=>d[l].cost_per_1k),backgroundColor:langs.map(l=>C[l])},
+ {label:'Minimal effort',data:langs.map(l=>m[l].cost_per_1k),backgroundColor:langs.map(l=>C[l]+'77')}]},
+ options:{plugins:{legend:{position:'bottom'},tooltip:{callbacks:{label:c=>' $'+c.raw.toFixed(3)+' per 1k'}}},scales:{y:{ticks:{callback:v=>'$'+v}}}}});
+const cats=['math','logic','knowledge','everyday'];
+new Chart(c3,{type:'bar',data:{labels:cats,datasets:langs.map(l=>({label:L[l],data:cats.map(c=>S.bycat[c][l].reasoning),backgroundColor:C[l]}))},options:{plugins:{legend:{position:'bottom'}}}});
+// table
+function render(){const fl=document.getElementById('fl').value,fs=document.getElementById('fs').value,fc=document.getElementById('fc').value;let h='';
+ R.forEach((r,i)=>{if(fl&&r.lang!==fl)return;if(fs&&(r.split+'|'+r.effort)!==fs)return;if(fc==='0'&&r.ok)return;const q=Q[r.id];
+  h+=`<tr class="row" data-i="${i}"><td>${r.id}${r.split==='parallel'&&r.effort==='minimal'?' <span class="mut">min</span>':''}</td><td><span class="pill ${r.lang}">${L[r.lang]}</span></td><td>${(q[r.lang]||'').slice(0,90)}</td><td>${q.answer}</td><td class="${r.ok?'ok':'bad'}">${r.out.replace(/</g,'&lt;')}</td><td>${r.pt}</td><td>${r.rt}</td></tr>`});
+ tb.innerHTML=h}
+['fl','fs','fc'].forEach(x=>document.getElementById(x).onchange=render);render();
+tb.onclick=e=>{const tr=e.target.closest('tr.row');if(!tr)return;const nx=tr.nextElementSibling;if(nx&&nx.classList.contains('tr'))return nx.remove();
+ const r=R[tr.dataset.i];const t=document.createElement('tr');t.className='tr';t.innerHTML=`<td colspan="7"><div class="trace">${(r.why||'(no trace)').replace(/</g,'&lt;')}</div></td>`;tr.after(t)};
+</script>
+<script type="module">
+import {AutoTokenizer} from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.5.1';
+const out=document.getElementById('tout'),list=document.getElementById('tlist'),tin=document.getElementById('tin');
+let tok;
+try{tok=await AutoTokenizer.from_pretrained('reflection/Beam-1.0-501B-A23B');}catch(e){out.textContent='Could not load the tokenizer in this browser ('+e.message+').';}
+function go(){if(!tok)return;const s=tin.value;const ids=tok.encode(s,{add_special_tokens:false});const words=s.trim().split(/\s+/).filter(Boolean).length||1;
+ out.innerHTML=`<b style="color:#ffd166;font-size:22px">${ids.length}</b> Beam tokens · ${[...s].length} characters · ${(ids.length/words).toFixed(1)} tokens per word`;
+ list.innerHTML=ids.slice(0,200).map(i=>`<span>${tok.decode([i]).replace(/</g,'&lt;')||'·'}</span>`).join('')}
+tin.oninput=go;document.querySelectorAll('[data-s]').forEach(b=>b.onclick=()=>{tin.value=b.dataset.s;go()});go();
+</script>
+</body></html>