Spaces:
Running
Running
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"> | |
| <title>HinglishBench - code-mixed reasoning eval for open LLMs</title> | |
| <style> | |
| :root { --bg:#0b0f17; --card:#131a26; --line:#243044; --txt:#e7ecf3; --dim:#9aa7bd; --acc:#fbbf24; --en:#60a5fa; --hi:#f472b6; } | |
| * { box-sizing: border-box; } | |
| body { margin:0; background:var(--bg); color:var(--txt); font:15px/1.55 -apple-system, "Segoe UI", Inter, Roboto, sans-serif; } | |
| .wrap { max-width:1060px; margin:0 auto; padding:28px 18px 60px; } | |
| h1 { font-size:26px; margin:6px 0 2px; } | |
| h2 { font-size:18px; margin:30px 0 10px; } | |
| .sub { color:var(--dim); margin:0 0 18px; } | |
| .badge { display:inline-block; background:#1c2940; color:var(--acc); border:1px solid #2c3d5c; border-radius:999px; padding:2px 10px; font-size:12px; margin-right:6px; } | |
| .card { background:var(--card); border:1px solid var(--line); border-radius:12px; padding:16px 18px; margin:14px 0; } | |
| table { width:100%; border-collapse:collapse; font-size:14px; } | |
| th, td { text-align:right; padding:8px 10px; border-bottom:1px solid var(--line); } | |
| th:first-child, td:first-child { text-align:left; } | |
| th { color:var(--dim); font-weight:600; } | |
| .en { color:var(--en); } .hi { color:var(--hi); } | |
| .gap-neg { color:#f87171; } .gap-pos { color:#34d399; } | |
| .controls { margin:10px 0; } | |
| select, input { background:#0e1522; color:var(--txt); border:1px solid var(--line); border-radius:8px; padding:7px 10px; font-size:14px; } | |
| input { width:100%; margin-top:8px; } | |
| .item { border-top:1px solid var(--line); padding:14px 0; } | |
| .q { display:grid; grid-template-columns:1fr 1fr; gap:14px; } | |
| .q div { background:#0e1522; border:1px solid var(--line); border-radius:8px; padding:10px 12px; font-size:13.5px; white-space:pre-wrap; } | |
| .qlabel { font-size:11px; text-transform:uppercase; letter-spacing:.06em; color:var(--dim); margin-bottom:4px; } | |
| .ans { color:var(--acc); font-weight:600; } | |
| .mrow { display:flex; flex-wrap:wrap; gap:8px; margin-top:8px; font-size:12.5px; } | |
| .chip { border:1px solid var(--line); border-radius:8px; padding:5px 9px; background:#101827; cursor:pointer; } | |
| .ok { color:#34d399; } .bad { color:#f87171; } | |
| details { margin-top:6px; font-size:13px; } | |
| summary { cursor:pointer; color:var(--dim); } | |
| pre { background:#0e1522; border:1px solid var(--line); border-radius:8px; padding:10px; overflow:auto; font-size:12.5px; white-space:pre-wrap; } | |
| .foot { color:var(--dim); font-size:12.5px; margin-top:26px; } | |
| @media (max-width:760px) { .q { grid-template-columns:1fr; } } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="wrap"> | |
| <span class="badge">HinglishBench v0.1</span><span class="badge">GSM8K parallel set, 82 pairs</span> | |
| <h1>HinglishBench: does code-mixed Hinglish cost open models?</h1> | |
| <p class="sub">Parallel English / romanized-Hinglish reasoning evaluation of Beam-1.0-501B-A23B, Gemma-3-27B-it and Qwen3-32B. | |
| 82 GSM8K test items, each with a verified Hinglish translation. Zero-shot, single run, fair chain-of-thought prompt.</p> | |
| <div class="card"> | |
| <h2 style="margin-top:0">Leaderboard</h2> | |
| <table id="board"> | |
| <thead><tr> | |
| <th>Model</th><th class="en">EN acc</th><th class="hi">Hinglish acc</th><th>Gap (HI-EN)</th><th>Median token tax (HI/EN)</th> | |
| </tr></thead> | |
| <tbody></tbody> | |
| </table> | |
| <p class="foot" style="margin:10px 0 0">Token tax = per-item ratio of completion tokens (Hinglish / English), median over 82 pairs. Beam runs at reasoning_effort=low; medium run: observed tax 1.39x (single run).</p> | |
| </div> | |
| <div class="card"> | |
| <h2 style="margin-top:0">Headline findings</h2> | |
| <ul> | |
| <li><b>Beam scores highest on Hinglish of the three tested</b> (95.1%, one item ahead of Gemma's 93.9%) despite English-predominant training - but a Hinglish answer costs a median ~1.5x tokens.</li> | |
| <li><b>The tax is not universal.</b> Gemma-3-27B scores Hinglish slightly above English (93.9% vs 91.5%) at no token cost; Qwen3-32B loses 7.3 points of accuracy on Hinglish.</li> | |
| <li><b>The two prompt variants we tested don't remove the tax.</b> A "think in English" system prompt changed accuracy by 0 and tokens by +10% on 30 items. reasoning_effort=medium didn't remove it in our run either (observed 1.39x vs 1.47x; single runs, no variance estimate).</li> | |
| </ul> | |
| </div> | |
| <h2>Per-item explorer</h2> | |
| <div class="controls"> | |
| <select id="fmodel"><option value="">All models</option></select> | |
| <select id="fresult"><option value="">All results</option><option value="enwins">EN correct, Hinglish wrong</option><option value="hiwins">Hinglish correct, EN wrong</option><option value="bothwrong">Both wrong</option></select> | |
| <input id="q" placeholder="Search items (keyword)..."> | |
| </div> | |
| <div id="items"></div> | |
| <p class="foot">Methodology, dataset and code: see the linked dataset card and repo. Beam outputs via HF Inference Providers (Baseten). Translations generated by Beam itself, number-verified automatically; 18 of 100 sampled items excluded for failing verification. Single run per cell - observed differences are reported without claiming statistical significance.</p> | |
| </div> | |
| <script src="data.js"></script> | |
| <script> | |
| const board = DATA.leaderboard, items = DATA.items; | |
| const tbody = document.querySelector('#board tbody'); | |
| for (const r of board) { | |
| const gap = (r.acc_hi - r.acc_en).toFixed(1); | |
| const tr = document.createElement('tr'); | |
| tr.innerHTML = `<td>${r.model}</td><td class="en">${r.acc_en}%</td><td class="hi">${r.acc_hi}%</td> | |
| <td class="${gap < 0 ? 'gap-neg' : 'gap-pos'}">${gap > 0 ? '+' : ''}${gap} pts</td> | |
| <td>${r.median_tax}x</td>`; | |
| tbody.appendChild(tr); | |
| } | |
| const models = [...new Set(items.flatMap(i => Object.keys(i.models)))]; | |
| const fm = document.getElementById('fmodel'); | |
| for (const m of models) { const o = document.createElement('option'); o.value = m; o.textContent = m; fm.appendChild(o); } | |
| function render() { | |
| const q = document.getElementById('q').value.toLowerCase(); | |
| const mf = fm.value, rf = document.getElementById('fresult').value; | |
| const box = document.getElementById('items'); box.innerHTML = ''; | |
| let shown = 0; | |
| for (const it of items) { | |
| if (q && !(it.en.toLowerCase().includes(q) || it.hi.toLowerCase().includes(q))) continue; | |
| const mnames = mf ? [mf] : models; | |
| let keep = true; | |
| if (rf) { | |
| keep = mnames.some(m => { | |
| const e = it.models[m]?.en, h = it.models[m]?.hi; | |
| if (!e || !h) return false; | |
| if (rf === 'enwins') return e.correct && !h.correct; | |
| if (rf === 'hiwins') return h.correct && !e.correct; | |
| if (rf === 'bothwrong') return !e.correct && !h.correct; | |
| return true; | |
| }); | |
| } | |
| if (!keep || shown >= 40) continue; | |
| shown++; | |
| const div = document.createElement('div'); div.className = 'item'; | |
| let chips = ''; | |
| for (const m of mnames) { | |
| const e = it.models[m]?.en, h = it.models[m]?.hi; | |
| if (!e || !h) continue; | |
| chips += `<div class="chip"><b>${m}</b><br> | |
| <span class="${e.correct ? 'ok' : 'bad'}">EN: ${e.pred ?? '?'} (${e.tokens} tok)</span> · | |
| <span class="${h.correct ? 'ok' : 'bad'}">HI: ${h.pred ?? '?'} (${h.tokens} tok)</span> | |
| <details><summary>outputs</summary><pre>EN: ${(e.raw || '').replace(/</g,'<')}\n\nHI: ${(h.raw || '').replace(/</g,'<')}</pre></details></div>`; | |
| } | |
| div.innerHTML = `<div class="q"> | |
| <div><div class="qlabel en">English</div>${it.en.replace(/</g,'<')}</div> | |
| <div><div class="qlabel hi">Hinglish</div>${it.hi.replace(/</g,'<')}</div></div> | |
| <div style="margin-top:6px">Answer: <span class="ans">${it.answer}</span></div> | |
| <div class="mrow">${chips}</div>`; | |
| box.appendChild(div); | |
| } | |
| if (!shown) box.innerHTML = '<p class="foot">No items match.</p>'; | |
| } | |
| fm.onchange = render; document.getElementById('fresult').onchange = render; | |
| document.getElementById('q').oninput = render; | |
| render(); | |
| </script> | |
| </body> | |
| </html> | |