| <!doctype html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1" /> |
| <title>Quorum — which leaderboard gaps are real?</title> |
| <style> |
| :root { color-scheme: light dark; --fg:#1b1b1b; --muted:#5a5a5a; --line:#e2e2e2; |
| --bg:#ffffff; --card:#fafafa; } |
| @media (prefers-color-scheme: dark) { |
| :root { --fg:#e9e9e9; --muted:#a5a5a5; --line:#333; --bg:#141414; --card:#1c1c1c; } |
| } |
| * { box-sizing: border-box; } |
| body { margin:0; background:var(--bg); color:var(--fg); line-height:1.6; |
| font-family: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif; } |
| main { max-width: 60rem; margin: 0 auto; padding: 2rem 1.2rem 4rem; } |
| h1 { font-size: 1.9rem; margin: 0 0 .2rem; letter-spacing: -.02em; } |
| .lede { font-size: 1.12rem; color: var(--muted); margin: 0 0 1.6rem; } |
| .note { background: var(--card); border:1px solid var(--line); border-left:3px solid #c0392b; |
| border-radius:6px; padding: .9rem 1.1rem; margin: 1.4rem 0; font-size:.95rem; } |
| .controls { display:flex; align-items:center; gap:1rem; flex-wrap:wrap; |
| background:var(--card); border:1px solid var(--line); |
| border-radius:8px; padding:1rem 1.2rem; margin:1.6rem 0; } |
| .controls label { font-weight:600; font-size:.95rem; } |
| input[type=range] { flex:1; min-width:14rem; } |
| #alphaval { font-variant-numeric: tabular-nums; font-weight:700; min-width:4.5rem; } |
| table { border-collapse: collapse; font-size: .88rem; width:100%; } |
| th, td { padding:.45rem .6rem; border:1px solid var(--line); text-align:center; } |
| th { background: var(--card); } |
| td.name, th.name { text-align:left; white-space:nowrap; } |
| .scroll { overflow-x:auto; margin: .8rem 0 1.6rem; } |
| .sep { background:#e8f5ec; color:#1d6b45; } |
| .nosep { background:#fdeaea; color:#a32222; font-weight:600; } |
| @media (prefers-color-scheme: dark) { |
| .sep { background:#14301f; color:#7fd7a6; } |
| .nosep { background:#3a1717; color:#f09a9a; } |
| } |
| small { color: var(--muted); } |
| a { color:#2e6fb5; } |
| footer { margin-top:2.5rem; padding-top:1.2rem; border-top:1px solid var(--line); |
| font-size:.9rem; color:var(--muted); } |
| </style> |
| </head> |
| <body> |
| <main> |
| <h1>Quorum</h1> |
| <p class="lede">Which leaderboard gaps does the data actually support?</p> |
|
|
| <p>The table below is a synthetic leaderboard. The scores differ. Read down the |
| <b>observed score</b> column and you will start ranking the models — that is what a |
| column of numbers does to a reader.</p> |
|
|
| <p>Now look at the pairwise matrix. Every pair recomputed with an <b>anytime-valid |
| confidence sequence</b> on the paired, scaffold-marginalised contrast. Red means the |
| interval contains zero: <b>this data cannot tell those two apart.</b></p> |
|
|
| <div class="controls"> |
| <label for="alpha">Family-wise alpha</label> |
| <input type="range" id="alpha" min="0" max="3" step="1" value="1" /> |
| <span id="alphaval"></span> |
| </div> |
|
|
| <div class="note" id="summary"></div> |
|
|
| <h2>Models</h2> |
| <div class="scroll" id="models"></div> |
|
|
| <h2>Pairwise separation</h2> |
| <div class="scroll" id="matrix"></div> |
|
|
| <p><small>Intervals are anytime-valid: they may be inspected after every observation |
| with no correction for peeking. Alpha is split across all candidate pairs by Bonferroni. |
| The “additional rollouts” figure extrapolates the observed shrinkage rate |
| — it is a projection, not a guarantee.</small></p> |
|
|
| <h2>How to read “not distinguishable”</h2> |
| <p>It means <i>this data cannot separate these two</i>. It does not mean the models are |
| equal, and it is not a criticism of either one. The demo data here is synthetic and the |
| labels are placeholders — nothing on this page measures, ranks or disparages a |
| real system.</p> |
|
|
| <h2>Run it on your own data</h2> |
| <p>This page is static, so it cannot take an upload. The same analysis runs locally in |
| one command:</p> |
| <pre style="background:var(--card);border:1px solid var(--line);border-radius:6px;padding:1rem;overflow-x:auto"><code>pip install -e . |
| quorum compare modelA modelB --benchmark synthetic-easy --budget $50</code></pre> |
|
|
| <footer> |
| Generated by <code>scripts/build_static_space.py</code> from the same estimator the |
| test suite exercises — |
| <a href="https://github.com/NagaYu/quorum">github.com/NagaYu/quorum</a>. |
| Apache-2.0. |
| </footer> |
| </main> |
|
|
| <script id="results" type="application/json">[{"alpha": "0.01", "summary": "6 of 10 pairs are not distinguishable at alpha = 0.01 (Bonferroni-split to 0.001 per pair, anytime-valid empirical-Bernstein). Red cells show the projected additional paired rollouts needed -- a projection from the observed shrinkage rate, not a guarantee.", "models": "<table><thead><tr><th class='name'>model</th><th>observed score</th><th>rollouts</th><th>pairs not separated</th></tr></thead><tbody><tr><td class='name'>model-A</td><td>0.4049</td><td>1,440</td><td>1</td></tr><tr><td class='name'>model-B</td><td>0.3153</td><td>1,440</td><td>3</td></tr><tr><td class='name'>model-C</td><td>0.2424</td><td>1,440</td><td>3</td></tr><tr><td class='name'>model-D</td><td>0.1931</td><td>1,440</td><td>3</td></tr><tr><td class='name'>model-E</td><td>0.1472</td><td>1,440</td><td>2</td></tr></tbody></table>", "matrix": "<table style='border-collapse:collapse;font-size:13px'><tr><th></th><th style='padding:6px'>model-A</th><th style='padding:6px'>model-B</th><th style='padding:6px'>model-C</th><th style='padding:6px'>model-D</th><th style='padding:6px'>model-E</th></tr><tr><th style='padding:6px;text-align:left'>model-A</th><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='nosep'><b>not distinguishable</b><br><small>[-0.046, +0.226]<br>+~377 rollouts</small></td><td class='sep'>separated<br><small>[+0.027, +0.298]</small></td><td class='sep'>separated<br><small>[+0.079, +0.349]</small></td><td class='sep'>separated<br><small>[+0.127, +0.396]</small></td></tr><tr><th style='padding:6px;text-align:left'>model-B</th><td class='nosep'><b>not distinguishable</b><br><small>[-0.046, +0.226]<br>+~377 rollouts</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='nosep'><b>not distinguishable</b><br><small>[-0.063, +0.209]<br>+~682 rollouts</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.013, +0.258]<br>+~87 rollouts</small></td><td class='sep'>separated<br><small>[+0.032, +0.305]</small></td></tr><tr><th style='padding:6px;text-align:left'>model-C</th><td class='sep'>separated<br><small>[+0.027, +0.298]</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.063, +0.209]<br>+~682 rollouts</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='nosep'><b>not distinguishable</b><br><small>[-0.084, +0.183]<br>+~1,717 rollouts</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.039, +0.230]<br>+~290 rollouts</small></td></tr><tr><th style='padding:6px;text-align:left'>model-D</th><td class='sep'>separated<br><small>[+0.079, +0.349]</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.013, +0.258]<br>+~87 rollouts</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.084, +0.183]<br>+~1,717 rollouts</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='nosep'><b>not distinguishable</b><br><small>[-0.088, +0.180]<br>+~2,029 rollouts</small></td></tr><tr><th style='padding:6px;text-align:left'>model-E</th><td class='sep'>separated<br><small>[+0.127, +0.396]</small></td><td class='sep'>separated<br><small>[+0.032, +0.305]</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.039, +0.230]<br>+~290 rollouts</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.088, +0.180]<br>+~2,029 rollouts</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td></tr></table>"}, {"alpha": "0.05", "summary": "5 of 10 pairs are not distinguishable at alpha = 0.05 (Bonferroni-split to 0.005 per pair, anytime-valid empirical-Bernstein). Red cells show the projected additional paired rollouts needed -- a projection from the observed shrinkage rate, not a guarantee.", "models": "<table><thead><tr><th class='name'>model</th><th>observed score</th><th>rollouts</th><th>pairs not separated</th></tr></thead><tbody><tr><td class='name'>model-A</td><td>0.4049</td><td>1,440</td><td>1</td></tr><tr><td class='name'>model-B</td><td>0.3153</td><td>1,440</td><td>2</td></tr><tr><td class='name'>model-C</td><td>0.2424</td><td>1,440</td><td>3</td></tr><tr><td class='name'>model-D</td><td>0.1931</td><td>1,440</td><td>2</td></tr><tr><td class='name'>model-E</td><td>0.1472</td><td>1,440</td><td>2</td></tr></tbody></table>", "matrix": "<table style='border-collapse:collapse;font-size:13px'><tr><th></th><th style='padding:6px'>model-A</th><th style='padding:6px'>model-B</th><th style='padding:6px'>model-C</th><th style='padding:6px'>model-D</th><th style='padding:6px'>model-E</th></tr><tr><th style='padding:6px;text-align:left'>model-A</th><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='nosep'><b>not distinguishable</b><br><small>[-0.019, +0.200]<br>+~157 rollouts</small></td><td class='sep'>separated<br><small>[+0.054, +0.272]</small></td><td class='sep'>separated<br><small>[+0.110, +0.322]</small></td><td class='sep'>separated<br><small>[+0.159, +0.369]</small></td></tr><tr><th style='padding:6px;text-align:left'>model-B</th><td class='nosep'><b>not distinguishable</b><br><small>[-0.019, +0.200]<br>+~157 rollouts</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='nosep'><b>not distinguishable</b><br><small>[-0.036, +0.182]<br>+~354 rollouts</small></td><td class='sep'>separated<br><small>[+0.015, +0.231]</small></td><td class='sep'>separated<br><small>[+0.061, +0.278]</small></td></tr><tr><th style='padding:6px;text-align:left'>model-C</th><td class='sep'>separated<br><small>[+0.054, +0.272]</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.036, +0.182]<br>+~354 rollouts</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='nosep'><b>not distinguishable</b><br><small>[-0.057, +0.156]<br>+~1,009 rollouts</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.012, +0.203]<br>+~99 rollouts</small></td></tr><tr><th style='padding:6px;text-align:left'>model-D</th><td class='sep'>separated<br><small>[+0.110, +0.322]</small></td><td class='sep'>separated<br><small>[+0.015, +0.231]</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.057, +0.156]<br>+~1,009 rollouts</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='nosep'><b>not distinguishable</b><br><small>[-0.061, +0.153]<br>+~1,206 rollouts</small></td></tr><tr><th style='padding:6px;text-align:left'>model-E</th><td class='sep'>separated<br><small>[+0.159, +0.369]</small></td><td class='sep'>separated<br><small>[+0.061, +0.278]</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.012, +0.203]<br>+~99 rollouts</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.061, +0.153]<br>+~1,206 rollouts</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td></tr></table>"}, {"alpha": "0.1", "summary": "4 of 10 pairs are not distinguishable at alpha = 0.1 (Bonferroni-split to 0.01 per pair, anytime-valid empirical-Bernstein). Red cells show the projected additional paired rollouts needed -- a projection from the observed shrinkage rate, not a guarantee.", "models": "<table><thead><tr><th class='name'>model</th><th>observed score</th><th>rollouts</th><th>pairs not separated</th></tr></thead><tbody><tr><td class='name'>model-A</td><td>0.4049</td><td>1,440</td><td>1</td></tr><tr><td class='name'>model-B</td><td>0.3153</td><td>1,440</td><td>2</td></tr><tr><td class='name'>model-C</td><td>0.2424</td><td>1,440</td><td>2</td></tr><tr><td class='name'>model-D</td><td>0.1931</td><td>1,440</td><td>2</td></tr><tr><td class='name'>model-E</td><td>0.1472</td><td>1,440</td><td>1</td></tr></tbody></table>", "matrix": "<table style='border-collapse:collapse;font-size:13px'><tr><th></th><th style='padding:6px'>model-A</th><th style='padding:6px'>model-B</th><th style='padding:6px'>model-C</th><th style='padding:6px'>model-D</th><th style='padding:6px'>model-E</th></tr><tr><th style='padding:6px;text-align:left'>model-A</th><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='nosep'><b>not distinguishable</b><br><small>[-0.007, +0.188]<br>+~77 rollouts</small></td><td class='sep'>separated<br><small>[+0.066, +0.260]</small></td><td class='sep'>separated<br><small>[+0.123, +0.311]</small></td><td class='sep'>separated<br><small>[+0.174, +0.357]</small></td></tr><tr><th style='padding:6px;text-align:left'>model-B</th><td class='nosep'><b>not distinguishable</b><br><small>[-0.007, +0.188]<br>+~77 rollouts</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='nosep'><b>not distinguishable</b><br><small>[-0.024, +0.170]<br>+~235 rollouts</small></td><td class='sep'>separated<br><small>[+0.028, +0.220]</small></td><td class='sep'>separated<br><small>[+0.074, +0.266]</small></td></tr><tr><th style='padding:6px;text-align:left'>model-C</th><td class='sep'>separated<br><small>[+0.066, +0.260]</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.024, +0.170]<br>+~235 rollouts</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='nosep'><b>not distinguishable</b><br><small>[-0.045, +0.145]<br>+~749 rollouts</small></td><td class='sep'>separated<br><small>[+0.000, +0.191]</small></td></tr><tr><th style='padding:6px;text-align:left'>model-D</th><td class='sep'>separated<br><small>[+0.123, +0.311]</small></td><td class='sep'>separated<br><small>[+0.028, +0.220]</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.045, +0.145]<br>+~749 rollouts</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='nosep'><b>not distinguishable</b><br><small>[-0.049, +0.142]<br>+~909 rollouts</small></td></tr><tr><th style='padding:6px;text-align:left'>model-E</th><td class='sep'>separated<br><small>[+0.174, +0.357]</small></td><td class='sep'>separated<br><small>[+0.074, +0.266]</small></td><td class='sep'>separated<br><small>[+0.000, +0.191]</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.049, +0.142]<br>+~909 rollouts</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td></tr></table>"}, {"alpha": "0.2", "summary": "3 of 10 pairs are not distinguishable at alpha = 0.2 (Bonferroni-split to 0.02 per pair, anytime-valid empirical-Bernstein). Red cells show the projected additional paired rollouts needed -- a projection from the observed shrinkage rate, not a guarantee.", "models": "<table><thead><tr><th class='name'>model</th><th>observed score</th><th>rollouts</th><th>pairs not separated</th></tr></thead><tbody><tr><td class='name'>model-A</td><td>0.4049</td><td>1,440</td><td>0</td></tr><tr><td class='name'>model-B</td><td>0.3153</td><td>1,440</td><td>1</td></tr><tr><td class='name'>model-C</td><td>0.2424</td><td>1,440</td><td>2</td></tr><tr><td class='name'>model-D</td><td>0.1931</td><td>1,440</td><td>2</td></tr><tr><td class='name'>model-E</td><td>0.1472</td><td>1,440</td><td>1</td></tr></tbody></table>", "matrix": "<table style='border-collapse:collapse;font-size:13px'><tr><th></th><th style='padding:6px'>model-A</th><th style='padding:6px'>model-B</th><th style='padding:6px'>model-C</th><th style='padding:6px'>model-D</th><th style='padding:6px'>model-E</th></tr><tr><th style='padding:6px;text-align:left'>model-A</th><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='sep'>separated<br><small>[+0.004, +0.176]</small></td><td class='sep'>separated<br><small>[+0.079, +0.249]</small></td><td class='sep'>separated<br><small>[+0.137, +0.299]</small></td><td class='sep'>separated<br><small>[+0.188, +0.346]</small></td></tr><tr><th style='padding:6px;text-align:left'>model-B</th><td class='sep'>separated<br><small>[+0.004, +0.176]</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='nosep'><b>not distinguishable</b><br><small>[-0.013, +0.159]<br>+~129 rollouts</small></td><td class='sep'>separated<br><small>[+0.041, +0.208]</small></td><td class='sep'>separated<br><small>[+0.087, +0.255]</small></td></tr><tr><th style='padding:6px;text-align:left'>model-C</th><td class='sep'>separated<br><small>[+0.079, +0.249]</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.013, +0.159]<br>+~129 rollouts</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='nosep'><b>not distinguishable</b><br><small>[-0.033, +0.133]<br>+~520 rollouts</small></td><td class='sep'>separated<br><small>[+0.013, +0.180]</small></td></tr><tr><th style='padding:6px;text-align:left'>model-D</th><td class='sep'>separated<br><small>[+0.137, +0.299]</small></td><td class='sep'>separated<br><small>[+0.041, +0.208]</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.033, +0.133]<br>+~520 rollouts</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td><td class='nosep'><b>not distinguishable</b><br><small>[-0.037, +0.130]<br>+~646 rollouts</small></td></tr><tr><th style='padding:6px;text-align:left'>model-E</th><td class='sep'>separated<br><small>[+0.188, +0.346]</small></td><td class='sep'>separated<br><small>[+0.087, +0.255]</small></td><td class='sep'>separated<br><small>[+0.013, +0.180]</small></td><td class='nosep'><b>not distinguishable</b><br><small>[-0.037, +0.130]<br>+~646 rollouts</small></td><td style='padding:6px;text-align:center;opacity:.35'>-</td></tr></table>"}]</script> |
| <script> |
| const DATA = JSON.parse(document.getElementById("results").textContent); |
| const slider = document.getElementById("alpha"); |
| function render() { |
| const r = DATA[slider.value]; |
| document.getElementById("alphaval").textContent = "alpha = " + r.alpha; |
| document.getElementById("summary").innerHTML = r.summary; |
| document.getElementById("models").innerHTML = r.models; |
| document.getElementById("matrix").innerHTML = r.matrix; |
| } |
| slider.addEventListener("input", render); |
| render(); |
| </script> |
| </body> |
| </html> |
|
|