Blame for index.html

<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Reliability table builder: check whether your probabilities mean what they say</title>
<meta name="description" content="Paste forecast probabilities with their outcomes and get a reliability table by band, the Brier score and a plain-English verdict on calibration. Runs in the browser, nothing is uploaded.">
<style>
:root{--bg:#fbfbfd;--ink:#1a1c22;--mute:#5f6573;--line:#e1e4ea;--accent:#1f5fbf;--chip:#eaf1fc;--card:#fff}
@media (prefers-color-scheme:dark){:root{--bg:#121318;--ink:#e9ebf0;--mute:#a0a6b4;--line:#2b2e37;--accent:#7fb0ff;--chip:#1b2433;--card:#171a21}}
*{box-sizing:border-box}
body{margin:0;background:var(--bg);color:var(--ink);font:16px/1.6 system-ui,-apple-system,"Segoe UI",sans-serif}
main{max-width:800px;margin:0 auto;padding:40px 16px 64px}
h1{font-size:1.9rem;line-height:1.2;margin:0 0 10px}
h2{font-size:1.15rem;margin:34px 0 8px}
p{margin:0 0 14px}
.lead{color:var(--mute);font-size:1.08rem}
.card{background:var(--card);border:1px solid var(--line);border-radius:14px;padding:18px 20px;margin:22px 0}
label{display:block;font-size:.9rem;color:var(--mute);margin-bottom:6px}
textarea{width:100%;min-height:150px;font:14px/1.5 ui-monospace,Consolas,monospace;padding:10px 12px;border:1px solid var(--line);border-radius:10px;background:var(--bg);color:var(--ink)}
.row{display:flex;gap:10px;flex-wrap:wrap;margin-top:10px;align-items:center}
button{font:inherit;padding:8px 14px;border-radius:10px;border:1px solid var(--line);background:var(--bg);color:var(--ink);cursor:pointer}
button.primary{background:var(--accent);color:#fff;border-color:var(--accent)}
select{font:inherit;padding:7px 10px;border-radius:10px;border:1px solid var(--line);background:var(--bg);color:var(--ink)}
table{width:100%;border-collapse:collapse;margin-top:14px;font-variant-numeric:tabular-nums}
th,td{text-align:left;padding:8px 10px;border-bottom:1px solid var(--line)}
th{font-size:.82rem;text-transform:uppercase;letter-spacing:.04em;color:var(--mute)}
tr:last-child td{border-bottom:0}
.verdict{background:var(--chip);border-radius:12px;padding:12px 16px;margin-top:14px}
.err{color:#b3261e}
a{color:var(--accent)}
footer{margin-top:48px;color:var(--mute);font-size:.88rem;border-top:1px solid var(--line);padding-top:16px}
</style>
</head>
<body>
<main>
<h1>Reliability table builder</h1>
<p class="lead">A forecaster that says "70%" should be right about 70 times in 100. Paste your forecasts and what happened, and this page builds the table that checks it.</p>

<div class="card">
<label for="data">One line per forecast: probability (0 to 1, or a percent), then 1 if it happened or 0 if it did not. Commas, spaces or tabs between them. Extra columns are ignored.</label>
<textarea id="data" spellcheck="false">0.72,1
0.65,1
0.65,0
0.30,0
0.28,1
0.81,1
0.55,0
0.47,1
0.22,0
0.90,1
0.61,0
0.35,0
0.15,0
0.77,1
0.52,1
0.44,0
0.68,1
0.25,0
0.83,1
0.58,0</textarea>
<div class="row">
<button class="primary" id="go">Build the table</button>
<label for="bands" style="margin:0">Bands</label>
<select id="bands"><option value="5">5 (20-point bands)</option><option value="10" selected>10 (10-point bands)</option></select>
<button id="clear">Clear</button>
</div>
<p class="err" id="msg" style="margin:10px 0 0"></p>
<div id="out"></div>
</div>

<h2>How to read it</h2>
<p>Each row groups the forecasts that claimed a similar chance. <strong>Average claim</strong> is the mean probability stated in that band; <strong>observed</strong> is the share of those forecasts whose outcome happened. When the two are close in every band the forecaster is well calibrated. A band where observed sits well below the claim is overconfident; well above, underconfident.</p>
<p>The <strong>Brier score</strong> is the mean squared gap between each probability and its outcome (1 or 0). Lower is better; always guessing 50% scores 0.25 on balanced data. It rewards both calibration and sharpness, so a forecaster can be perfectly calibrated and still have a poor Brier score if it never commits.</p>
<p>Small samples swing. With 20 forecasts a band may hold two or three cases, so its observed share can only be 0%, 33%, 50% or 100%. Treat anything under about 100 forecasts per band as a sketch, not a verdict.</p>

<h2>A worked example on real data</h2>
<p>The <a href="https://betbetter.world/studies/market-calibration">market calibration study</a> ran this exact table on both sides of 18,238 settled sports markets. Every band landed within about two points of its claim, with outcomes priced below 50% happening slightly more often than claimed and those above 50% slightly less. Its open data set, one row per outcome with the margin-free probability, is a good file to paste here if you want to see what a large, well-calibrated sample looks like. Margin-free model probabilities to test the same way are available from the <a href="https://betbetter.world/api/">Bet Better API</a> with no key.</p>

<footer>Built by Bet Better Pty Ltd, Melbourne, as a teaching tool for forecast evaluation. Everything runs in your browser; nothing is uploaded. Not betting advice.</footer>
</main>
<script>
(function(){
  var data=document.getElementById('data'), out=document.getElementById('out'), msg=document.getElementById('msg'), bandsSel=document.getElementById('bands');
  function pct(x){return (x*100).toFixed(1)+'%';}
  function run(){
    out.innerHTML=''; msg.textContent='';
    var rows=[], bad=0;
    data.value.split(/\r?\n/).forEach(function(line){
      var parts=line.trim().split(/[\s,;\t]+/).filter(Boolean);
      if(parts.length<2) return;
      var p=parseFloat(parts[0]), y=parseFloat(parts[1]);
      if(isNaN(p)||isNaN(y)){bad++;return;}
      if(p>1) p=p/100;
      if(p<0||p>1||(y!==0&&y!==1)){bad++;return;}
      rows.push([p,y]);
    });
    if(rows.length<2){msg.textContent='Need at least two valid lines of "probability, outcome".';return;}
    var k=parseInt(bandsSel.value,10), w=1/k, bands=[];
    for(var i=0;i<k;i++) bands.push({lo:i*w,hi:(i+1)*w,n:0,sp:0,sy:0});
    var brier=0;
    rows.forEach(function(r){
      var i=Math.min(k-1,Math.floor(r[0]/w)); var b=bands[i]; b.n++; b.sp+=r[0]; b.sy+=r[1];
      brier+=(r[0]-r[1])*(r[0]-r[1]);
    });
    brier/=rows.length;
    var html='<table><thead><tr><th>Claimed</th><th>Forecasts</th><th>Average claim</th><th>Observed</th><th>Gap</th></tr></thead><tbody>';
    var maxGap=0, nBig=0;
    bands.forEach(function(b){
      if(!b.n) return;
      var claim=b.sp/b.n, obs=b.sy/b.n, gap=(obs-claim)*100;
      if(b.n>=10){ if(Math.abs(gap)>maxGap) maxGap=Math.abs(gap); if(Math.abs(gap)>5) nBig++; }
      html+='<tr><td>'+Math.round(b.lo*100)+' to '+Math.round(b.hi*100)+'%</td><td>'+b.n+'</td><td>'+pct(claim)+'</td><td>'+pct(obs)+'</td><td>'+(gap>=0?'+':'')+gap.toFixed(1)+' pts</td></tr>';
    });
    html+='</tbody></table>';
    var base=rows.reduce(function(a,r){return a+r[1];},0)/rows.length;
    var verdict;
    if(rows.length<50) verdict='Only '+rows.length+' forecasts: the table is a sketch. Gaps of 10 points or more are expected from noise alone at this size.';
    else if(nBig===0) verdict='Every band with at least 10 forecasts sits within 5 points of its claim. On this sample the forecasts are well calibrated.';
    else verdict=nBig+' band'+(nBig>1?'s':'')+' with at least 10 forecasts miss'+(nBig>1?'':'es')+' by more than 5 points (largest gap '+maxGap.toFixed(1)+'). Look at whether the misses are all in one direction: below the claim means overconfident, above means underconfident.';
    out.innerHTML='<div class="verdict"><strong>'+rows.length+' forecasts.</strong> Brier score '+brier.toFixed(4)+' (always guessing the base rate of '+pct(base)+' would score '+(base*(1-base)).toFixed(4)+'). '+verdict+(bad?' '+bad+' line'+(bad>1?'s':'')+' skipped.':'')+'</div>'+html;
  }
  document.getElementById('go').addEventListener('click',run);
  document.getElementById('clear').addEventListener('click',function(){data.value='';out.innerHTML='';msg.textContent='';});
  bandsSel.addEventListener('change',run);
  run();
})();
</script>
</body>
</html>