/* Home: dense tables. Colour marks severity and gaps; everything else is plain figures. */ (function () { "use strict"; const T = window.TB; const { D, el, link, pct0, fmt1, money, shortModel, rowPath, rowPage, title, strip, RZ, RZ_HELP, visibleShare, mixBar } = T; const $ = id => document.getElementById(id); /* Shared cells. */ function rowHead(r) { const th = el("th", "rowh"); const name = el("span", "rn"); name.append(link(rowPage(r), title(r), "rname"), el("span", "harn", r.agent)); th.append(el("span", "rk", String(r.rank)), name); return th; } function heat(td, share, scale, hue) { if (share > 0) td.style.setProperty("--a", String(Math.min(1, share / scale))); td.classList.add("heat", "heat--" + (hue || "warm")); } function num(v, cls) { return el("span", cls || "n", v === null || v === undefined ? "–" : String(v)); } const holeHref = (r, key) => rowPage(r) + "#k=" + key; function countCell(r, key, v, n, opts = {}) { const td = el("td", "c"); if (!v) { td.append(el("span", "z", "·")); return td; } heat(td, v / (opts.of || n), opts.scale || 0.5, opts.hue); const a = link(holeHref(r, key), String(v) + (opts.of ? "/" + opts.of : "")); a.title = opts.help ? opts.help + " — open the trials" : "open the trials"; td.append(a); return td; } function headRow(cols, first) { const tr = el("tr"); const th0 = el("th", "rowh", first || ""); tr.append(th0); cols.forEach(c => { const th = el("th", c.cls || null, c.name); if (c.help) th.title = c.help; if (c.sort) { th.classList.add("sortable"); th.dataset.sort = c.sort; } tr.append(th); }); const thead = el("thead"); thead.append(tr); return thead; } /* Notable problems, as plain coloured text: only what changes how a row reads. */ function flags(r) { const h = r.scan.holes, m = h.models, out = []; if (m.mismatch && !m.alias) out.push(["bad", "served " + shortModel(m.served[0].model)]); else if (m.alias) out.push(["note", "served " + shortModel(m.served[0].model) + " (alias)"]); else if (m.foreignTrials) out.push(["warn", "fallback " + m.served.filter(s => !s.configured).map(s => shortModel(s.model).replace(/^claude-/, "")).join(", ") + " in " + m.foreignTrials + " trials"]); if (h.cost.unbilled) out.push(["bad", "output undercounted"]); if (r.lb.disqualified) out.push(["bad", r.lb.disqualified + " disqualified by the PR pipeline"]); return out; } const hasLb = D.rows.some(r => r.lb.source !== "recount"); const SITE = { attempts: 5, lbLabel: "Leader­board", ...(D.site || {}) }; const ESTIMATE_HELP = "our estimate: rewarded trials, less review leads (an unexcused high or critical finding) and trials partly run by a fallback model"; const compactN = n => (n >= 1e6 ? (n / 1e6).toFixed(1) + "M" : n >= 1e3 ? Math.round(n / 1e3) + "k" : String(n)); /* 1. Leaderboard: the figures first, the attempt strip on the right (it scrolls on small screens). */ let sortKey = "rank", sortDir = 1; const leadsOnly = $("leads-only"); const asRun = D.rows.some(r => r.lb.asRun !== undefined); const SORTS = { rank: r => r.rank, score: r => -r.lb.accuracy, adj: r => -r.scan.adjusted.accuracy, leads: r => -r.scan.leads, run: r => -(r.lb.asRun ?? 0), }; function board() { const cols = [ ...(asRun ? [ { name: "As run", sort: "run", help: "rewarded trials over all attempts, as the run recorded them" }, { name: "Review", cls: "c", help: SITE.reviewHelp || "points the run's own review took off" }] : []), { name: hasLb ? SITE.lbLabel : "Score", sort: "score", help: hasLb ? "published score ± 95% half-width" : "rewarded trials ± 95% half-width over the attempt rounds" }, { name: "Our estimate", sort: "adj", help: ESTIMATE_HELP }, { name: "Leads", sort: "leads", help: "rewarded trials with an unexcused high or critical finding: they open the highlights" }, { name: "Notes", cls: "l" }, { name: D.tasks.length + " tasks × " + SITE.attempts + " attempts, easiest first", cls: "l" }, ]; const t = $("board"); t.replaceChildren(headRow(cols, "Model · harness")); const tb = el("tbody"); D.rows.slice().sort((a, b) => sortDir * (SORTS[sortKey](a) - SORTS[sortKey](b)) || a.rank - b.rank).forEach(r => { const tr = el("tr"), adj = r.scan.adjusted; tr.append(rowHead(r)); if (asRun) { tr.append(el("td", null, r.lb.asRun === undefined ? "–" : fmt1(r.lb.asRun))); const off = r.lb.asRun === undefined ? 0 : r.lb.accuracy - r.lb.asRun; const rv = el("td", "c" + (off <= -0.05 ? " hot" : " z"), off <= -0.05 ? "−" + fmt1(-off) : "·"); if (r.lb.reviewNote) rv.title = r.lb.reviewNote; tr.append(rv); } const sc = el("td"); sc.append(num(fmt1(r.lb.accuracy), "n big"), el("span", "pm", "±" + fmt1(r.lb.ci95))); sc.title = r.scan.rewarded + " of " + r.scan.trials + " trials rewarded" + (r.lb.passAt4 ? " · pass@4 " + r.lb.passAt4 + "%" : ""); tr.append(sc); const ad = el("td"); ad.append(num(fmt1(adj.accuracy), "n big" + (adj.accuracy - r.lb.accuracy <= -1 ? " hot" : ""))); ad.title = ESTIMATE_HELP + ": " + adj.rewarded + " rewarded" + (adj.leads ? ", less " + adj.leads + " leads" : "") + (adj.fallback ? ", less " + adj.fallback + " fallback" : ""); tr.append(ad); const ld = el("td"); if (r.scan.leads) ld.append(link(rowPath(r) + "highlights/index.html", String(r.scan.leads), "n big hot")); else ld.append(el("span", "z", "0")); tr.append(ld); const nt = el("td", "l notes"); if (r.why) { const w = el("span", "why", r.short || r.why); w.title = r.why; nt.append(w); } flags(r).forEach(([lvl, text]) => nt.append(el("span", "fl fl--" + lvl, text))); tr.append(nt); const s = el("td", "l"); s.append(strip(r, { leadsOnly: leadsOnly.checked })); tr.append(s); tb.append(tr); }); t.append(tb); t.querySelectorAll("th.sortable").forEach(th => { if (th.dataset.sort === sortKey) th.classList.add("sorted"); th.addEventListener("click", () => { sortDir = sortKey === th.dataset.sort ? -sortDir : 1; sortKey = th.dataset.sort; board(); }); }); } leadsOnly.addEventListener("change", board); /* A table with grouped column heads; cells(r) returns one cell per column, in order. */ function grouped(id, groups, cells) { const g = el("tr", "groups"), hr = el("tr"), first = new Set(); g.append(el("th", "rowh", "")); hr.append(el("th", "rowh", "Model · harness")); let n = 0; groups.forEach(([name, cols], i) => { const th = el("th", "grp" + (i ? " gstart" : ""), name); th.colSpan = cols.length; g.append(th); if (i) first.add(n); cols.forEach(([cname, cls, help], j) => { const c = el("th", (cls || "") + (j === 0 && i ? " gstart" : ""), cname); c.title = help; hr.append(c); n++; }); }); const thead = el("thead"); thead.append(g, hr); const t = $(id); t.replaceChildren(thead); const tb = el("tbody"); D.rows.forEach(r => { const tr = el("tr"); tr.append(rowHead(r)); cells(r).forEach((td, k) => { if (first.has(k)) td.classList.add("gstart"); tr.append(td); }); tb.append(tr); }); t.append(tb); } const zero = (cls, v) => el("td", cls + " z", v || "·"); /* 2. What the traces leave out, and how much: one column per gap; the detail is on each row's page. */ function gaps() { grouped("gaps-table", [ ["Output tokens", [ ["Text in the trace", "l", "the row's output tokens: tool calls, messages and reasoning with text in the trace, and (orange) output with no text in it"], ["Why not", "l", "why some output tokens have no text in the trace"]]], ["Input tokens", [ ["Web content not in trace", "", "web results and pages the model read whose text isn't in the trace: ≈ input tokens, from the prompt's growth on steps that used the web (rough), and the trials that read some"]]], ["Steps", [ ["Compacted", "", "trials whose history was compacted into a summary: earlier steps aren't in the trace"], ["No trajectory", "", "listed trials without a trajectory"]]], ["Cost", [ ...(hasLb ? [[SITE.lbLabel, "", "the published total"]] : []), ["Recorded", "", "sum of the trials' recorded costs"], ["Our estimate", "", "recorded + uncosted trials priced from their tokens + undercounted reasoning, at the row's fitted rates; ≥ where some trials can't be priced"]]], ], r => { const h = r.scan.holes, n = r.scan.trials, tk = h.tokens, c = h.cost, w = h.web || {}; const bar = el("td", "l"); bar.append(mixBar(tk)); const why = el("td", "l rz rz--" + h.text.why, RZ[h.text.why]); why.title = RZ_HELP[h.text.why]; let web; if (w.hidden) { web = el("td", "c"); heat(web, w.hidden / n, 0.5); const a = link(holeHref(r, "webHidden"), "≈" + compactN(w.tokens) + " tok"); a.title = w.hidden + " of " + w.trials + " web trials read content not in the trace · median ≈" + compactN(w.median) + " tokens a trial"; web.append(a, el("small", null, w.hidden + " trials")); } else if (w.trials && w.sized < w.trials && h.webUnrecorded) { web = el("td", "c"); web.append(el("span", "muted", "can't size"), el("small", null, h.webUnrecorded + " trials")); web.title = "results not in the trace, but no per-step token counts to size them from"; } else web = zero("c"); const est = el("td", "n" + (c.estimate && c.estimate > c.recorded * 1.02 ? " bad" : "")); est.append(document.createTextNode(c.estimate === null ? "–" : (c.lowerBound ? "≥" : "") + money(c.estimate))); const parts = []; if (c.imputedTrials) parts.push("+" + money(c.imputed) + " priced"); if (c.unbilled) parts.push("+" + money(c.unbilled) + " undercounted"); if (c.unpriced) parts.push(c.unpriced + " unpriced"); if (parts.length) est.append(el("small", null, parts.join(" · "))); return [bar, why, web, countCell(r, "compacted", h.compacted, n, { scale: 0.2 }), countCell(r, "noTrajectory", h.noTrajectory, n, { scale: 0.2 }), ...(hasLb ? [el("td", "n", r.lb.cost === null ? "–" : money(r.lb.cost))] : []), el("td", "n", money(c.recorded)), est]; }); } /* 4. Findings by family: a dot per family, its area the share of trials; filled when some were rewarded. */ function families() { const keys = Object.keys(D.categories).filter(k => D.rows.some(r => (r.scan.categories[k] || {}).trials)); const t = $("families-table"); t.replaceChildren(headRow(keys.map(k => ({ name: D.categories[k], cls: "c" })), "Row")); const tb = el("tbody"); D.rows.forEach(r => { const tr = el("tr"); tr.append(rowHead(r)); keys.forEach(k => { const v = r.scan.categories[k] || { trials: 0, rewarded: 0 }; const td = el("td", "c dotc"); if (v.trials) { const share = v.trials / r.scan.trials; const dot = link(rowPage(r) + "#found", "", "dot" + (v.rewarded ? " dot--won" : "")); const size = Math.round(5 + 25 * Math.sqrt(Math.min(1, share / 0.15))); dot.style.width = dot.style.height = size + "px"; dot.title = D.categories[k] + " · " + v.trials + " trials (" + (100 * share).toFixed(1) + "%) · " + v.rewarded + " rewarded"; td.append(dot, el("small", null, (100 * share).toFixed(1) + "%")); } tr.append(td); }); tb.append(tr); }); t.append(tb); } /* 4. Checks × rows: each cell opens that row's excerpts for the check. */ const HARN = { "Claude Code": "CC", "Terminus 2": "T2", "Gemini CLI": "Gemini CLI", "mini-SWE-agent": "mini-SWE", "Cursor CLI": "Cursor" }; const rewardedOnly = $("checks-rewarded"); function checks() { const t = $("checks-table"); const hr = el("tr"); hr.append(el("th", "rowh", "Check")); D.rows.forEach(r => { const th = el("th", "vh"); const span = el("span"); const a = link(rowPage(r), r.rank + " " + r.model + " " + r.effort + " · " + (HARN[r.agent] || r.agent)); a.title = title(r) + " · " + r.agent; span.append(a); th.append(span); hr.append(th); }); const thead = el("thead"); thead.append(hr); t.replaceChildren(thead); const tb = el("tbody"); const order = Object.keys(D.categories).concat([null]); const sev = { critical: 0, high: 1, medium: 2 }; const pick = v => (v ? (rewardedOnly.checked ? v.rewarded : v.trials) : 0); let last; Object.keys(D.checks).sort((a, b) => order.indexOf(D.checks[a].category) - order.indexOf(D.checks[b].category) || sev[D.checks[a].severity] - sev[D.checks[b].severity] || a.localeCompare(b)).forEach(id => { const c = D.checks[id]; if (!D.rows.some(r => pick(D.matrix[r.id][id]))) return; if (c.category !== last) { last = c.category; const cr = el("tr", "cat"), cth = el("th", null, c.category ? D.categories[c.category] : "Other"); cth.colSpan = D.rows.length + 1; cr.append(cth); tb.append(cr); } const tr = el("tr"); const th = el("th", "rowh chk"); th.append(el("span", "sev sev--" + c.severity, c.severity[0].toUpperCase()), el("span", "ctitle", c.title)); th.title = id + " · " + c.severity; tr.append(th); D.rows.forEach(r => { const v = D.matrix[r.id][id], n = pick(v), td = el("td", "c"); if (!n) { tr.append(td); return; } heat(td, n / r.scan.trials, 0.06, c.severity === "medium" ? "amber" : "warm"); const a = link(rowPath(r) + "highlights/index.html#check=" + encodeURIComponent(id), String(n)); a.title = r.rank + " " + title(r) + " · " + r.agent + " — " + v.trials + " trials, " + v.rewarded + " rewarded"; td.append(a); if (!rewardedOnly.checked && v.rewarded !== v.trials) td.append(el("small", null, "(" + v.rewarded + ")")); tr.append(td); }); tb.append(tr); }); t.append(tb); } rewardedOnly.addEventListener("change", checks); /* Method notes. */ const defs = $("rz-defs"); Object.keys(RZ).forEach(k => defs.append(el("dt", "rz rz--" + k, RZ[k]), el("dd", null, RZ_HELP[k]))); const notes = $("notes"), seen = new Set(); D.rows.forEach(r => { const note = r.scan.holes.sourceNote; if (!note || seen.has(r.agent)) return; seen.add(r.agent); notes.append(el("dt", null, r.agent), el("dd", null, note)); }); if (!notes.children.length) { notes.previousElementSibling.hidden = true; notes.hidden = true; } /* Counts in the lede, and leaderboard rows without published trajectories. */ const fill = (sel, text) => document.querySelectorAll(sel).forEach(n => { n.textContent = text; }); fill("[data-rows]", String(D.rows.length)); fill("[data-trials]", D.rows.reduce((n, r) => n + r.scan.trials, 0).toLocaleString("en")); const missing = SITE.unpublished || []; fill("[data-unpublished]", missing.length ? "Not here, as Datacurve publishes no trajectories for them: " + missing.map(u => "#" + u.rank + " " + u.model + " · " + u.effort + " (" + u.accuracy + "%)").join(", ") + "." : ""); board(); gaps(); families(); checks(); })();