kinex-benchmark / app.js
Jerrycool's picture
Squash history
cde5f1e
Raw History Blame Contribute Delete
62.7 kB
'use strict';
const $ = s => document.querySelector(s);
const $$ = s => [...document.querySelectorAll(s)];
const esc = value => String(value ?? '').replace(/[&<>"']/g, c => ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
const number = v => v == null ? '—' : Number(v).toLocaleString('en-US');
const compact = v => v == null ? '—' : Math.abs(v) >= 1e9 ? (v/1e9).toFixed(2)+'B' : Math.abs(v) >= 1e6 ? (v/1e6).toFixed(2)+'M' : Math.abs(v) >= 1e3 ? (v/1e3).toFixed(1)+'K' : Math.round(v).toString();
const pct = v => v == null ? '—' : (v*100).toFixed(1)+'%';
const elapsed = v => v == null ? '—' : v >= 3600 ? (v/3600).toFixed(1)+' h' : v >= 60 ? (v/60).toFixed(1)+' min' : v.toFixed(1)+' s';
const arrow = '<span aria-hidden="true">↗</span>';
const colors = {'codex-astra':'#39819c','kinex-astra':'#756ee4','cc-opus':'#b48566','kinex-opus':'#958bc8','cc-opus-vertex-high':'#c9875a','cc-opus-vertex-xhigh':'#8c5a3a','kinex-opus-vertex-high':'#5b8bd8','kinex-opus-vertex-xhigh':'#3a5ba6'};
let data, epoch = 0, current = {}, matrixMetric = 'success';
let filters = {query:'',family:'',status:'',variant:'codex-astra',sort:'catalog'};
let sessionState = {filter:'all',query:'',count:40,view:'timeline'};
let imageItems = [], imageIndex = 0;
const sessionCache = new Map(), fileCache = new Map();
const episodes = (variant, family) => data.episodes.filter(e => (!variant || e.variant===variant) && (!family || e.family===family));
const epFor = (key, variant) => data.episodes.find(e => e.task_key===key && e.variant===variant);
const variantFor = id => data.variants.find(v => v.id===id);
const familyName = id => data.families.find(f => f.id===id)?.name || id;
const variantName = id => {const v=variantFor(id);return v ? `${v.agent} + ${v.model}` : id;};
const taskRoute = (key, variant='codex-astra', section='overview') => `#task/${key}/${variant}${section==='overview' ? '' : '/'+section}`;
const outcome = e => !e ? 'pending' : e.valid===false ? 'invalid' : typeof e.success!=='boolean' ? 'unreported' : e.success ? 'pass' : 'fail';
const badge = (e, link) => {const s=outcome(e),tag=link?'a':'span';return `<${tag} class="badge ${s}" ${link ? `href="${esc(link)}"` : ''}><span class="status-dot"></span>${s==='pending'?((data.active_campaign?.initialization_blocked||[]).includes((link||'').split('/').slice(1,4).join('/'))?'Blocked':'Pending'):s==='invalid'?'Invalid':s==='unreported'?'Unreported':s==='pass'?'Pass':'Fail'}</${tag}>`;};
const external = (url,label,cls='') => typeof url==='string' && url.startsWith('https://') ? `<a class="${cls}" href="${esc(url)}" target="_blank" rel="noopener">${esc(label)} ${arrow}</a>` : '';
const modelLabel = v => `${v.model} <span class="separator">/</span> ${v.effort}`;
const icon = v => `<img class="agent-logo" src="assets/${v.agent==='Kinex'?'kinex':v.agent==='Codex'?'openai':'anthropic'}-logo.svg" alt="" width="23" height="23">`;
function stats(es) {
return UsageMetrics.stats(es);
}
window.benchmarkStats = stats;
const tokens = e => e && Number.isFinite(e.usage?.input_tokens) && Number.isFinite(e.usage?.output_tokens) ? e.usage.input_tokens+e.usage.output_tokens : null;
function metric(value,label,detail='',attrs='') {return `<div class="metric" ${attrs}><span class="metric-label">${label}</span><strong>${value}</strong>${detail?`<small>${detail}</small>`:''}</div>`;}
function usageDefinition(){return 'Input = fresh input + cache writes. Cached = cache reads. Cache hit rate = cached / (input + cached), weighted by tokens. Output includes reasoning. Metrics cover current selected evaluated episodes; unavailable values are never zero.';}
function variantTabs(selected,taskKey,section) {
return `<div class="variant-tabs" aria-label="Choose agent and model">${data.variants.map(v=>`<a class="variant-tab ${v.id===selected?'active':''}" style="--variant:${colors[v.id]}" href="${taskKey?taskRoute(taskKey,v.id,epFor(taskKey,v.id)?section:'overview'):'#runs/'+v.id}" ${v.id===selected?'aria-current="page"':''}><span class="tab-agent">${icon(v)}<strong>${esc(v.agent)}</strong>${taskKey?badge(epFor(taskKey,v.id)):''}</span><span class="tab-model">${modelLabel(v)}</span></a>`).join('')}</div>`;
}
const rangeNumber = (lo,hi,formatter=compact) => UsageMetrics.bounds(lo,hi,formatter);
const costValue = s => UsageMetrics.bounds(s.costLow,s.costHigh,UsageMetrics.money);
const exactMoney = v => v == null ? '—' : '$'+v.toLocaleString('en-US',{minimumFractionDigits:6,maximumFractionDigits:6});
const costNote = s => !s.n ? 'Not evaluated' : s.costCount!==s.n ? `Pricing available for ${s.costCount} / ${s.n} episodes` : s.conditional ? 'Estimated · 5m cache-write basis' : 'Estimated · published rate rules';
function usageTiles(s,includeCost=true) {
return `<div class="usage-metrics">${metric(rangeNumber(s.inputLow,s.inputHigh),'Input tokens','Fresh input + cache writes',`title="${rangeNumber(s.inputLow,s.inputHigh,number)}; excludes cache reads"`)}${metric(rangeNumber(s.cacheLow,s.cacheHigh),'Cached tokens','Cache reads',`title="${rangeNumber(s.cacheLow,s.cacheHigh,number)}"`)}${metric(rangeNumber(s.cacheRateLow,s.cacheRateHigh,pct),'Cache hit rate','Cached / (input + cached)')}${metric(compact(s.output),'Output tokens',s.reasoning==null?'Includes reasoning when reported':compact(s.reasoning)+' reasoning')}${includeCost?metric(costValue(s),'Total cost',costNote(s)):''}</div>`;
}
function costBreakdown(s,open=false) {
if (!s.n) return '';
const names={fresh_input:'Fresh input',cache_read:'Cache reads',cache_write:'Cache writes',output:'Output',unclassified_input:'Unclassified input'};
const order=['fresh_input','cache_write','cache_read','output','unclassified_input'];
const rows=[...s.buckets].sort((a,b)=>order.indexOf(a.category)-order.indexOf(b.category)||a.tier.localeCompare(b.tier));
return `<details class="cost-breakdown" ${open?'open':''}><summary><span>Cost breakdown <span class="estimate-tag">USD · estimated</span></span><span class="summary-plus">+</span></summary><div class="cost-breakdown-body">${s.costCount!==s.n?`<p class="coverage-note">A complete cost is unavailable. The breakdown covers ${s.costCount} of ${s.n} episodes only.</p>`:''}<div class="table-scroll"><table class="price-table"><thead><tr><th>Usage category</th><th>Pricing tier</th><th>Tokens</th><th>USD / 1M tokens</th><th>Cost (USD)</th></tr></thead><tbody>${rows.map(b=>`<tr><th scope="row">${names[b.category]}</th><td>${esc(b.tier)}</td><td class="numeric">${number(b.tokens)}</td><td class="numeric">${rangeNumber(b.rate,b.rate_upper,UsageMetrics.money)}</td><td class="numeric">${rangeNumber(b.lower,b.upper,exactMoney)}</td></tr>`).join('')}</tbody><tfoot><tr><th colspan="4">${s.costCount===s.n?'Total estimated cost':'Known subtotal'}</th><td class="numeric">${s.costCount===s.n?rangeNumber(s.costLow,s.costHigh,exactMoney):exactMoney(s.knownCost)}</td></tr></tfoot></table></div><p>Each row = tokens × its rate / 1,000,000. Cache reads are charged separately; reasoning is already included in output.</p><p>Astra: above 272K input tokens per request, input/cache rates double and output rates increase by 1.5×. ${number(s.longContext)} such requests in this selection. Claude Code uses its recorded list-price estimate, reconciled to the tiers above.${s.conditional?' Kinex + Opus on Vertex uses a 5-minute cache-write assumption; the recorded audit does not establish cache TTL.':''}${s.bounded?' A range allocates unclassified input across the possible rates; it does not treat missing cache counters as zero.':''}</p><p>API-equivalent estimates, not verified account charges. Subscription allocation, discounts, credits, taxes, simulator costs and archived attempts are excluded. Rates checked ${esc(UsageMetrics.pricingDate||'—')}.</p><div class="inline-links">${UsageMetrics.sources.map(x=>external(x.url,x.label)).join('')}<a href="cost-data.json" target="_blank">Calculation data ↗</a></div></div></details>`;
}
function modelGroups() {
const groups=new Map();
for (const v of data.variants) {
const key=v.model+' / '+v.effort;
if(!groups.has(key))groups.set(key,{key,model:v.model,effort:v.effort,variants:[]});
groups.get(key).variants.push(v);
}
return [...groups.values()];
}
let expandedModels=null;
function comparisonCell(v,f) {
const s=stats(episodes(v.id,f.id)),link=`#runs/${v.id}?family=${f.id}`;
let value;
if(matrixMetric==='success')value=`<div class="comparison-value"><strong>${pct(s.rate)}</strong><span>${s.n?s.passes+' / '+s.n+' passed':'Not evaluated'}</span></div><div class="mini-track"><span style="width:${(s.rate||0)*100}%;background:${colors[v.id]}"></span></div>`;
else if(matrixMetric==='tokens')value=`<dl class="comparison-usage"><div><dt>Input</dt><dd title="${rangeNumber(s.inputLow,s.inputHigh,number)}">${rangeNumber(s.inputLow,s.inputHigh)}</dd></div><div><dt>Cached</dt><dd title="${rangeNumber(s.cacheLow,s.cacheHigh,number)}">${rangeNumber(s.cacheLow,s.cacheHigh)}</dd></div><div><dt>Cache hit</dt><dd>${rangeNumber(s.cacheRateLow,s.cacheRateHigh,pct)}</dd></div><div><dt>Output</dt><dd>${compact(s.output)}</dd></div></dl>`;
else value=`<div class="comparison-value"><strong>${costValue(s)}</strong><span>${costNote(s)}</span></div>${costBreakdown(s)}`;
return `<td data-harness="${esc(v.agent)}" style="--variant:${colors[v.id]}">${value}<a class="comparison-explore" href="${link}">${s.n} evaluated · ${f.tasks-s.n} pending ${arrow}</a></td>`;
}
function comparisonTable() {
return `<section class="comparison-panel" aria-labelledby="comparison-title"><div class="section-heading bare"><div><h2 id="comparison-title">Compare environments</h2><p>Choose a model to compare its harnesses. Coverage is shown for each environment.</p></div><div class="segmented" aria-label="Comparison metric">${[['success','Success rate'],['tokens','Token usage'],['cost','Total cost']].map(([k,l])=>`<button data-matrix="${k}" aria-pressed="${matrixMetric===k}">${l}</button>`).join('')}</div></div><p class="comparison-definition">${matrixMetric==='tokens'?usageDefinition():matrixMetric==='cost'?'Estimated USD at the recorded or documented API rates. Expand a breakdown to inspect token categories and pricing tiers.':'Native passes / evaluated tasks. Pending tasks are excluded; compare coverage alongside success rates.'}</p><div class="model-comparisons">${modelGroups().map((g,i)=>`<details class="model-comparison panel" data-model-group="${esc(g.key)}" ${expandedModels?expandedModels.has(g.key)?'open':'':i===0?'open':''}><summary><span><strong>${esc(g.model)}</strong><span class="model-group-meta">${esc(g.effort)} effort · ${g.variants.map(v=>esc(v.agent)).join(' vs ')}</span></span><span class="model-group-count">${g.variants.length} harnesses <span class="disclosure-arrow">⌄</span></span></summary><div class="table-scroll"><table class="comparison-table grouped-comparison ${matrixMetric}"><thead><tr><th>Environment</th>${g.variants.map(v=>`<th><a href="#runs/${v.id}" class="agent-inline">${icon(v)}${esc(v.agent)} ${arrow}</a></th>`).join('')}</tr></thead><tbody>${data.families.map(f=>`<tr><th scope="row"><span class="family-code">${f.id.slice(-2)}</span>${esc(f.name)}<small>${f.tasks} tasks</small></th>${g.variants.map(v=>comparisonCell(v,f)).join('')}</tr>`).join('')}</tbody></table></div></details>`).join('')}</div></section>`;
}
function campaignProgress(){
const c=data.active_campaign;if(!c)return '';const n=c.counts||{};
return `<section class="panel" style="padding:18px 24px;margin-bottom:24px" aria-label="Current evaluation campaign"><strong>BEHAVIOR evaluation · GPT-6 Astra / high</strong><p>${number(n.finished||0)} / ${number(c.planned_episodes)} episodes completed · ${number(n.running||0)} running · ${number(n.queued||0)} queued${n.not_run?' · '+number(n.not_run)+' unavailable':''}${n.needs_review?' · '+number(n.needs_review)+' under review':''}. ${c.closed&&c.published_episodes===n.finished?'Evaluations have finished. All available results, videos and evidence are listed below.':'Results, videos and evidence are published after verification.'}</p><p class="source-note">Updated ${esc(c.updated_at)} · <a href="${esc(c.status_url)}" target="_blank" rel="noopener">Campaign status and provenance ↗</a></p></section>`;
}
function opusPilotProgress(){
const c=data.pilot_campaigns?.['cc-opus'];if(!c)return '';const n=c.counts||{};
return `<section class="panel" style="padding:18px 24px;margin-bottom:24px" aria-label="Claude Code campaign"><strong>Claude Code · Opus 5.5 / high</strong><p>${number(n.finished||0)} / ${number(c.planned_episodes)} episodes completed · ${number(n.running||0)} running · ${number(n.queued||0)} pending${n.needs_review?' · '+number(n.needs_review)+' under review':''}${n.interrupted?' · '+number(n.interrupted)+' interrupted':''}.</p><p class="source-note">${c.quota_waiting?'Waiting for model-service quota recovery · ':''}${number(c.published_episodes)} episode records published · <a href="${esc(c.status_url)}" target="_blank" rel="noopener">Results and protocol ↗</a></p></section>`;
}
function methodology() {
return `<details class="panel methodology" id="methodology"><summary><span><span class="eyebrow">Reading the benchmark</span><strong>Protocol, metrics & provenance</strong></span><span class="summary-plus">+</span></summary><div class="method-grid"><div><h3>One selected episode</h3><p>Each task–agent pair shows its currently selected episode (seed 0 by default; explicit exceptions retain the measured seed). Previous attempts remain in the archive and are excluded from the displayed metrics. Results are added as episodes finish.</p></div><div><h3>Native success</h3><p>Success rate is passes / evaluated tasks, pooled over the selected episodes. Pending tasks do not count as failures. Results follow the original native verifier; Native notes identify verified simulator source issues.</p></div><div><h3>Measured configuration</h3><p>Each episode retains its actual instruction, control budget, runtime version and source revision. Historical runs retain their measured configuration. Availability is shown for each task and agent.</p></div><div><h3>Token accounting</h3><p>${usageDefinition()} Missing counters display as unavailable, never as zero.</p></div></div><div class="method-links"><a href="sources.json" target="_blank">Pinned datasets ↗</a><a href="attempt-history.json" target="_blank">Historical attempts ↗</a><button class="text-button" data-export="all">Download comparison CSV ↓</button></div></details>`;
}
function taskBrowser(variant=null) {
return `<section class="task-browser" id="tasks"><div class="section-heading bare"><div><div class="eyebrow">${variant?'Episode index':'Browse the benchmark'}</div><h2>Explore tasks <span class="heading-count">${data.tasks.length}</span></h2></div><button class="button secondary" data-export="${variant||'all'}">Export CSV ↓</button></div><div class="panel"><div class="filters"><label class="search-field"><span aria-hidden="true">⌕</span><input id="task-search" type="search" aria-label="Search tasks" placeholder="Search task name, instruction or ID" value="${esc(filters.query)}"></label><select id="task-family" aria-label="Environment"><option value="">All environments</option>${data.families.map(f=>`<option value="${f.id}" ${filters.family===f.id?'selected':''}>${f.name}</option>`).join('')}</select>${variant?'':`<select id="task-variant" aria-label="Agent for outcome filter">${data.variants.map(v=>`<option value="${v.id}" ${filters.variant===v.id?'selected':''}>${v.agent} · ${v.model}</option>`).join('')}</select>`}<select id="task-status" aria-label="Outcome"><option value="">All outcomes</option>${['pass','fail','pending'].map(s=>`<option value="${s}" ${filters.status===s?'selected':''}>${s[0].toUpperCase()+s.slice(1)}</option>`).join('')}</select><select id="task-sort" aria-label="Sort tasks"><option value="catalog" ${filters.sort==='catalog'?'selected':''}>Catalog order</option><option value="tokens" ${filters.sort==='tokens'?'selected':''}>Most input tokens</option><option value="cost" ${filters.sort==='cost'?'selected':''}>Highest cost</option><option value="name" ${filters.sort==='name'?'selected':''}>Task name</option></select></div><div class="table-scroll"><table class="task-table ${variant?'single-variant':''}"><thead><tr><th>Task</th>${variant?'<th>Native result</th><th title="Fresh input + cache writes">Input tokens<small>Fresh + cache writes</small></th><th>Cached tokens<small>Cache reads</small></th><th>Cache hit</th><th>Output</th><th>Total cost<small>Estimated USD</small></th><th>Control steps</th>':data.variants.map(v=>`<th class="${v.id===filters.variant?'focus-column':''}">${v.agent}<small>${v.model} · ${v.effort}</small></th>`).join('')}</tr></thead><tbody id="task-rows"></tbody></table></div><div class="table-footer"><span id="task-count" aria-live="polite"></span><span id="filter-context"></span><button class="text-button" id="reset-filters">Reset filters</button></div></div></section>`;
}
function renderRows() {
const variant=current.type==='runs'?current.variant:null, selected=variant||filters.variant, q=filters.query.trim().toLowerCase();
let tasks=data.tasks.filter(t=>(!filters.family||t.family===filters.family)&&(!filters.status||outcome(epFor(t.key,selected))===filters.status)&&(!q||[t.key,t.title,t.catalog_instruction,t.native_id,familyName(t.family)].some(s=>s?.toLowerCase().includes(q))));
if(filters.sort==='name') tasks.sort((a,b)=>a.title.localeCompare(b.title));
if(filters.sort==='tokens') tasks.sort((a,b)=>(UsageMetrics.usage(epFor(b.key,selected)).inputHigh??-1)-(UsageMetrics.usage(epFor(a.key,selected)).inputHigh??-1));
if(filters.sort==='cost') tasks.sort((a,b)=>(UsageMetrics.price(epFor(b.key,selected))?.upper??-1)-(UsageMetrics.price(epFor(a.key,selected))?.upper??-1));
$('#task-rows').innerHTML=tasks.length?tasks.map(t=>{const ep=epFor(t.key,selected),us=UsageMetrics.usage(ep),cs=stats(ep?[ep]:[]);return `<tr data-task="${t.key}"><th scope="row"><a class="task-title" href="${taskRoute(t.key,selected)}">${esc(t.title)} <span class="row-arrow">↗</span></a><span class="task-meta">${familyName(t.family)} <span>·</span> <code>${t.key}</code>${data.task_notes[t.key]?'<span class="native-marker" title="Simulator source issue">◆ Native note</span>':''}</span></th>${variant?`<td>${badge(ep,taskRoute(t.key,selected))}</td><td class="numeric" title="${rangeNumber(us.inputLow,us.inputHigh,number)}">${rangeNumber(us.inputLow,us.inputHigh)}</td><td class="numeric" title="${rangeNumber(us.cacheLow,us.cacheHigh,number)}">${rangeNumber(us.cacheLow,us.cacheHigh)}</td><td class="numeric">${rangeNumber(us.cacheRateLow,us.cacheRateHigh,pct)}</td><td class="numeric">${compact(us.output)}</td><td class="numeric" title="${esc(costNote(cs))}">${costValue(cs)}</td><td class="numeric">${number(ep?.steps)}<small>${ep?'of '+number(ep.measured_protocol?.max_steps):number(t.planned_protocol.max_control_steps)+' planned'}</small></td>`:data.variants.map(v=>`<td class="${v.id===filters.variant?'focus-column':''}">${badge(epFor(t.key,v.id),taskRoute(t.key,v.id))}</td>`).join('')}</tr>`;}).join(''):'<tr><td colspan="${variant?8:data.variants.length+1}" class="empty-state">No tasks match these filters. <button class="text-button" data-reset>Clear filters</button></td></tr>';
$('#task-count').textContent=`${tasks.length} of ${data.tasks.length} tasks`;
$('#filter-context').textContent=variant?variantName(variant):'Outcome filter & usage/cost sort: '+variantName(filters.variant);
$$('.task-table thead th').forEach((th,i)=>{if(!variant&&i)th.classList.toggle('focus-column',data.variants[i-1].id===filters.variant);});
}
function overview() {
current={type:'overview'};
$('#main').innerHTML=`<section class="hero"><div><div class="eyebrow"><span class="tiny-logo">✦</span> RLE-Bench · Embodied agent evaluation</div><h1>Coding agents.<br><span>Physical tasks.</span></h1><p>Compare native success, token usage and recorded behavior.<br>Every result opens into the full agent session and its evidence.</p></div><div class="hero-facts"><div><strong>6</strong><span>environments</span></div><div><strong>164</strong><span>tasks per agent</span></div><div><strong>${data.variants.length}</strong><span>agent–model variants</span></div><span class="progress-label"><i></i>Evaluation in progress · seed 0</span></div></section>
${campaignProgress()}${opusPilotProgress()}<div class="agent-cards">${data.variants.map(v=>{const s=stats(episodes(v.id));return `<a class="agent-card ${s.n?'':'planned'}" href="#runs/${v.id}" style="--variant:${colors[v.id]}"><div class="card-header"><span class="agent-name">${icon(v)}${esc(v.agent)}</span><span class="state-label">${s.n?'Partial':'Planned'}</span></div><p class="card-model">${modelLabel(v)}</p><div class="card-rate"><strong>${pct(s.rate)}</strong><span>native success</span></div><div class="card-stats"><span><b>${s.n?s.passes+' / '+s.n:'—'}</b> passed</span><span class="card-cost" title="${esc(costNote(s))}"><b>${costValue(s)}</b> total cost<small>Estimated USD</small></span></div><div class="completion-track"><span style="width:${s.n/data.tasks.length*100}%"></span></div><div class="card-bottom"><span>${s.n} / ${data.tasks.length} evaluated</span><span>${s.n?'Explore runs':'View planned'} ${arrow}</span></div></a>`;}).join('')}</div><div class="scope-note"><span><i class="status-dot"></i>Success rates and costs cover selected evaluated tasks only.</span><span>Recorded effort · Cache-aware costs · Native verdicts preserved</span></div><div id="comparison-mount">${comparisonTable()}</div>${taskBrowser()}${methodology()}`;
renderRows();
}
function runsPage(variant,params,restoredFilters=null) {
if(!variantFor(variant)){overview();return;}
if(current.type!=='runs'||current.variant!==variant)filters={query:'',family:'',status:'',variant,sort:'catalog'};
current={type:'runs',variant};filters.variant=variant;
if(params.has('family')){filters.family=params.get('family');filters.status='';filters.query='';}
if(restoredFilters)filters={...restoredFilters};
const v=variantFor(variant),s=stats(episodes(variant));
$('#main').innerHTML=`<div class="breadcrumbs"><a href="#overview">Benchmark</a><span>/</span><span>Agent report</span></div>${variantTabs(variant)}<section class="report-hero" style="--variant:${colors[variant]}"><div><div class="eyebrow">${s.n?'Selected results':'Planned evaluation'}</div><h1>${esc(v.agent)} <span class="muted-times">×</span> ${v.model}</h1><p>${esc(v.effort)} effort · One seed per task · ${s.n} evaluated · ${data.tasks.length-s.n} pending</p></div><button class="button secondary" data-export="${variant}">Download results ↓</button></section><div class="report-metrics">${metric(pct(s.rate),'Native success',s.n?`${s.passes} passed / ${s.n} evaluated`:'No recorded episodes')}${metric(costValue(s),'Total cost',costNote(s))}${metric(number(s.responses),'Model responses',s.n?'Provider-reported responses':'Not yet measured')}${metric(elapsed(s.wall==null?null:s.wall/s.n),'Mean wall time',s.n?'Per evaluated episode':'Not yet measured')}</div>${usageTiles(s,false)}${costBreakdown(s)}<div class="family-cards">${data.families.map(f=>{const fs=stats(episodes(variant,f.id));return `<button class="family-card ${filters.family===f.id?'active':''}" data-report-family="${f.id}"><div><strong>${f.name}</strong><span>${pct(fs.rate)}</span></div><div class="mini-track"><span style="width:${(fs.rate||0)*100}%;background:${colors[variant]}"></span></div><small>${fs.n?`${fs.passes} / ${fs.n} passed`:'Pending'} · ${f.tasks-fs.n} untested</small></button>`;}).join('')}</div><p class="scope-note single">${s.n?'Native success is computed from the selected evaluated episodes. Native notes identify simulator source issues. '+usageDefinition():'This combination has no published episodes. The catalog and planned budgets are shown below.'}</p>${taskBrowser(variant)}${methodology()}`;
renderRows();
}
function nativeNote(task,ep) {
const note=data.task_notes[task.key];if(!note)return '';
return `<aside class="native-note" data-native-note><span class="note-symbol">!</span><div><strong>Native source · ${esc(note.kind)}</strong><p>${esc(note.text)}</p><div class="inline-links">${external(note.source,'Pinned source')}${external(note.evidence,'Diagnostic evidence')}${external(ep?.links?.analysis,'Episode record')}${external(ep?.links?.terminal_frame,'Terminal frame')}</div></div></aside>`;
}
function videoPanel(ep,mini=false) {
const m=ep.media||{},speed=m.speed||1,links=ep.links;
return `<div class="video-panel ${mini?'mini-video':''}"><div class="video-caption"><span><i class="status-dot"></i> Native recording</span><span>${m.width||'—'} × ${m.height||'—'} · ${m.duration_s==null?'':m.duration_s.toFixed(2)+' s'}</span></div><video id="episode-video" controls playsinline preload="metadata" style="aspect-ratio:${m.width||2}/${m.height||1}" poster="${esc(links.poster||'')}" src="${esc(links.video)}"></video><div class="video-toolbar"><label>Speed <select id="playback-speed" aria-label="Video playback speed">${[.25,.5,1,2].map(x=>`<option value="${x}" ${x===1?'selected':''}>${x*speed}× simulation</option>`).join('')}</select></label><div><button class="button small" id="final-frame">Final frame</button>${external(links.video,'Video','button small')}${external(links.terminal_frame,'Still','button small')}</div></div><div id="video-error" class="inline-error" hidden>Video unavailable. Open the pinned video directly using the Video link.</div></div>`;
}
function episodeMetrics(ep) {
const u=ep.usage||{},s=stats([ep]);
return `<section class="episode-usage" aria-label="Token usage and cost">${usageTiles(s)}<p class="usage-caption">${usageDefinition()}</p>${costBreakdown(s,true)}<div class="usage-activity"><span><strong>${u.response_count==null&&Number.isFinite(u.known_response_count)?number(u.known_response_count)+'+':number(u.response_count)}</strong> model responses</span><span><strong>${number(ep.call_activity?.model_tool_calls)}</strong> tool calls</span>${external(ep.links.usage,'Original usage record')}</div></section>`;
}
// Align words and punctuation while preserving the exact original whitespace.
function instructionDiff(original, revised) {
const split = text => text.match(/\s+|[\p{L}\p{N}_]+|[^\s]/gu) || [];
const a=split(original), b=split(revised);
const lcs=Array.from({length:a.length+1},()=>new Uint16Array(b.length+1));
for(let i=a.length-1;i>=0;i--)for(let j=b.length-1;j>=0;j--)
lcs[i][j]=a[i]===b[j]?lcs[i+1][j+1]+1:Math.max(lcs[i+1][j],lcs[i][j+1]);
const before=[],after=[];
const append=(list,text,changed)=>{const last=list.at(-1);if(last&&last.changed===changed)last.text+=text;else list.push({text,changed});};
let i=0,j=0;
while(i<a.length||j<b.length){
if(i<a.length&&j<b.length&&a[i]===b[j]){append(before,a[i++],false);append(after,b[j++],false);}
else if(j<b.length&&(i===a.length||lcs[i][j+1]>=lcs[i+1][j]))append(after,b[j++],true);
else append(before,a[i++],true);
}
const html=(parts,kind)=>parts.map(p=>p.changed&&p.text.trim()?`<${kind==='removed'?'del':'mark'} class="instruction-${kind}">${esc(p.text)}</${kind==='removed'?'del':'mark'}>`:esc(p.text)).join('');
return {original:html(before,'removed'),revised:html(after,'added')};
}
function instructionPanel(task,ep) {
const native=ep?ep.native_instruction:task.native_instruction;
const measured=ep?(ep.instruction||(ep.instruction_policy==='original_native'?native:null)):task.catalog_instruction;
const changed=Boolean(native&&measured&&native!==measured);
const diff=changed?instructionDiff(native,measured):null;
const choices=[['measured',ep?'Measured':'Planned',measured],['native','Original native',native],['current','Current catalog',task.catalog_instruction]].filter(([key,,text])=>text&&(key!=='current'||(ep&&text!==measured)));
const currentChanged=native&&task.catalog_instruction&&native!==task.catalog_instruction;
return `<section class="panel instruction-panel"><div class="section-heading"><div><h2>Task instruction</h2><p>${ep?'The exact instruction used for this episode.':'The current catalog instruction for the planned evaluation.'}</p></div><div class="segmented" aria-label="Instruction version">${choices.map(([k,label],i)=>`<button data-instruction="${k}" aria-pressed="${i===0}">${label}</button>`).join('')}</div></div>${choices.map(([k,,text],i)=>{
const comparison=k==='current'?(currentChanged?instructionDiff(native,text):null):diff;
const html=comparison?(k==='native'?comparison.original:comparison.revised):esc(text);
const note=comparison?(k==='native'?'Struck-through text was removed or replaced in the '+(ep?'measured':'planned')+' instruction.':'Highlighted text was added or replaced relative to the original native instruction.'):(native?'No changes from the original native instruction.':'Original native text is unavailable for comparison.');
return `<div class="instruction-version" id="instruction-${k}" ${i?'hidden':''}><p class="instruction-diff-note">${comparison?`<span class="diff-swatch ${k==='native'?'removed':'added'}" aria-hidden="true"></span>`:''}${note}${k==='current'?' This catalog text is not the measured episode instruction.':''}</p><div class="instruction-text">${html}</div></div>`;
}).join('')}<div class="instruction-footer"><span>${ep?`Policy: ${esc(ep.instruction_policy)}`:'Planned seed: '+(task.planned_seed_by_variant?.[current.variant]??task.planned_protocol.seed)}</span><span>${ep?'Recorded':'Planned'} budget: ${number(ep?.measured_protocol?.max_steps??task.planned_protocol.max_control_steps)} steps</span></div></section>`;
}
function overviewDetail(task,ep) {
const initializationBlocked=(data.active_campaign?.initialization_blocked||[]).includes(task.key+'/'+current.variant);
if(!ep)return `<div class="pending-layout"><section class="pending-panel"><span class="pending-orbit">○</span><span class="eyebrow">${initializationBlocked?'Initialization blocked':'Awaiting evaluation'}</span><h2>${initializationBlocked?'Native seed 0 could not initialize':'No recorded episode yet'}</h2><p>${variantName(current.variant)} / ${esc(variantFor(current.variant).effort)}</p><div class="pending-facts">${metric(number(task.planned_protocol.max_control_steps),'Control-step budget')}${metric(String(task.planned_seed_by_variant?.[current.variant]??task.planned_protocol.seed),'Seed')}${metric(task.planned_protocol.control_frequency_hz+' Hz','Control frequency')}</div><p class="subtle">${initializationBlocked?'No model was called. Seed 0 is retained by user decision; this is excluded from task success rates.':'This task is included in the planned catalog. Results and usage will appear after evaluation.'}</p></section>${instructionPanel(task,null)}</div>${nativeNote(task,null)}`;
const m=ep.media||{},u=ep.usage||{},a=ep.analysis||{};
const reason=a.evidence_note||a.failure_reason||a.cause||a.observed||a.visible_completion;
return `<div class="episode-layout"><div>${videoPanel(ep)}<p class="media-note">${m.source_fps?m.source_fps+' FPS source capture · ':''}${m.speed||1}× simulation speed. Session timestamps use wall time and are not synchronized to video frames.</p></div><aside class="result-panel panel"><div class="result-head"><span class="eyebrow">Native verdict</span>${badge(ep)}</div><div class="result-number">${number(ep.steps)}<span> / ${number(ep.measured_protocol?.max_steps)} steps</span></div><div class="completion-track"><span style="width:${Math.min(100,(ep.steps||0)/(ep.measured_protocol?.max_steps||1)*100)}%"></span></div><dl class="facts"><dt>Termination</dt><dd>${esc(ep.verdict?.termination||'—')}</dd><dt>Execution</dt><dd>${esc(ep.execution?.status||'—')}</dd><dt>Evidence</dt><dd>${ep.valid?'Valid':'Invalid'}</dd><dt>Simulation time</dt><dd>${elapsed(ep.simulation_time_s)}</dd><dt>Wall time</dt><dd>${elapsed(ep.wall_time_s)}</dd><dt>Estimated cost</dt><dd>${costValue(stats([ep]))}</dd><dt>Seed</dt><dd>${ep.seed}</dd><dt>Agent version</dt><dd>${ep.kinex_version?'Kinex '+esc(ep.kinex_version):ep.codex_version?'Codex '+esc(ep.codex_version):ep.claude_code_version?'Claude Code '+esc(ep.claude_code_version):esc(variantFor(ep.variant).agent)}</dd></dl><div class="result-links">${external(ep.links.verdict,'Native verdict')}${external(ep.links.provenance,'Source identity')}</div></aside></div>${nativeNote(task,ep)}${episodeMetrics(ep)}<div class="detail-two-column">${instructionPanel(task,ep)}<section class="panel episode-analysis"><div class="section-heading"><div><h2>Episode notes</h2><p>Recorded observations and task-specific interpretation.</p></div></div><div class="analysis-body">${reason?`<p>${esc(reason)}</p>`:`<p>${esc(a.outcome||'The native result and original evidence are retained for this episode.')}</p>`}<details><summary>Read recorded analysis</summary>${Object.entries(a).filter(([k])=>k!=='preflight').map(([k,v])=>`<div class="analysis-entry"><span>${esc(k.replaceAll('_',' '))}</span><p>${esc(typeof v==='string'?v:JSON.stringify(v))}</p></div>`).join('')}</details><div class="inline-links">${external(ep.links.analysis,'Analysis JSON')}${external(ep.links.terminal_frame,'Terminal evidence')}</div></div></section></div><div class="detail-launchers"><a href="${taskRoute(task.key,ep.variant,'session')}"><span class="launcher-icon">≡</span><div><strong>Read the full session</strong><span>${number(ep.session_counts?.visible_events)} events · ${number(ep.session_counts?.observed_images)} observations</span></div>${arrow}</a><a href="${taskRoute(task.key,ep.variant,'artifacts')}"><span class="launcher-icon">⌘</span><div><strong>Inspect agent artifacts</strong><span>${ep.resources?.length||0} saved files · workspace & evidence</span></div>${arrow}</a></div>`;
}
function taskPage(key,variant,section='overview') {
const task=data.tasks.find(t=>t.key===key);if(!task||!variantFor(variant)){notFound();return;}
const ep=epFor(key,variant), validSections=['overview','session','artifacts','evidence','attempts'];
if(!validSections.includes(section)||(!ep&&!['overview','attempts'].includes(section)))section='overview';
current={type:'task',key,variant,section,ep,task};
sessionState={filter:'all',query:'',count:40,view:'timeline'};
const index=data.tasks.indexOf(task),history=data.attempt_history.filter(h=>h.task_key===key&&h.variant===variant);
document.title=`${task.title} · ${variantFor(variant).agent} — Kinex Benchmark`;
$('#main').innerHTML=`<div class="breadcrumbs"><a data-task-return href="${esc(taskListReturn?.hash || '#tasks?task='+encodeURIComponent(key))}">← Back to tasks</a><span>/</span><a href="#runs/${variant}?family=${task.family}">${familyName(task.family)}</a><span>/</span><code>${key}</code></div><div class="task-heading"><div><div class="eyebrow">${familyName(task.family)} · ${key}</div><h1>${esc(task.title)}</h1><p><code>${esc(task.native_id)}</code></p></div><div class="task-arrows">${index?`<a class="button" href="${taskRoute(data.tasks[index-1].key,variant)}" aria-label="Previous task">←</a>`:''}${index<data.tasks.length-1?`<a class="button" href="${taskRoute(data.tasks[index+1].key,variant)}" aria-label="Next task">→</a>`:''}</div></div>${variantTabs(variant,key,section)}<nav class="detail-nav" aria-label="Episode sections">${[['overview','Overview',null],['session','Session',ep?.session_counts?.visible_events],['artifacts','Artifacts',ep?.resources?.length],['evidence','Evidence',null],['attempts','Previous attempts',history.length]].map(([k,l,n])=>!ep&&!['overview','attempts'].includes(k)?`<span class="disabled-tab" aria-disabled="true">${l}</span>`:`<a href="${taskRoute(key,variant,k)}" class="${k===section?'active':''}" ${k===section?'aria-current="page"':''}>${l}${n!=null?`<span>${number(n)}</span>`:''}</a>`).join('')}<span class="detail-model">${variantFor(variant).model} / high</span></nav><div id="detail-content">${section==='overview'?overviewDetail(task,ep):section==='session'?sessionShell(ep):section==='artifacts'?artifactsPanel(ep):section==='evidence'?evidencePanel(task,ep):historyPanel(history)}</div>`;
setupVideo();
if(section==='session')loadSession(ep,epoch);
if(section==='artifacts'&&ep.resources?.length)loadResource(0,epoch);
}
function setupVideo() {
const v=$('#episode-video');if(!v)return;
v.onerror=()=>{if($('#video-error'))$('#video-error').hidden=false;};
}
function sessionShell(ep) {
return `<section class="panel session-panel"><div class="section-heading"><div><h2>Session & observations</h2><p id="session-summary">Loading the original transcript…</p></div><div class="inline-links">${external(ep.links.transcript,'Transcript JSON')}${external(ep.links.native_session,'Native session')}</div></div><div class="session-toolbar"><div class="segmented"><button data-session-view="timeline" aria-pressed="true">Timeline</button><button data-session-view="gallery" aria-pressed="false">Observations</button></div><select id="event-filter" aria-label="Filter session events"><option value="all">All events</option><option value="assistant">Assistant</option><option value="tools">Tool results</option><option value="errors">Tool errors</option><option value="images">With images</option><option value="context">User & context</option></select><input id="event-search" type="search" placeholder="Search text, tool name or event ID" aria-label="Search session"><span id="event-count" aria-live="polite"></span></div><div id="session-events"><div class="loading"><span class="spinner"></span>Loading session…</div></div><div class="session-more"><button id="more-events" class="button secondary" hidden>Load more events</button></div></section>`;
}
async function loadSession(ep,token) {
try {
if(!sessionCache.has(ep.id)) {
const response=await fetch(ep.links.transcript);if(!response.ok)throw Error('HTTP '+response.status);
const raw=await response.json();const entries=Array.isArray(raw)?raw:raw.messages||raw.entries||raw.events||[];
if(!Array.isArray(entries))throw Error('Unsupported transcript format');
sessionCache.set(ep.id,entries);
if(sessionCache.size>6)sessionCache.delete(sessionCache.keys().next().value);
}
if(token!==epoch)return;
const entries=sessionCache.get(ep.id),images=entries.flatMap((m,i)=>(m.parts||[]).filter(p=>p.type==='image'&&p.src).map(p=>({url:new URL(p.src,ep.links.transcript).href,label:`Event ${i+1} · ${m.id||m.role||'Observation'}`})));
$('#session-summary').textContent=`${number(entries.length)} events · ${number(images.length)} observation images · ${entries.filter(e=>e.error).length} tool errors. Event timestamps are in UTC.`;
renderSession();
} catch(error) {if(token!==epoch)return;$('#session-events').innerHTML=`<div class="empty-state">Transcript preview could not load (${esc(error.message)}).<br><button class="button secondary" data-retry-session>Retry</button> ${external(ep.links.transcript,'Open original transcript')}</div>`;}
}
function sessionText(entry) {return [entry.id,entry.role,entry.tool,...(entry.parts||[]).map(p=>p.text||JSON.stringify(p.input||p))].join('\n');}
function renderSession() {
const entries=sessionCache.get(current.ep.id);if(!entries)return;
const {filter,query,count,view}=sessionState, q=query.trim().toLowerCase();
let matches=entries.map((entry,i)=>({entry,index:i})).filter(({entry:e})=>(filter==='all'||filter==='assistant'&&e.role==='assistant'||filter==='tools'&&e.role==='toolResult'||filter==='errors'&&e.error||filter==='images'&&(e.parts||[]).some(p=>p.type==='image')||filter==='context'&&['user','system','developer'].includes(e.role))&&(!q||sessionText(e).toLowerCase().includes(q)));
imageItems=[];
const imagePart=(p,index,entry,cls='observation')=>{
const url=new URL(p.src,current.ep.links.transcript).href;if(!url.startsWith('https://'))return '';
const n=imageItems.length;imageItems.push({url,label:`Event ${index+1} · ${entry.id||entry.role||'Observation'}`});
return `<button class="${cls}" data-image="${n}" aria-label="Open observation from event ${index+1}"><img src="${esc(url)}" alt="Agent observation at event ${index+1}" loading="lazy" decoding="async"><span>Event ${index+1} ${arrow}</span></button>`;
};
if(view==='gallery') {
const images=matches.flatMap(({entry,index})=>(entry.parts||[]).filter(p=>p.type==='image'&&p.src).map(part=>({entry,index,part})));
$('#session-events').innerHTML=images.length?`<div class="observation-gallery">${images.slice(0,count).map(({entry,index,part})=>imagePart(part,index,entry,'gallery-image')).join('')}</div>`:'<div class="empty-state">No observation images match these filters.</div>';
$('#event-count').textContent=`${Math.min(count,images.length)} / ${images.length} images`;$('#more-events').hidden=count>=images.length;
} else {
$('#session-events').innerHTML=matches.length?`<div class="timeline">${matches.slice(0,count).map(({entry:e,index})=>{
const role=e.role==='toolResult'?'Tool result':e.role==='assistant'?'Assistant':({user:'User',system:'System',developer:'Developer'}[e.role]||e.role||e.type||'Event');
const tool=e.tool||(e.parts||[]).find(p=>p.type==='tool')?.name;
return `<article class="session-entry ${e.error?'event-error':''}" id="event-${index}"><div class="event-index">${index+1}</div><div class="event-content"><div class="event-header"><strong>${esc(role)}</strong>${tool?`<code>${esc(tool)}</code>`:''}${e.error?'<span class="error-tag">Error</span>':''}<time>${esc(e.timestamp?e.timestamp.replace('T',' ').replace(/\.\d+Z$/,' UTC'):'')}</time></div><div class="event-parts">${(e.parts||[{type:'text',text:typeof e==='string'?e:JSON.stringify(e,null,2)}]).map(p=>{
if(p.type==='image'&&p.src)return imagePart(p,index,e);
if(p.type==='tool')return `<details class="tool-block"><summary><span>Tool call · <strong>${esc(p.name)}</strong></span><code>${esc((typeof p.input==='string'?p.input:JSON.stringify(p.input)).slice(0,130))}</code></summary><pre>${esc(typeof p.input==='string'?p.input:JSON.stringify(p.input,null,2))}</pre></details>`;
const txt=p.text??p.thinking??JSON.stringify(p,null,2);
if(txt==='')return '<span class="empty-event">No text recorded.</span>';
if(p.type==='thinking'||p.type==='reasoning')return `<details class="tool-block"><summary>Recorded reasoning</summary><pre>${esc(txt)}</pre></details>`;
if(e.role==='toolResult'||txt.length>1800)return `<details class="tool-block ${e.error?'error-block':''}"><summary><span>${e.role==='toolResult'?'Tool output':'Message'}</span><span class="output-excerpt">${esc(txt.slice(0,160))}</span></summary><pre>${esc(txt)}</pre></details>`;
return `<div class="message-text">${esc(txt)}</div>`;
}).join('')}</div><div class="event-id">${esc(e.id||'')}</div></div></article>`;
}).join('')}</div>`:'<div class="empty-state">No events match these filters.</div>';
$('#event-count').textContent=`${Math.min(count,matches.length)} / ${matches.length} events`;$('#more-events').hidden=count>=matches.length;
}
$('#more-events').textContent=view==='gallery'?'Load more observations':'Load next 40 events';
}
function artifactsPanel(ep) {
const resources=ep.resources||[];
return `<section class="panel artifacts-panel"><div class="section-heading"><div><h2>Agent artifacts</h2><p>Final files created during this episode. Their edit history remains in the session.</p></div><div class="inline-links">${external(ep.links.workspace,'Workspace archive')}${external(ep.links.workspace_changes,'Workspace manifest')}</div></div>${resources.length?`<div class="file-browser"><aside class="file-list" aria-label="Artifact files"><div class="file-list-title">${resources.length} saved files</div>${resources.map((r,i)=>`<button data-resource="${i}" class="file-item ${i===0?'active':''}" aria-pressed="${i===0}"><span class="file-extension">${esc(r.name.split('.').at(-1).toUpperCase().slice(0,4))}</span><span>${esc(r.name)}</span></button>`).join('')}</aside><div class="file-preview"><div class="file-toolbar"><code id="resource-name">${esc(resources[0].name)}</code><div><button class="button small" id="copy-resource">Copy</button><a class="button small" id="resource-open" target="_blank" rel="noopener" href="${esc(resources[0].file)}">Open file ↗</a></div></div><p id="resource-kind" class="file-caption"></p><pre id="resource-preview" tabindex="0">Loading file…</pre></div></div>`:'<div class="empty-state">No standalone tool or memo files were saved in this episode. The original session and workspace evidence remain available.</div>'}</section><div class="artifact-shortcuts">${evidenceCard('Execution evidence','Original verdict, protocol and runtime identity.',[['verdict','Native verdict'],['protocol','Protocol'],['provenance','Source identity']],ep)}${evidenceCard('Session & interactions','Original tool calls, observations and edited workspace.',[['native_session','Native session'],['trace','Trace'],['workspace_changes','Workspace changes']],ep)}</div>`;
}
async function fetchText(url) {
if(fileCache.has(url))return fileCache.get(url);
const r=await fetch(url);if(!r.ok)throw Error('HTTP '+r.status);
if(Number(r.headers.get('content-length'))>3e6)throw Error('File exceeds the 3 MB preview limit; use Open file');
const text=await r.text();if(text.length>3e6)throw Error('File exceeds the preview limit; use Open file');
fileCache.set(url,text);if(fileCache.size>20)fileCache.delete(fileCache.keys().next().value);return text;
}
async function loadResource(index,token) {
const r=current.ep?.resources?.[index];if(!r)return;
current.resource=index;$('#resource-name').textContent=r.name;$('#resource-open').href=r.file;$('#resource-kind').textContent=r.kind||'Final workspace snapshot';$('#resource-preview').textContent='Loading file…';
$$('[data-resource]').forEach(b=>{b.classList.toggle('active',+b.dataset.resource===index);b.setAttribute('aria-pressed',+b.dataset.resource===index);});
try{const text=await fetchText(r.file);if(token===epoch&&current.resource===index){$('#resource-preview').textContent=text;current.resourceText=text;}}
catch(e){if(token===epoch&&current.resource===index){$('#resource-preview').textContent='Preview unavailable: '+e.message;current.resourceText=null;}}
}
const evidenceGroups=[
['Native evaluation','Owner verdicts, measured task and control protocol.',[['verdict','Native verdict'],['protocol','Measured protocol'],['native_goal','Task instructions'],['owner_journal','Owner action journal']]],
['Session & workspace','The original agent trajectory and final files.',[['native_session','Native session'],['transcript','Readable transcript'],['trace','Agent trace'],['workspace','Workspace archive'],['workspace_changes','Workspace manifest']]],
['Recording & review','Published media and checks against the recorded evidence.',[['video','Video'],['poster','Poster'],['terminal_frame','Terminal frame'],['recording_manifest','Recording manifest'],['media_check','Media validation'],['analysis','Episode analysis']]],
['Usage & provenance','Provider counters and the actual code used by the episode.',[['usage','Token usage'],['provider_usage','Provider usage ledger'],['provenance','Source identities'],['tool_errors','Tool errors']]]
];
function evidenceCard(title,description,items,ep) {
return `<section class="panel evidence-card"><h3>${title}</h3><p>${description}</p><div>${items.filter(([k])=>ep.links[k]).flatMap(([k,l])=>(Array.isArray(ep.links[k])?ep.links[k]:[ep.links[k]]).map((url,i)=>external(url,l+(Array.isArray(ep.links[k])?' '+(i+1):'')))).join('')}</div></section>`;
}
function evidencePanel(task,ep) {
const used=new Set(evidenceGroups.flatMap(g=>g[2].map(x=>x[0]))),extras=Object.keys(ep.links).filter(k=>!used.has(k));
const source={...data.sources[ep.variant],...(ep.source_dataset_revision?{dataset_revision:ep.source_dataset_revision}:{})};
return `<div class="evidence-intro"><h2>Evidence & provenance</h2><p>Original files pinned to their published Dataset revisions. Native verdicts and measured source identities remain unchanged.</p></div><div class="evidence-grid">${evidenceGroups.map(([title,desc,items])=>evidenceCard(title,desc,items,ep)).join('')}${extras.length?evidenceCard('Additional records','Episode-specific checks and retained references.',extras.map(k=>[k,k.replaceAll('_',' ')]),ep):''}</div><section class="panel provenance-panel"><div class="section-heading"><div><h2>Recorded identity</h2><p>Source and protocol belonging to this episode.</p></div></div><dl class="provenance-list"><dt>Episode</dt><dd><code>${esc(ep.source_episode_id)}</code></dd><dt>Model</dt><dd>${esc(ep.model)} / ${ep.effort}</dd><dt>Dataset</dt><dd>${external('https://huggingface.co/datasets/'+source.dataset,source.dataset)}</dd><dt>Dataset revision</dt><dd><code>${source.dataset_revision}</code></dd>${ep.kinex_revision?`<dt>Kinex revision</dt><dd><code>${esc(ep.kinex_revision)}</code></dd>`:''}${Object.entries(ep.provenance?.sources||{}).map(([k,v])=>`<dt>${esc(k)}</dt><dd><code>${esc(v.revision||'Unrecorded revision')}</code>${v.dirty?' <span class="tag">Local changes recorded</span>':''}</dd>`).join('')}<dt>Recorded budget</dt><dd>${number(ep.measured_protocol?.max_steps)} steps</dd><dt>Current planned budget</dt><dd>${number(task.planned_protocol.max_control_steps)} steps</dd></dl><details class="protocol-json"><summary>Inspect the measured protocol JSON</summary><pre>${esc(JSON.stringify(ep.measured_protocol,null,2))}</pre></details></section><section class="panel evidence-preview-panel"><div class="section-heading"><div><h2>Preview an evidence file</h2><p>Inspect the recorded JSON without leaving this page.</p></div><select id="evidence-select" aria-label="Choose evidence preview"><option value="">Choose a file…</option>${[['verdict','Native verdict'],['protocol','Measured protocol'],['provenance','Source identities'],['usage','Token usage'],['analysis','Episode analysis'],['media_check','Media validation'],['recording_manifest','Recording manifest']].filter(([k])=>ep.links[k]).map(([k,l])=>`<option value="${k}">${l}</option>`).join('')}</select></div><pre id="evidence-preview" hidden tabindex="0"></pre></section>`;
}
function historyPanel(history) {
return `<section class="panel history-panel"><div class="section-heading"><div><h2>Previous attempts</h2><p>Archived attempts are preserved separately and excluded from the current success rate, usage and costs.</p></div><a href="attempt-history.json" target="_blank" class="button secondary">Full archive ↗</a></div>${history.length?history.map(h=>`<article class="history-row"><div class="history-outcome">${badge(h)}<small>${h.valid===false?'Interrupted / invalid':typeof h.success==='boolean'?'Archived result':'Verdict not recorded'}</small></div><div><strong>${esc((h.status||'Archived attempt').replaceAll('_',' '))}</strong><p><code>${esc(h.id||h.job||h.dataset_revision||'Historical record')}</code></p><span class="history-stats">${h.steps!=null?number(h.steps)+' steps · ':''}${h.usage?'Input '+rangeNumber(UsageMetrics.usage(h).inputLow,UsageMetrics.usage(h).inputHigh)+' · cached '+compact(h.usage.cached_input_tokens)+' · ':''}${h.dataset_revision?'Revision '+h.dataset_revision.slice(0,8):''}</span>${h.instruction?`<details><summary>Measured instruction</summary><p>${esc(h.instruction)}</p></details>`:''}<div class="inline-links">${Object.entries(h.links||{}).filter(([k])=>['video','verdict','provenance','transcript','terminal_frame','native_session'].includes(k)).map(([k,u])=>external(u,k.replaceAll('_',' '))).join('')}</div></div></article>`).join(''):'<div class="empty-state">No earlier attempts are listed for this task and agent combination.</div>'}</section>`;
}
function notFound() {$('#main').innerHTML='<div class="empty-state"><h1>Page not found</h1><p><a href="#overview">Return to the benchmark</a></p></div>';}
// Keep each task list's view with its browser history entry.
history.scrollRestoration = 'manual';
let taskListReturn = null;
const navigationHash = () => location.hash || '#overview';
function saveTaskList(row = null) {
if (!['overview', 'runs'].includes(current.type) || !$('#task-rows')) return;
row ||= $$('#task-rows tr[data-task]').find(r => r.getBoundingClientRect().bottom > 100);
const view = {
hash: navigationHash(), filters: {...filters}, x: window.scrollX, y: window.scrollY,
task: row?.dataset.task, offset: row?.getBoundingClientRect().top,
tableX: $('#tasks .table-scroll').scrollLeft,
matrixMetric, expandedModels: $$('[data-model-group][open]').map(el => el.dataset.modelGroup)
};
history.replaceState({...history.state, benchmarkNavigation: {list: view}}, '');
return view;
}
function restoreTaskList(view) {
const row = $$('#task-rows tr[data-task]').find(r => r.dataset.task === view.task);
const y = row && view.offset != null
? window.scrollY + row.getBoundingClientRect().top - view.offset : view.y;
$('#tasks .table-scroll').scrollLeft = view.tableX;
window.scrollTo({left: view.x, top: y, behavior: 'instant'});
}
document.addEventListener('click', event => {
const link = event.target.closest('a[href]');
if (!link || event.button !== 0 || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey ||
link.target === '_blank' || !link.getAttribute('href').startsWith('#')) return;
const view = saveTaskList(link.closest('tr[data-task]'));
if (view && link.getAttribute('href').startsWith('#task/')) taskListReturn = view;
}, true);
window.addEventListener('pagehide', () => saveTaskList());
function route() {
epoch++;$$('video').forEach(v=>{v.pause();v.removeAttribute('src');v.load();});
const [hash,query='']=location.hash.slice(1).split('?'),parts=hash.split('/'),params=new URLSearchParams(query);
const saved = history.state?.benchmarkNavigation;
const returning = current.type === 'task' && taskListReturn &&
(taskListReturn.hash === navigationHash() ||
['#overview', '#tasks'].includes(taskListReturn.hash) && ['#overview', '#tasks'].includes(navigationHash()));
const view = saved?.list?.hash === navigationHash() ? saved.list : returning ? taskListReturn : null;
if (parts[0] === 'task') {
taskListReturn = saved ? saved.origin || null : (['overview', 'runs', 'task'].includes(current.type) ? taskListReturn : null);
history.replaceState({...history.state, benchmarkNavigation: {origin: taskListReturn}}, '');
} else if (view) {
filters = {...view.filters};
matrixMetric = view.matrixMetric;
expandedModels = new Set(view.expandedModels);
}
document.title='Kinex Benchmark — RLE-Bench';
if(parts[0]==='task')taskPage(parts[1]+'/'+parts[2],parts[3],parts[4]||'overview');
else if(parts[0]==='runs')runsPage(parts[1],params,view?.filters);
else overview();
$$('[data-nav]').forEach(a=>a.setAttribute('aria-current',a.dataset.nav===(hash||'overview')?'page':'false'));
if(view && $('#task-rows'))restoreTaskList(view);
else if(hash==='tasks') {
const row = $$('#task-rows tr[data-task]').find(r => r.dataset.task === params.get('task'));
(row || $('#tasks')).scrollIntoView({block: row ? 'center' : 'start', behavior: 'instant'});
}
else if(hash==='methodology'){$('#methodology').open=true;$('#methodology').scrollIntoView();}
else window.scrollTo({left:0,top:0,behavior:'instant'});
if(parts[0]!=='task')taskListReturn=null;
}
function resetFilters() {filters={query:'',family:'',status:'',variant:filters.variant,sort:'catalog'};if(current.type==='runs')runsPage(current.variant,new URLSearchParams());else overview();$('#tasks').scrollIntoView();}
function showToast(text){$('#toast').textContent=text;$('#toast').classList.add('visible');setTimeout(()=>$('#toast').classList.remove('visible'),2200);}
function exportCSV(variant) {
const ids=variant==='all'?data.variants.map(v=>v.id):[variant];
const rows=[['task','environment','title','agent','model','effort','status','native_success','input_tokens_fresh_plus_cache_write','input_tokens_lower','input_tokens_upper','cached_tokens','cached_tokens_lower','cached_tokens_upper','cache_hit_rate','cache_hit_rate_lower','cache_hit_rate_upper','output_tokens','inclusive_input_tokens_original','cache_write_tokens_original','estimated_cost_usd','estimated_cost_lower_usd','estimated_cost_upper_usd','cost_basis','cache_write_basis','cache_write_assumed','steps','episode_id','reasoning_output_tokens','recorded_budget','planned_budget','source_episode_id','inclusive_total_tokens_original']];
for(const t of data.tasks)for(const id of ids){
const e=epFor(t.key,id),v=variantFor(id),u=UsageMetrics.usage(e),c=UsageMetrics.price(e);
rows.push([t.key,familyName(t.family),t.title,v.agent,v.model,v.effort,outcome(e),e?e.success:'',u.input,u.inputLow,u.inputHigh,u.cached,u.cacheLow,u.cacheHigh,u.cacheRate,u.cacheRateLow,u.cacheRateHigh,u.output,u.inclusive,e?.usage?.cache_write_input_tokens,c&&c.lower===c.upper?c.lower:null,c?.lower,c?.upper,c?.basis,c?.cache_write_basis,c?.cache_write_assumed,e?.steps,e?.id,e?.usage?.reasoning_output_tokens,e?.measured_protocol?.max_steps,t.planned_protocol.max_control_steps,e?.source_episode_id,tokens(e)]);
}
const csv=rows.map(row=>row.map(v=>'"'+String(v??'').replaceAll('"','""')+'"').join(',')).join('\r\n');
const url=URL.createObjectURL(new Blob([csv],{type:'text/csv;charset=utf-8'})),a=document.createElement('a');a.href=url;a.download='kinex-benchmark-'+variant+'.csv';a.click();setTimeout(()=>URL.revokeObjectURL(url),1000);
}
function showImage(index) {
if(!imageItems.length)return;imageIndex=(index+imageItems.length)%imageItems.length;const item=imageItems[imageIndex];$('#large-image').src=item.url;$('#image-original').href=item.url;$('#image-caption').textContent=`${imageIndex+1} / ${imageItems.length} · ${item.label}`;if(!$('#image-viewer').open)$('#image-viewer').showModal();
}
$('#main').addEventListener('click',async event=>{
const b=event.target.closest('button,a');if(!b)return;
if(b.dataset.matrix){expandedModels=new Set($$('[data-model-group][open]').map(x=>x.dataset.modelGroup));matrixMetric=b.dataset.matrix;$('#comparison-mount').innerHTML=comparisonTable();}
if(b.dataset.familyLink){filters.family=b.dataset.familyLink;filters.query='';filters.status='';if(location.hash==='#tasks'){overview();$('#tasks').scrollIntoView();}}
if(b.dataset.reportFamily){filters.family=filters.family===b.dataset.reportFamily?'':b.dataset.reportFamily;$('#task-family').value=filters.family;$$('[data-report-family]').forEach(x=>x.classList.toggle('active',x.dataset.reportFamily===filters.family));renderRows();$('#tasks').scrollIntoView({behavior:'smooth'});}
if(b.id==='reset-filters'||b.hasAttribute('data-reset'))resetFilters();
if(b.dataset.export)exportCSV(b.dataset.export);
if(b.dataset.instruction){$$('[data-instruction]').forEach(x=>x.setAttribute('aria-pressed',x===b));$$('.instruction-version').forEach(x=>x.hidden=x.id!=='instruction-'+b.dataset.instruction);}
if(b.id==='final-frame'){const v=$('#episode-video');if(v.readyState<1)await new Promise(r=>v.addEventListener('loadedmetadata',r,{once:true}));v.pause();v.currentTime=Math.max(0,v.duration-.001);}
if(b.dataset.sessionView){sessionState.view=b.dataset.sessionView;sessionState.count=40;$$('[data-session-view]').forEach(x=>x.setAttribute('aria-pressed',x===b));renderSession();}
if(b.id==='more-events'){sessionState.count+=40;renderSession();}
if(b.hasAttribute('data-image'))showImage(+b.dataset.image);
if(b.hasAttribute('data-resource'))loadResource(+b.dataset.resource,epoch);
if(b.hasAttribute('data-retry-session'))loadSession(current.ep,epoch);
if(b.id==='copy-resource'){if(current.resourceText!=null){try{await navigator.clipboard.writeText(current.resourceText);showToast('File copied to clipboard');}catch{showToast('Copy unavailable. Use Open file to select the text.');}}}
});
$('#main').addEventListener('input',event=>{
if(event.target.id==='task-search'){filters.query=event.target.value;renderRows();}
if(event.target.id==='event-search'){sessionState.query=event.target.value;sessionState.count=40;renderSession();}
});
$('#main').addEventListener('change',async event=>{
const {id,value}=event.target;
if(id==='task-family'){filters.family=value;renderRows();$$('[data-report-family]').forEach(x=>x.classList.toggle('active',x.dataset.reportFamily===value));}
if(id==='task-variant'){filters.variant=value;renderRows();}
if(id==='task-status'){filters.status=value;renderRows();}
if(id==='task-sort'){filters.sort=value;renderRows();}
if(id==='playback-speed')$('#episode-video').playbackRate=Number(value);
if(id==='event-filter'){sessionState.filter=value;sessionState.count=40;renderSession();}
if(id==='evidence-select'){
const preview=$('#evidence-preview'),token=epoch;preview.hidden=!value;if(!value)return;preview.textContent='Loading evidence…';
try{const text=await fetchText(current.ep.links[value]);if(token!==epoch||$('#evidence-select').value!==value)return;try{preview.textContent=JSON.stringify(JSON.parse(text),null,2);}catch{preview.textContent=text;}}
catch(e){if(token===epoch)preview.textContent='Preview unavailable: '+e.message;}
}
});
$('#close-viewer').onclick=()=>$('#image-viewer').close();$('#previous-image').onclick=()=>showImage(imageIndex-1);$('#next-image').onclick=()=>showImage(imageIndex+1);
$('#image-viewer').addEventListener('click',event=>{if(event.target===$('#image-viewer'))$('#image-viewer').close();});
$('#image-viewer').addEventListener('keydown',event=>{if(event.key==='ArrowLeft'){event.preventDefault();showImage(imageIndex-1);}if(event.key==='ArrowRight'){event.preventDefault();showImage(imageIndex+1);}});
async function boot(){try{const r=await fetch('data.json');if(!r.ok)throw Error('HTTP '+r.status);data=await r.json();
try{const c=await fetch('cost-data.json');if(c.ok)UsageMetrics.init(await c.json());}catch{UsageMetrics.init({episodes:{},sources:[]});}
for(const task of data.tasks){
if(task.family==='task02'){
const {suite_name,task_id}=task.native_identity;
task.title=`LIBERO-${suite_name.replace(/^libero_/, '')}-${String(task_id+1).padStart(2,'0')}`;
}
}
window.benchmarkData=data;$('#updated').textContent='Results updated '+data.updated_at.slice(0,10);window.addEventListener('hashchange',route);route();}catch(e){$('#main').innerHTML=`<div class="empty-state"><h2>Results could not load</h2><p>${esc(e.message)}</p><a href="data.json">Download the results index</a></div>`;}}
boot();