Sizes use published parameter counts or backbone sizes, including the base model for adapters. Click a size for its source; click a bar for the measurement. Partial valid coverage
${row.primary ? 'The latest complete measurement selected by the toolkit for this dataset and protocol.' : 'This is a retained run, not the measurement currently selected for the main ranking.'} ${row.smoke ? 'This run uses a smoke-test sample.' : ''} ${row.origin?.startsWith('imported') ? 'This measurement was imported from retained upstream evaluation outputs.' : ''}
${stats}
Metric
Value
Direction
${metricRows}
${types ? `
By question type
Type
Questions
Accuracy
Coverage
${types}
` : ''}
Valid coverage: ${formatValue(row.coverage, 'coverage')}. Accuracy uses ${typeof overall.hard_questions === 'number' ? number(overall.hard_questions) + ' hard-reference questions' : 'the benchmark’s hard-reference denominator'}; probability metrics use valid probability references. Unsupported answers remain in the accuracy denominator.${isDecisionIndex(row.benchmark) ? ' Public DI excludes private components and is not the full official index.' : ''}
`;
}).join('')}`).join('');
openDialog('Side by side', 'MODEL COMPARISON', 'Accuracy on identical benchmark snapshots · higher is better', `
${legend}
Bars show accuracy within each benchmark. Missing or unsupported measurements are left empty. Coverage is included in each bar’s tooltip; open a model profile for complete metrics.
${sections}
Decision Index bars show accuracy, not the public-only composite index. No average across benchmarks is calculated.
VisualDecisionBench Image and Video are views of one family. The combined TSV is available in the dataset repository and is not counted as an additional suite. Public DI is an official public-only score, with private components excluded.
${external('https://huggingface.co/datasets/Valen-Team/JevalKit_Datasets', `Explore the datasets ${icon('diagonal', 13)}`, 'button primary')}${external('https://github.com/Valen-Team/JevalKit/blob/main/docs/benchmarks.md', `Protocols and scoring ${icon('diagonal', 13)}`, 'button')}
`);
}
function showMethodology() {
openDialog('Read the result, not just the rank.', 'EVALUATION NOTES', 'Native inference. Pinned benchmarks. Explicit coverage.', `
One ranking per benchmark
Ranks apply to the selected benchmark and metric only. Clicking a column changes the ranking. The table uses the latest complete measurement for each pinned dataset and protocol, rather than picking the highest repeat. No cross-benchmark aggregate is invented.
What qualifies for a score?
Only complete, non-smoke measurements with conformant inputs, no runtime failures and at least one valid answer are eligible. Unsupported questions stay in the hard-label accuracy denominator. The archive retains earlier errors, capability exclusions, smoke runs and repeated measurements.
Accuracy and calibration answer different questions
Accuracy uses hard references. Brier, NLL and ECE use valid references under the benchmark’s scoring protocol; inspect valid coverage alongside these metrics. Some suites include soft references that contribute probability metrics but not hard-label accuracy.
Coverage matters
A gold dot marks partial valid coverage. The coverage column refers to the benchmark selected under “Rank within.” “Full coverage only” filters that selected benchmark. A dash means no measurement. “Unsupported” indicates a complete run with no supported inputs and does not receive a rank.
Decision Index is public-only
The separate Decision Index tab can show its public-only official score. It excludes private components and should not be read as the complete official Decision Index.
Latency needs matching conditions
Latency is mean seconds per shared-state request. Hardware appears in each measurement. Author runtimes and serving environments differ, so latency alone is not a controlled hardware comparison.
A reproducible snapshot
This Space shows a static snapshot from ${prettyDate(data.snapshot.last_run_at)} UTC, with ${data.snapshot.counts.eligible_scores} eligible primary measurements. ${data.snapshot.counts.evaluated_families} of ${data.snapshot.counts.model_families} registered families have measurements; OpenAI-Decisions is still pending. Public data includes selection flags, dataset and prediction hashes, and retained run summaries.
`).join('');
openDialog('Every run has a place.', 'THE RUN ARCHIVE', `${data.results.length} retained evaluation artifacts · ${data.snapshot.counts.eligible_scores} current eligible scores`, `
Historical, incomplete and smoke measurements remain visible here. Only current eligible full runs enter the main ranking.
Model / selection
Benchmark
Run status
Errors
${rows}
`);
}
function changeGroup(group) {
state.group = group; state.reversed = false;
state.rankBy = group === 'text' ? 'JevAltBench' : GROUPS[group].names[0];
if (group === 'index') state.metric = 'public_decision_index';
else if (state.metric === 'public_decision_index') state.metric = 'accuracy';
renderControls(); renderRows();
}
function addSelection(id) {
if (state.selected.includes(id)) state.selected = state.selected.filter(name => name !== id);
else if (state.selected.length >= 3) { toast('Choose up to 3 models for comparison.'); return false; }
else state.selected.push(id);
renderRows(); return true;
}
function resetFilters() {
state.query = ''; state.family = 'all'; state.fullCoverage = false;
renderControls(); renderRows();
}
document.addEventListener('click', event => {
const button = event.target.closest('button');
if (!button || !data) return;
if (button.dataset.run) { showRun(button.dataset.run); return; }
if (button.dataset.model) { showModel(button.dataset.model); return; }
if (button.dataset.group) { changeGroup(button.dataset.group); return; }
if (button.dataset.rank) {
if (state.rankBy === button.dataset.rank) state.reversed = !state.reversed;
else { state.rankBy = button.dataset.rank; state.reversed = false; }
renderControls(); renderRows(); return;
}
if (button.dataset.add) { if (addSelection(button.dataset.add)) toast(state.selected.includes(button.dataset.add) ? 'Added to comparison.' : 'Removed from comparison.'); return; }
if (button.dataset.downloadRun) {
const row = data.results.find(r => r.id === button.dataset.downloadRun);
download(`JevalKit-${row.model}-${row.benchmark}.json`, JSON.stringify({ source: data.snapshot.source_url, ...row }, null, 2)); return;
}
const action = button.dataset.action;
if (button.dataset.sizePreset !== undefined) {
const preset = sizePresets[Number(button.dataset.sizePreset)];
chartState.min = preset.min; chartState.max = preset.max(); chartState.includeUnknown = !!preset.unknown;
renderChart(); return;
}
if (action === 'chart-expand') { chartState.expanded = !chartState.expanded; renderChart(); }
if (action === 'close') document.querySelector('#detail-dialog').close();
if (action === 'methodology') showMethodology();
if (action === 'benchmarks') showBenchmarks();
if (action === 'archive') showArchive();
if (action === 'compare') showComparison();
if (action === 'clear-compare') { state.selected = []; renderRows(); }
if (action === 'reset') resetFilters();
if (action === 'csv') download(`JevalKit-${state.group}-${state.metric}.csv`, csvFor(visibleModels(), map, tableNames(), state.metric), 'text/csv;charset=utf-8');
if (action === 'json') download('JevalKit-leaderboard.json', JSON.stringify(data, null, 2));
});
document.addEventListener('input', event => {
if (event.target.id === 'model-search') { state.query = event.target.value; renderRows(); }
if (!data) return;
if (event.target.id === 'size-min' || event.target.id === 'size-max') {
const bound = event.target.id === 'size-min' ? 'min' : 'max';
chartState[bound] = Number(event.target.value);
if (chartState.min > chartState.max) chartState[bound === 'min' ? 'max' : 'min'] = chartState[bound];
chartState.includeUnknown = chartState.min === 0 && chartState.max === sizeLimit;
renderChart();
}
});
document.addEventListener('change', event => {
const target = event.target;
if (!data) return;
if (target.id === 'chart-benchmark') {
chartState.benchmark = target.value;
if (isDecisionIndex(target.value)) chartState.metric = 'public_decision_index';
else if (chartState.metric === 'public_decision_index') chartState.metric = 'accuracy';
renderChart(); return;
}
if (target.id === 'chart-metric') { chartState.metric = target.value; renderChart(); return; }
if (target.id === 'size-unknown') { chartState.includeUnknown = target.checked; renderChart(); return; }
if (target.dataset.select) { if (!addSelection(target.dataset.select)) target.checked = false; return; }
if (target.id === 'metric-select') { state.metric = target.value; state.reversed = false; }
if (target.id === 'rank-select') { state.rankBy = target.value; state.reversed = false; }
if (target.id === 'family-select') state.family = target.value;
if (target.id === 'full-coverage') state.fullCoverage = target.checked;
if (['metric-select', 'rank-select', 'family-select', 'full-coverage'].includes(target.id)) renderRows();
});
const dialog = document.querySelector('#detail-dialog');
matchMedia('(max-width: 680px)').addEventListener('change', () => { if (data) renderRows(); });
dialog.addEventListener('close', () => { document.body.style.overflow = ''; });
dialog.addEventListener('click', event => { if (event.target === dialog) { const rect = dialog.getBoundingClientRect(); if (event.clientX < rect.left || event.clientX > rect.right || event.clientY < rect.top || event.clientY > rect.bottom) dialog.close(); } });
async function boot() {
try {
const response = await fetch('./data/leaderboard.json');
if (!response.ok) throw new Error(`The data snapshot returned HTTP ${response.status}.`);
data = await response.json();
if (data.schema_version !== 1 || !data.results.length || !data.models.length) throw new Error('The data snapshot has an unsupported format.');
map = resultMap(data); benchmarks = new Map(data.benchmarks.map(b => [b.name, b]));
sizeLimit = Math.max(30, Math.ceil(Math.max(...data.models.map(model => parameterSize(model) || 0)) / 10) * 10);
chartState.max = sizeLimit;
const params = new URLSearchParams(location.search);
if (GROUPS[params.get('view')]) state.group = params.get('view');
if (METRICS[params.get('metric')]) state.metric = params.get('metric');
else if (state.group === 'index') state.metric = 'public_decision_index';
if (activeNames().includes(params.get('rank'))) state.rankBy = params.get('rank');
else if (!activeNames().includes(state.rankBy)) state.rankBy = activeNames()[0];
if (state.metric === 'public_decision_index' && !['index', 'all'].includes(state.group)) state.metric = 'accuracy';
state.query = params.get('q') || '';
renderPage();
} catch (error) {
document.querySelector('#app').innerHTML = `JevalKit${preferenceControls()}
The results couldn’t load.
${esc(error.message)} Please reload, or open the source snapshot on Hugging Face.
JevalKit