import test from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { chartModels, sizeMatches, chartAxis, plotValue, parameterSize } from '../chart.js'; import { resultMap, valueOf } from '../ranking.js'; test('parameter bounds are inclusive and undisclosed sizes require explicit inclusion', () => { const state = { min: 1, max: 4, includeUnknown: false }; assert.equal(sizeMatches({ parameters_b: 1 }, state), true); assert.equal(sizeMatches({ parameters_b: 4 }, state), true); assert.equal(sizeMatches({ parameters_b: 0.8 }, state), false); assert.equal(sizeMatches({ parameters_b: 9 }, state), false); for (const parameters_b of [null, undefined, '4', -1, 0, NaN]) { assert.equal(parameterSize({ parameters_b }), null); assert.equal(sizeMatches({ parameters_b }, state), false); assert.equal(sizeMatches({ parameters_b }, { ...state, includeUnknown: true }), true); } }); test('chart excludes ineligible and missing scores, preserves zero and sorts lower metrics correctly', () => { const models = ['zero', 'good', 'bad', 'failed', 'missing', 'large'].map(id => ({ id, parameters_b: id === 'large' ? 27 : 2 })); const map = new Map([ ['zero|bench', { eligible: true, nll: 0 }], ['good|bench', { eligible: true, nll: 0.3 }], ['bad|bench', { eligible: true, nll: 2 }], ['failed|bench', { eligible: false, nll: 0.1 }], ['large|bench', { eligible: true, nll: 0.01 }], ]); assert.deepEqual(chartModels(models, map, { benchmark: 'bench', metric: 'nll', min: 1, max: 4, includeUnknown: false }).map(m => m.id), ['zero', 'good', 'bad']); }); test('chart axes retain the official score scale and support all-zero measurements', () => { assert.equal(plotValue(0.45, 'accuracy'), 45); assert.equal(plotValue(38.54, 'public_decision_index'), 38.54); assert.equal(chartAxis([38.54, 62.2], 'public_decision_index'), 100); assert.equal(chartAxis([0.3, 0.8], 'accuracy'), 100); for (const values of [[0], [0.01, 0.06], [1.6, 2.22], [19, 80]]) { assert.ok(chartAxis(values, 'nll') >= Math.max(...values)); assert.ok(chartAxis(values, 'nll') > 0); } }); test('published chart uses the same audited primary measurements as the table', () => { const data = JSON.parse(readFileSync(new URL('../data/leaderboard.json', import.meta.url), 'utf8')); const metadata = JSON.parse(readFileSync(new URL('../data/model-sizes.json', import.meta.url), 'utf8')); const map = resultMap(data); for (const model of data.models) { assert.ok(Object.hasOwn(metadata, model.id), `Missing size provenance: ${model.id}`); for (const [key, value] of Object.entries(metadata[model.id])) assert.equal(model[key], value); if (parameterSize(model) !== null) { assert.match(model.parameter_source, /^https:\/\//); assert.ok(model.parameter_label.length > 0); } } const state = { benchmark: 'DecisionIndex', metric: 'public_decision_index', min: 0, max: 30, includeUnknown: true }; const models = chartModels(data.models, map, state); assert.ok(models.length >= 19); assert.ok(models.every((model, index) => index === 0 || valueOf(map.get(`${models[index - 1].id}|DecisionIndex`), state.metric) >= valueOf(map.get(`${model.id}|DecisionIndex`), state.metric)), 'Scores must be sorted in descending order'); assert.ok(models.every(m => valueOf(map.get(`${m.id}|DecisionIndex`), state.metric) !== null)); assert.ok(chartModels(data.models, map, { ...state, max: 1, includeUnknown: false }).every(m => m.parameters_b <= 1)); assert.equal(data.models.find(m => m.id === 'Clef-Flash').parameters_b, 9); assert.equal(data.models.find(m => m.id === 'Laya').parameters_b, 0.421); });