Spaces:
Running
Running
| import test from 'node:test'; | |
| import assert from 'node:assert/strict'; | |
| import { readFileSync } from 'node:fs'; | |
| import { chartModels, sizeMatches, chartAxis, plotValue, parameterSize } from '../chart.js'; | |
| import { resultMap, valueOf } from '../ranking.js'; | |
| test('parameter bounds are inclusive and undisclosed sizes require explicit inclusion', () => { | |
| const state = { min: 1, max: 4, includeUnknown: false }; | |
| assert.equal(sizeMatches({ parameters_b: 1 }, state), true); | |
| assert.equal(sizeMatches({ parameters_b: 4 }, state), true); | |
| assert.equal(sizeMatches({ parameters_b: 0.8 }, state), false); | |
| assert.equal(sizeMatches({ parameters_b: 9 }, state), false); | |
| for (const parameters_b of [null, undefined, '4', -1, 0, NaN]) { | |
| assert.equal(parameterSize({ parameters_b }), null); | |
| assert.equal(sizeMatches({ parameters_b }, state), false); | |
| assert.equal(sizeMatches({ parameters_b }, { ...state, includeUnknown: true }), true); | |
| } | |
| }); | |
| test('chart excludes ineligible and missing scores, preserves zero and sorts lower metrics correctly', () => { | |
| const models = ['zero', 'good', 'bad', 'failed', 'missing', 'large'].map(id => ({ id, parameters_b: id === 'large' ? 27 : 2 })); | |
| const map = new Map([ | |
| ['zero|bench', { eligible: true, nll: 0 }], | |
| ['good|bench', { eligible: true, nll: 0.3 }], | |
| ['bad|bench', { eligible: true, nll: 2 }], | |
| ['failed|bench', { eligible: false, nll: 0.1 }], | |
| ['large|bench', { eligible: true, nll: 0.01 }], | |
| ]); | |
| assert.deepEqual(chartModels(models, map, { benchmark: 'bench', metric: 'nll', min: 1, max: 4, includeUnknown: false }).map(m => m.id), ['zero', 'good', 'bad']); | |
| }); | |
| test('chart axes retain the official score scale and support all-zero measurements', () => { | |
| assert.equal(plotValue(0.45, 'accuracy'), 45); | |
| assert.equal(plotValue(38.54, 'public_decision_index'), 38.54); | |
| assert.equal(chartAxis([38.54, 62.2], 'public_decision_index'), 100); | |
| assert.equal(chartAxis([0.3, 0.8], 'accuracy'), 100); | |
| for (const values of [[0], [0.01, 0.06], [1.6, 2.22], [19, 80]]) { | |
| assert.ok(chartAxis(values, 'nll') >= Math.max(...values)); | |
| assert.ok(chartAxis(values, 'nll') > 0); | |
| } | |
| }); | |
| test('published chart uses the same audited primary measurements as the table', () => { | |
| const data = JSON.parse(readFileSync(new URL('../data/leaderboard.json', import.meta.url), 'utf8')); | |
| const metadata = JSON.parse(readFileSync(new URL('../data/model-sizes.json', import.meta.url), 'utf8')); | |
| const map = resultMap(data); | |
| for (const model of data.models) { | |
| assert.ok(Object.hasOwn(metadata, model.id), `Missing size provenance: ${model.id}`); | |
| for (const [key, value] of Object.entries(metadata[model.id])) assert.equal(model[key], value); | |
| if (parameterSize(model) !== null) { | |
| assert.match(model.parameter_source, /^https:\/\//); | |
| assert.ok(model.parameter_label.length > 0); | |
| } | |
| } | |
| const state = { benchmark: 'DecisionIndex', metric: 'public_decision_index', min: 0, max: 30, includeUnknown: true }; | |
| const models = chartModels(data.models, map, state); | |
| assert.ok(models.length >= 19); | |
| assert.ok(models.every((model, index) => index === 0 || | |
| valueOf(map.get(`${models[index - 1].id}|DecisionIndex`), state.metric) >= | |
| valueOf(map.get(`${model.id}|DecisionIndex`), state.metric)), 'Scores must be sorted in descending order'); | |
| assert.ok(models.every(m => valueOf(map.get(`${m.id}|DecisionIndex`), state.metric) !== null)); | |
| assert.ok(chartModels(data.models, map, { ...state, max: 1, includeUnknown: false }).every(m => m.parameters_b <= 1)); | |
| assert.equal(data.models.find(m => m.id === 'Clef-Flash').parameters_b, 9); | |
| assert.equal(data.models.find(m => m.id === 'Laya').parameters_b, 0.421); | |
| }); | |