import type { BenchmarkData, DatasetInfo, EvalRecord, LeaderboardEntry, WinRatePayload } from '../types/benchmark'; import type { ResolvedScopeView } from './scopeSelector'; import { compareModelNames } from './modelNames'; function buildWinRates( data: BenchmarkData, entries: LeaderboardEntry[], datasets: DatasetInfo[], ): WinRatePayload { const models = entries.map((entry) => entry.displayName); const includedDatasets = new Set(datasets); const columns = data.datasets.flatMap((dataset, index) => includedDatasets.has(dataset) ? [{ index, higherIsBetter: dataset.higherIsBetter }] : [] ); const evaluations = data.models.map(() => new Map()); const modelIndices = new Map(data.models.map((model, index) => [model.id, index])); for (const record of data.evaluations) { evaluations[record.m].set(record.d, record); } const scores = entries.map((entry) => { const modelIndex = modelIndices.get(entry.modelId); const modelEvaluations = modelIndex === undefined ? undefined : evaluations[modelIndex]; return columns.map(({ index, higherIsBetter }) => { const record = modelEvaluations?.get(index); const score = record?.[scoreField[entry.variant]] ?? null; return score === null || higherIsBetter ? score : -score; }); }); const matrix = models.map(() => Array(models.length).fill(null)); const counts: WinRatePayload['counts'] = {}; for (let i = 0; i < models.length; i++) { for (let j = i + 1; j < models.length; j++) { let scoreWins = 0; let scoreLosses = 0; let completionWins = 0; let completionLosses = 0; let ties = 0; for (let k = 0; k < columns.length; k++) { const left = scores[i][k]; const right = scores[j][k]; if (left === right) ties++; else if (left === null) completionLosses++; else if (right === null) completionWins++; else if (left > right) scoreWins++; else scoreLosses++; } const wins = scoreWins + completionWins; const losses = scoreLosses + completionLosses; matrix[i][j] = 100 * (wins + ties / 2) / columns.length; matrix[j][i] = 100 * (losses + ties / 2) / columns.length; counts[`${models[i]}__${models[j]}`] = [wins, losses, ties, scoreWins, completionWins]; counts[`${models[j]}__${models[i]}`] = [losses, wins, ties, scoreLosses, completionLosses]; } } return { models, matrix, counts }; } const scoreField = { default: 'defScore', tuned: 'tunScore', selected: 'selScore', } as const; export function getWinRateView(data: BenchmarkData, view: ResolvedScopeView): WinRatePayload | null { const { datasets, visibleEntries, winRate } = view; if (datasets.length === 0 || visibleEntries.length < 2) return null; const raw = winRate ?? buildWinRates(data, visibleEntries, datasets); const visible = new Set(visibleEntries.map((entry) => entry.displayName)); const indices = raw.models.flatMap((name, index) => visible.has(name) ? [index] : []); const ordered = indices .map((index) => { const rates = indices.flatMap((opponent) => { const rate = raw.matrix[index][opponent]; return opponent !== index && rate !== null ? [rate] : []; }); const mean = rates.length > 0 ? rates.reduce((sum, rate) => sum + rate, 0) / rates.length : -Infinity; return { index, mean }; }) .sort((a, b) => b.mean - a.mean || compareModelNames(raw.models[a.index], raw.models[b.index]) ) .map(({ index }) => index); return { models: ordered.map((i) => raw.models[i]), matrix: ordered.map((i) => ordered.map((j) => raw.matrix[i][j])), counts: raw.counts, }; }