Spaces:
Running
Running
File size: 3,742 Bytes
d189f56 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | import type { BenchmarkData, DatasetInfo, EvalRecord, LeaderboardEntry, WinRatePayload } from '../types/benchmark';
import type { ResolvedScopeView } from './scopeSelector';
import { compareModelNames } from './modelNames';
function buildWinRates(
data: BenchmarkData,
entries: LeaderboardEntry[],
datasets: DatasetInfo[],
): WinRatePayload {
const models = entries.map((entry) => entry.displayName);
const includedDatasets = new Set(datasets);
const columns = data.datasets.flatMap((dataset, index) =>
includedDatasets.has(dataset) ? [{ index, higherIsBetter: dataset.higherIsBetter }] : []
);
const evaluations = data.models.map(() => new Map<number, EvalRecord>());
const modelIndices = new Map(data.models.map((model, index) => [model.id, index]));
for (const record of data.evaluations) {
evaluations[record.m].set(record.d, record);
}
const scores = entries.map((entry) => {
const modelIndex = modelIndices.get(entry.modelId);
const modelEvaluations = modelIndex === undefined ? undefined : evaluations[modelIndex];
return columns.map(({ index, higherIsBetter }) => {
const record = modelEvaluations?.get(index);
const score = record?.[scoreField[entry.variant]] ?? null;
return score === null || higherIsBetter ? score : -score;
});
});
const matrix = models.map(() => Array<number | null>(models.length).fill(null));
const counts: WinRatePayload['counts'] = {};
for (let i = 0; i < models.length; i++) {
for (let j = i + 1; j < models.length; j++) {
let scoreWins = 0;
let scoreLosses = 0;
let completionWins = 0;
let completionLosses = 0;
let ties = 0;
for (let k = 0; k < columns.length; k++) {
const left = scores[i][k];
const right = scores[j][k];
if (left === right) ties++;
else if (left === null) completionLosses++;
else if (right === null) completionWins++;
else if (left > right) scoreWins++;
else scoreLosses++;
}
const wins = scoreWins + completionWins;
const losses = scoreLosses + completionLosses;
matrix[i][j] = 100 * (wins + ties / 2) / columns.length;
matrix[j][i] = 100 * (losses + ties / 2) / columns.length;
counts[`${models[i]}__${models[j]}`] = [wins, losses, ties, scoreWins, completionWins];
counts[`${models[j]}__${models[i]}`] = [losses, wins, ties, scoreLosses, completionLosses];
}
}
return { models, matrix, counts };
}
const scoreField = {
default: 'defScore',
tuned: 'tunScore',
selected: 'selScore',
} as const;
export function getWinRateView(data: BenchmarkData, view: ResolvedScopeView): WinRatePayload | null {
const { datasets, visibleEntries, winRate } = view;
if (datasets.length === 0 || visibleEntries.length < 2) return null;
const raw = winRate ?? buildWinRates(data, visibleEntries, datasets);
const visible = new Set(visibleEntries.map((entry) => entry.displayName));
const indices = raw.models.flatMap((name, index) => visible.has(name) ? [index] : []);
const ordered = indices
.map((index) => {
const rates = indices.flatMap((opponent) => {
const rate = raw.matrix[index][opponent];
return opponent !== index && rate !== null ? [rate] : [];
});
const mean = rates.length > 0
? rates.reduce((sum, rate) => sum + rate, 0) / rates.length
: -Infinity;
return { index, mean };
})
.sort((a, b) =>
b.mean - a.mean || compareModelNames(raw.models[a.index], raw.models[b.index])
)
.map(({ index }) => index);
return {
models: ordered.map((i) => raw.models[i]),
matrix: ordered.map((i) => ordered.map((j) => raw.matrix[i][j])),
counts: raw.counts,
};
}
|