Spaces:
Running
Running
Download src/lib/winRate.ts from TabBioMed/leaderboard: direct link, hf CLI and curl.
- Browser
- Download file 3.74 kB
-
https://huggingface.co/spaces/TabBioMed/leaderboard/resolve/main/src/lib/winRate.ts
- Command line
-
hf download hf://spaces/TabBioMed/leaderboard/src/lib/winRate.ts
-
curl -L -o winRate.ts https://huggingface.co/spaces/TabBioMed/leaderboard/resolve/main/src/lib/winRate.ts
3.74 kB
| import type { BenchmarkData, DatasetInfo, EvalRecord, LeaderboardEntry, WinRatePayload } from '../types/benchmark'; | |
| import type { ResolvedScopeView } from './scopeSelector'; | |
| import { compareModelNames } from './modelNames'; | |
| function buildWinRates( | |
| data: BenchmarkData, | |
| entries: LeaderboardEntry[], | |
| datasets: DatasetInfo[], | |
| ): WinRatePayload { | |
| const models = entries.map((entry) => entry.displayName); | |
| const includedDatasets = new Set(datasets); | |
| const columns = data.datasets.flatMap((dataset, index) => | |
| includedDatasets.has(dataset) ? [{ index, higherIsBetter: dataset.higherIsBetter }] : [] | |
| ); | |
| const evaluations = data.models.map(() => new Map<number, EvalRecord>()); | |
| const modelIndices = new Map(data.models.map((model, index) => [model.id, index])); | |
| for (const record of data.evaluations) { | |
| evaluations[record.m].set(record.d, record); | |
| } | |
| const scores = entries.map((entry) => { | |
| const modelIndex = modelIndices.get(entry.modelId); | |
| const modelEvaluations = modelIndex === undefined ? undefined : evaluations[modelIndex]; | |
| return columns.map(({ index, higherIsBetter }) => { | |
| const record = modelEvaluations?.get(index); | |
| const score = record?.[scoreField[entry.variant]] ?? null; | |
| return score === null || higherIsBetter ? score : -score; | |
| }); | |
| }); | |
| const matrix = models.map(() => Array<number | null>(models.length).fill(null)); | |
| const counts: WinRatePayload['counts'] = {}; | |
| for (let i = 0; i < models.length; i++) { | |
| for (let j = i + 1; j < models.length; j++) { | |
| let scoreWins = 0; | |
| let scoreLosses = 0; | |
| let completionWins = 0; | |
| let completionLosses = 0; | |
| let ties = 0; | |
| for (let k = 0; k < columns.length; k++) { | |
| const left = scores[i][k]; | |
| const right = scores[j][k]; | |
| if (left === right) ties++; | |
| else if (left === null) completionLosses++; | |
| else if (right === null) completionWins++; | |
| else if (left > right) scoreWins++; | |
| else scoreLosses++; | |
| } | |
| const wins = scoreWins + completionWins; | |
| const losses = scoreLosses + completionLosses; | |
| matrix[i][j] = 100 * (wins + ties / 2) / columns.length; | |
| matrix[j][i] = 100 * (losses + ties / 2) / columns.length; | |
| counts[`${models[i]}__${models[j]}`] = [wins, losses, ties, scoreWins, completionWins]; | |
| counts[`${models[j]}__${models[i]}`] = [losses, wins, ties, scoreLosses, completionLosses]; | |
| } | |
| } | |
| return { models, matrix, counts }; | |
| } | |
| const scoreField = { | |
| default: 'defScore', | |
| tuned: 'tunScore', | |
| selected: 'selScore', | |
| } as const; | |
| export function getWinRateView(data: BenchmarkData, view: ResolvedScopeView): WinRatePayload | null { | |
| const { datasets, visibleEntries, winRate } = view; | |
| if (datasets.length === 0 || visibleEntries.length < 2) return null; | |
| const raw = winRate ?? buildWinRates(data, visibleEntries, datasets); | |
| const visible = new Set(visibleEntries.map((entry) => entry.displayName)); | |
| const indices = raw.models.flatMap((name, index) => visible.has(name) ? [index] : []); | |
| const ordered = indices | |
| .map((index) => { | |
| const rates = indices.flatMap((opponent) => { | |
| const rate = raw.matrix[index][opponent]; | |
| return opponent !== index && rate !== null ? [rate] : []; | |
| }); | |
| const mean = rates.length > 0 | |
| ? rates.reduce((sum, rate) => sum + rate, 0) / rates.length | |
| : -Infinity; | |
| return { index, mean }; | |
| }) | |
| .sort((a, b) => | |
| b.mean - a.mean || compareModelNames(raw.models[a.index], raw.models[b.index]) | |
| ) | |
| .map(({ index }) => index); | |
| return { | |
| models: ordered.map((i) => raw.models[i]), | |
| matrix: ordered.map((i) => ordered.map((j) => raw.matrix[i][j])), | |
| counts: raw.counts, | |
| }; | |
| } | |