File size: 3,742 Bytes
d189f56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
import type { BenchmarkData, DatasetInfo, EvalRecord, LeaderboardEntry, WinRatePayload } from '../types/benchmark';
import type { ResolvedScopeView } from './scopeSelector';
import { compareModelNames } from './modelNames';

function buildWinRates(
  data: BenchmarkData,
  entries: LeaderboardEntry[],
  datasets: DatasetInfo[],
): WinRatePayload {
  const models = entries.map((entry) => entry.displayName);
  const includedDatasets = new Set(datasets);
  const columns = data.datasets.flatMap((dataset, index) =>
    includedDatasets.has(dataset) ? [{ index, higherIsBetter: dataset.higherIsBetter }] : []
  );
  const evaluations = data.models.map(() => new Map<number, EvalRecord>());
  const modelIndices = new Map(data.models.map((model, index) => [model.id, index]));
  for (const record of data.evaluations) {
    evaluations[record.m].set(record.d, record);
  }

  const scores = entries.map((entry) => {
    const modelIndex = modelIndices.get(entry.modelId);
    const modelEvaluations = modelIndex === undefined ? undefined : evaluations[modelIndex];
    return columns.map(({ index, higherIsBetter }) => {
      const record = modelEvaluations?.get(index);
      const score = record?.[scoreField[entry.variant]] ?? null;
      return score === null || higherIsBetter ? score : -score;
    });
  });

  const matrix = models.map(() => Array<number | null>(models.length).fill(null));
  const counts: WinRatePayload['counts'] = {};
  for (let i = 0; i < models.length; i++) {
    for (let j = i + 1; j < models.length; j++) {
      let scoreWins = 0;
      let scoreLosses = 0;
      let completionWins = 0;
      let completionLosses = 0;
      let ties = 0;
      for (let k = 0; k < columns.length; k++) {
        const left = scores[i][k];
        const right = scores[j][k];
        if (left === right) ties++;
        else if (left === null) completionLosses++;
        else if (right === null) completionWins++;
        else if (left > right) scoreWins++;
        else scoreLosses++;
      }
      const wins = scoreWins + completionWins;
      const losses = scoreLosses + completionLosses;
      matrix[i][j] = 100 * (wins + ties / 2) / columns.length;
      matrix[j][i] = 100 * (losses + ties / 2) / columns.length;
      counts[`${models[i]}__${models[j]}`] = [wins, losses, ties, scoreWins, completionWins];
      counts[`${models[j]}__${models[i]}`] = [losses, wins, ties, scoreLosses, completionLosses];
    }
  }
  return { models, matrix, counts };
}

const scoreField = {
  default: 'defScore',
  tuned: 'tunScore',
  selected: 'selScore',
} as const;

export function getWinRateView(data: BenchmarkData, view: ResolvedScopeView): WinRatePayload | null {
  const { datasets, visibleEntries, winRate } = view;
  if (datasets.length === 0 || visibleEntries.length < 2) return null;

  const raw = winRate ?? buildWinRates(data, visibleEntries, datasets);
  const visible = new Set(visibleEntries.map((entry) => entry.displayName));
  const indices = raw.models.flatMap((name, index) => visible.has(name) ? [index] : []);

  const ordered = indices
    .map((index) => {
      const rates = indices.flatMap((opponent) => {
        const rate = raw.matrix[index][opponent];
        return opponent !== index && rate !== null ? [rate] : [];
      });
      const mean = rates.length > 0
        ? rates.reduce((sum, rate) => sum + rate, 0) / rates.length
        : -Infinity;
      return { index, mean };
    })
    .sort((a, b) =>
      b.mean - a.mean || compareModelNames(raw.models[a.index], raw.models[b.index])
    )
    .map(({ index }) => index);

  return {
    models: ordered.map((i) => raw.models[i]),
    matrix: ordered.map((i) => ordered.map((j) => raw.matrix[i][j])),
    counts: raw.counts,
  };
}