leaderboard / src /types /benchmark.ts
dbonet's picture paumateo's picture
Publish TabBioMed leaderboard
d189f56
Raw History Blame Contribute Delete
5.18 kB
export type ModelFamily =
| 'Foundation models'
| 'Tree-based'
| 'Neural networks'
| 'Baselines';
export type TaskType = 'binary' | 'classification' | 'regression' | 'multiregression';
export type PrimaryMetric = 'roc_auc' | 'log_loss' | 'mse';
export type DomainSlug =
| 'ehr'
| 'drug'
| 'genom'
| 'mtbls'
| 'ptomics'
| 'sc'
| 'sb'
| 'trans';
export type QuadrantId = 'Q1' | 'Q2' | 'Q3' | 'Q4';
export interface DomainInfo {
slug: DomainSlug;
name: string;
fullName: string;
nDatasets: number;
nWithMissing: number;
samplesMin: number;
samplesMax: number;
featuresMin: number;
featuresMax: number;
taskCounts: Partial<Record<TaskType, number>>;
}
export interface ModelInfo {
id: string;
family: ModelFamily;
hpoEnabled: boolean;
classificationOnly: boolean;
paretoColor: string;
failureSummary: {
defaultFailed: number;
tunedFailed: number;
usable: number;
totalApplicable: number;
causes: Record<string, number>;
};
hpoComparison: {
pairedCount: number;
tunedBetter: number;
tie: number;
defaultBetter: number;
};
}
export interface DatasetInfo {
id: string;
domainSlug: DomainSlug;
domain: string;
clinicalSubpanel: 'ehr_up_to_2000' | 'ehr_over_2000' | null;
observationGroup: string;
sourceFamily: string;
task: TaskType;
primaryMetric: PrimaryMetric;
higherIsBetter: boolean;
numSamples: number;
numFeatures: number;
numCategorical: number;
catFraction: number;
numClasses: number | null;
imbalanceRatio: number | null;
nFolds: number;
splitStrategy: 'fixed' | 'cv';
quadrant: QuadrantId;
quadrantLabel: string;
missingCellFraction: number;
missingFeatureCount: number;
hasMissing: boolean;
hasZeroCodedMissingNote: boolean;
description: string;
sourceShort: string;
sourceUrl?: string;
license: string | null;
licenseUrl: string | null;
}
export interface EvalRecord {
m: number;
d: number;
defScore: number | null;
tunScore: number | null;
selScore: number | null;
selVar: 'tuned' | 'default' | 'failed';
src: 'tuned_test' | 'default_only' | 'default_fallback' | 'failed';
defTrain1k: number | null;
defInfer1k: number | null;
tunTrain1k: number | null;
tunInfer1k: number | null;
selFit1k: number | null;
failCause: string | null;
}
export interface LeaderboardEntry {
key: string;
displayName: string;
modelId: string;
variant: 'selected' | 'tuned' | 'default';
family: ModelFamily;
hpoEnabled: boolean;
classificationOnly: boolean;
paretoColor: string;
rank: number;
elo: number;
eloLo: number | null;
eloHi: number | null;
tunedMinusDefaultElo: number | null;
meanWinRate: number;
normScoreMean: number;
normScoreMedian: number;
meanRank: number;
rankMin: number;
rankMax: number;
improvabilityPct: number;
nCompleted: number;
nImputed: number;
nDefaultFallback: number;
nDatasets: number;
completionPct: number;
medianTrainPer1k: number | null;
medianInferPer1k: number | null;
medianFinalFitPer1k: number | null;
isParetoTrain: boolean;
isParetoInfer: boolean;
}
export interface WinRatePayload {
models: string[];
matrix: (number | null)[][];
/** [wins, losses, ties, scoreWins, compWins] */
counts: Record<string, [number, number, number, number, number]>;
}
export interface PrecomputedModeResult {
nDatasets: number;
datasetIds?: string[];
entries: LeaderboardEntry[];
winRate?: WinRatePayload;
}
export interface PrecomputedScope {
key: string;
label: string;
nDatasets: number;
nModels: number;
datasetIds: string[];
samplesMin: number;
samplesMax: number;
featuresMin: number;
featuresMax: number;
selected: PrecomputedModeResult;
paired: PrecomputedModeResult;
nDatasetsNoOverlap?: number;
selectedNoOverlap?: PrecomputedModeResult;
pairedNoOverlap?: PrecomputedModeResult;
}
export interface DomainFailureRow {
domain: string;
domainSlug: DomainSlug;
nExpected: number;
nFailed: number;
gpu_oom: number;
host_ram_oom: number;
time_limit: number;
segmentation_fault: number;
mixed: number;
}
export interface BenchmarkData {
meta: {
benchmarkName: string;
subtitle: string;
generatedAt: string;
resultsRevision: string;
dataRevision: string;
nDatasets: number;
nDomains: number;
nModelsAllTasks: number;
nModelsClassification: number;
medLog10Samples: number;
medLog10Features: number;
};
domains: DomainInfo[];
models: ModelInfo[];
datasets: DatasetInfo[];
evaluations: EvalRecord[];
scopes: Record<string, PrecomputedScope>;
domainFailures: DomainFailureRow[];
}
export type CompetitorView = 'selected' | 'paired' | 'tuned_only' | 'default_only';
export type MissingPolicyView = 'rf_imputed' | 'no_overlap';
export type MissingSlice = 'all' | 'with_missing' | 'no_missing' | 'high_missing';
export interface ActiveScopeSelection {
/** Primary scope axis: 'global', 'domain', 'regime', 'task', 'feature' */
scopeKey: string;
competitorView: CompetitorView;
missingPolicy: MissingPolicyView;
families: ModelFamily[];
excludedModels: string[];
searchQuery: string;
}