Spaces:
Running
Running
File size: 5,314 Bytes
d70f132 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 | import resultsJson from "@/data/results.json";
import type {
Category,
DatasetEntry,
DatasetInfo,
MetricKey,
ModelRow,
Provenance,
ResultsFile,
Split,
Track,
} from "./types";
/* JSON tuples ([lo, hi] CIs) widen to number[] under resolveJsonModule,
so we narrow through unknown once, here, and nowhere else. */
const results = resultsJson as unknown as ResultsFile;
/* ------------------------------------------------------------------ */
/* Platform version strings — single source of truth. */
/* Every surface (run manifests, methodology exemplar, nav badge) */
/* must reference these, never a literal. */
/* ------------------------------------------------------------------ */
export const NORMALIZER_VERSION = "v0.1";
export const HARNESS_VERSION = "medasr-bench v0.4";
/* ------------------------------------------------------------------ */
/* Dataset registry */
/* ------------------------------------------------------------------ */
/** The anchor (first live) dataset — what no-arg accessors resolve to. */
export const ANCHOR_DATASET_ID = "vimedcss";
/** All dataset profiles, in registry order. */
export function getDatasets(): DatasetInfo[] {
return results.datasets.map((d) => d.info);
}
/** Only datasets with leaderboards running today. */
export function getLiveDatasets(): DatasetInfo[] {
return results.datasets
.filter((d) => d.info.status === "live")
.map((d) => d.info);
}
/** Full registry entry (profile + model rows). No-arg returns the anchor. */
export function getDatasetEntry(id: string = ANCHOR_DATASET_ID): DatasetEntry {
const entry = results.datasets.find((d) => d.info.id === id);
if (!entry) {
const known = results.datasets.map((d) => d.info.id).join(", ");
throw new Error(
`Unknown dataset id "${id}". Registered datasets: ${known}.`,
);
}
return entry;
}
/** Dataset profile by id. No-arg returns the anchor (ViMedCSS). */
export function getDataset(id: string = ANCHOR_DATASET_ID): DatasetInfo {
return getDatasetEntry(id).info;
}
/** All model rows for a dataset, in seed order. No-arg: anchor dataset. */
export function getModels(datasetId: string = ANCHOR_DATASET_ID): ModelRow[] {
return getDatasetEntry(datasetId).models;
}
/** Single model by slug within a dataset, or undefined. */
export function getModel(
slug: string,
datasetId: string = ANCHOR_DATASET_ID,
): ModelRow | undefined {
return getDatasetEntry(datasetId).models.find((m) => m.slug === slug);
}
/** Metrics where a higher value is better (everything else: lower wins). */
export const HIGHER_IS_BETTER: ReadonlySet<MetricKey> = new Set([
"mtrExact",
"mtrFuzzy",
"rtfx",
]);
/**
* Models that have `metric` measured on `split`, sorted best-first
* (ascending for WER-family, descending for MTR/RTFx).
* Rows without the metric (running/planned) are excluded.
*/
export function getRanked(
split: Split,
metric: MetricKey = "wer",
datasetId: string = ANCHOR_DATASET_ID,
): ModelRow[] {
const dir = HIGHER_IS_BETTER.has(metric) ? -1 : 1;
return getDatasetEntry(datasetId)
.models.filter((m) => typeof m.metrics?.[split]?.[metric] === "number")
.sort(
(a, b) =>
dir * ((a.metrics![split]![metric] as number) -
(b.metrics![split]![metric] as number)),
);
}
/* ------------------------------------------------------------------ */
/* Label maps */
/* ------------------------------------------------------------------ */
export const categoryLabels: Record<Category, string> = {
"vn-open": "Vietnamese open",
"multilingual-open": "Multilingual open",
medical: "Medical-specialized",
"api-intl": "Commercial API (intl)",
"api-vn": "Commercial API (VN)",
};
export const trackLabels: Record<Track, string> = {
A: "Track A — Zero-shot",
B: "Track B — Contextual biasing",
C: "Track C — Adapted",
};
export const provenanceLabels: Record<Provenance, string> = {
"platform-verified": "Platform-verified",
"api-snapshot": "API snapshot",
"self-reported": "Self-reported",
"paper-imported": "Paper-imported",
running: "Running",
planned: "Planned",
};
export const metricLabels: Record<MetricKey, string> = {
wer: "WER",
cer: "CER",
csWer: "CS-WER",
nWer: "N-WER",
mtrExact: "MTR (exact)",
mtrFuzzy: "MTR (fuzzy)",
rtfx: "RTFx",
};
export const splitLabels: Record<Split, string> = {
test: "Test",
hard: "Hard",
};
/* ------------------------------------------------------------------ */
/* Formatters */
/* ------------------------------------------------------------------ */
/** Fixed-point number, or em-dash when missing. fmt(23.5) → "23.50". */
export function fmt(x: number | null | undefined, digits = 2): string {
if (x === null || x === undefined || Number.isNaN(x)) return "—";
return x.toFixed(digits);
}
/** 95% CI as "lo–hi" (en dash), or "" when missing. fmtCI([22.8, 24.5]) → "22.80–24.50". */
export function fmtCI(ci: [number, number] | undefined, digits = 2): string {
if (!ci) return "";
return `${ci[0].toFixed(digits)}–${ci[1].toFixed(digits)}`;
}
|