Spaces:
Running
Running
| // Precompute per-evaluation multi-metric and per-slice score matrices and | |
| // emit them as one JSON map at data/eval-matrices.json. | |
| // | |
| // The runtime eval-summary endpoint currently only returns one | |
| // (metric, model) row per model — sourced from the primary metric of | |
| // `eval_results_view`. The other declared `leaderboard_metrics` (and | |
| // per-slice subtask scores hiding inside `fact_results`) are dropped on | |
| // the floor, which is why the eval page can't render a multi-metric | |
| // leaderboard or a slice dropdown. | |
| // | |
| // Both views are reconstructable from the warehouse parquet files; this | |
| // script does the join once at build time so the runtime path is a flat | |
| // O(1) lookup against the precomputed map. | |
| // | |
| // Output schema: | |
| // { | |
| // snapshot_id, // pinned for cache busting | |
| // generated_at, | |
| // evals: { | |
| // "<evaluation_id>": { | |
| // // Per-(model, metric) values across the eval's full | |
| // // leaderboard_metrics list. Drives the multi-metric matrix. | |
| // leaderboard_rows: [ | |
| // { | |
| // model_route_id, | |
| // values: { "<column_key>": score | null }, | |
| // verified: { "<column_key>": boolean }, | |
| // comparability_status: { "<column_key>": "ok"|"mixed_scale"|"no_bounds" }, | |
| // // Judge panel behind the cell's number, and the same | |
| // // (model, metric)'s other panels — the pivot has one row per | |
| // // model, so these ride the cell. Both omitted when the | |
| // // benchmark names no judge. | |
| // judge_condition: { "<column_key>": "<judge_condition JSON>" }, | |
| // judge_alternates: { | |
| // "<column_key>": [{ judge_condition, score }] | |
| // } | |
| // } | |
| // ], | |
| // // Subtask-scope metric entries to *append* to the eval's | |
| // // leaderboard_metrics. Each carries column_key | |
| // // "<metric_id>::<slice_key>" so it slots into values{} above. | |
| // subtask_metrics: [BenchmarkLeaderboardMetric] | |
| // } | |
| // } | |
| // } | |
| // | |
| // Run via the build chain (`pnpm build`) or standalone: | |
| // node scripts/build-eval-matrices.mjs | |
| import { DuckDBInstance } from "@duckdb/node-api" | |
| import fs from "node:fs/promises" | |
| import path from "node:path" | |
| import { fileURLToPath } from "node:url" | |
| // DuckDB-node returns its list / struct / map values as opaque wrapper | |
| // classes whose payload lives behind `.items` / `.entries`. Walk through | |
| // these so downstream code can treat the result as plain JSON. | |
| function normalizeDuck(value) { | |
| if (value == null) return value | |
| if (typeof value === "bigint") return Number(value) | |
| if (Array.isArray(value)) return value.map(normalizeDuck) | |
| if (typeof value === "object") { | |
| const ctor = value.constructor?.name ?? "" | |
| if (ctor === "DuckDBListValue" || ctor === "DuckDBArrayValue") { | |
| return (value.items ?? []).map(normalizeDuck) | |
| } | |
| if (ctor === "DuckDBStructValue") { | |
| return normalizeDuck(value.entries) | |
| } | |
| if (ctor === "DuckDBMapValue" && Array.isArray(value.entries)) { | |
| const out = {} | |
| for (const e of value.entries) out[String(e.key)] = normalizeDuck(e.value) | |
| return out | |
| } | |
| if (ctor === "DuckDBDecimalValue" && typeof value.toString === "function") { | |
| return Number(value.toString()) | |
| } | |
| if (ctor.startsWith("DuckDB") && typeof value.toString === "function") { | |
| return value.toString() | |
| } | |
| const out = {} | |
| for (const [k, v] of Object.entries(value)) out[k] = normalizeDuck(v) | |
| return out | |
| } | |
| return value | |
| } | |
| function readDuckRows(reader) { | |
| return reader.getRowObjects().map(normalizeDuck).map((row) => normalizeDuck(row)) | |
| } | |
| // Deterministic pick among a cell's judge arms when the snapshot marks no | |
| // headline: the widest panel wins (a mean of three judges is the reading a | |
| // source presents as its own), then the canonical condition JSON ascending. | |
| // On a snapshot WITH is_headline the query has already left exactly one arm | |
| // standing, so this never fires. | |
| export function judgeRank(condition) { | |
| if (!condition) return 0 | |
| try { | |
| const parsed = JSON.parse(condition) | |
| return Array.isArray(parsed?.judges) ? parsed.judges.length : 0 | |
| } catch { | |
| return 0 | |
| } | |
| } | |
| export function winsJudgePick(previous, condition) { | |
| if (previous === undefined) return true | |
| const delta = judgeRank(condition) - judgeRank(previous) | |
| if (delta !== 0) return delta > 0 | |
| return String(condition ?? "") < String(previous ?? "") | |
| } | |
| // The identity a cell's facts pool under. The producer aggregates on | |
| // `model_aggregation_key` — the id a fold's members all share — so pooling | |
| // on the raw `model_id` splits one cell into a cell per folded variant and | |
| // then loses the ones whose id the view never carries. `model_id` is also | |
| // NULL on facts whose model never resolved, where the aggregation key still | |
| // holds the raw spelling. | |
| export function factModelKeySql(alias, hasAggregationKey) { | |
| return hasAggregationKey | |
| ? `COALESCE(${alias}.model_aggregation_key, ${alias}.model_id)` | |
| : `${alias}.model_id` | |
| } | |
| // One score column pooled the producer's way: the MEDIAN of the first-party | |
| // rows, falling back to the median of all rows (stage J `tri_agg`). A | |
| // snapshot with no `evaluator_relationship` gets the plain median. | |
| export function pooledMedianSql(column, { alias = "f", hasRelationship = true } = {}) { | |
| const col = `${alias}.${column}` | |
| const allRows = `MEDIAN(${col}) FILTER (WHERE ${col} IS NOT NULL)` | |
| if (!hasRelationship) return allRows | |
| return ( | |
| `COALESCE(MEDIAN(${col}) FILTER (` + | |
| `WHERE ${alias}.evaluator_relationship = 'first_party' AND ${col} IS NOT NULL), ` + | |
| `${allRows})` | |
| ) | |
| } | |
| // A cell's pooled score, by the producer's own rule, so a slice cell and | |
| // the root cell above it are pooled identically. A flat AVG over the raw | |
| // rows would report a number no source published on any cell holding more | |
| // than one fact. Canonical and raw scales pool SEPARATELY (as stage J | |
| // does) and the canonical value wins, so a cell is never a median taken | |
| // across two scales. | |
| export function pooledFactScoreSql({ | |
| alias = "f", | |
| hasCanonical = true, | |
| hasRelationship = true, | |
| } = {}) { | |
| const raw = pooledMedianSql("score", { alias, hasRelationship }) | |
| if (!hasCanonical) return raw | |
| return `COALESCE(${pooledMedianSql("score_canonical", { alias, hasRelationship })}, ${raw})` | |
| } | |
| // Read a snapshot sidecar from whatever SNAPSHOT_URL points at. Node's | |
| // `fetch` has no file: scheme, so a file:// snapshot (the documented local | |
| // v2 loop) is read off disk — otherwise the read fails and the build writes | |
| // an unpinned `snapshot_id: "unknown"`. Directory paths with no scheme are | |
| // a disk read too; everything else is an HTTPS snapshot and goes over fetch. | |
| export async function readSnapshotSidecar(base, name) { | |
| const url = `${base}/${name}` | |
| if (url.startsWith("file://")) return fs.readFile(fileURLToPath(url), "utf8") | |
| if (!/^[a-z][a-z0-9+.-]*:\/\//i.test(url)) return fs.readFile(url, "utf8") | |
| const response = await fetch(url) | |
| if (!response.ok) throw new Error(`${response.status} ${response.statusText}`) | |
| return response.text() | |
| } | |
| const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..") | |
| const WAREHOUSE = path.join(ROOT, ".cache/hf-data/warehouse/latest") | |
| const OUT_PATH = path.join(ROOT, "data/eval-matrices.json") | |
| async function main() { | |
| // Sanity check: the warehouse parquet files must exist locally. | |
| // `pnpm cache-hf-data` (legacy) or a manual download populates them; the | |
| // v2 build-time path streams them via duckdb's HTTPS reader, so when we | |
| // can't find them locally we point duckdb at SNAPSHOT_URL instead. | |
| const snapshotUrl = process.env.SNAPSHOT_URL?.replace(/\/+$/, "") | |
| let base = WAREHOUSE | |
| let useRemote = false | |
| try { | |
| await fs.access(path.join(WAREHOUSE, "eval_results_view.parquet")) | |
| } catch { | |
| if (!snapshotUrl) { | |
| console.error( | |
| "[build-eval-matrices] no local warehouse cache and no SNAPSHOT_URL — abort.", | |
| ) | |
| process.exit(1) | |
| } | |
| base = snapshotUrl | |
| useRemote = true | |
| } | |
| const t0 = Date.now() | |
| const db = await DuckDBInstance.create() | |
| const con = await db.connect() | |
| const fileRef = (name) => { | |
| const url = useRemote ? `${base}/${name}` : path.join(base, name) | |
| return `'${url.replace(/'/g, "''")}'` | |
| } | |
| // Reading snapshot_id from snapshot_meta.json so the output can be | |
| // matched against the pinned build snapshot. | |
| let snapshotId = "unknown" | |
| try { | |
| const metaText = useRemote | |
| ? await readSnapshotSidecar(base, "snapshot_meta.json") | |
| : await fs.readFile(path.join(WAREHOUSE, "snapshot_meta.json"), "utf8") | |
| snapshotId = JSON.parse(metaText).snapshot_id ?? "unknown" | |
| } catch (err) { | |
| console.warn( | |
| `[build-eval-matrices] couldn't resolve snapshot_id: ${err instanceof Error ? err.message : String(err)}`, | |
| ) | |
| } | |
| // 1. All (eval, model, metric, score) rows. Includes non-primary | |
| // metrics that getEvalSummaryById currently filters out. | |
| // Exactly one row per (eval, model, metric): the model's headline | |
| // reading where the snapshot marks one, and otherwise the pipeline's | |
| // ranked-best row (position ranks on the canonical scale in the | |
| // metric's direction; unranked rows — feedback arms, flagged | |
| // scales — sort last). The verified flag and the comparability | |
| // verdict ride the same chosen row, so a cell's value, checkmark and | |
| // badge can't come from different observations. `score_canonical` | |
| // puts every cell of a column on one scale; older snapshots without | |
| // the column fall back to raw. | |
| const viewCols = new Set( | |
| readDuckRows( | |
| await con.runAndReadAll( | |
| `SELECT column_name FROM (DESCRIBE SELECT * FROM read_parquet(${fileRef("eval_results_view.parquet")}))`, | |
| ), | |
| ).map((r) => r.column_name), | |
| ) | |
| const hasCanonical = viewCols.has("score_canonical") | |
| const hasPosition = viewCols.has("position") | |
| const viewHasHeadline = viewCols.has("is_headline") | |
| const viewHasStatus = viewCols.has("comparability_status") | |
| const viewHasJudge = viewCols.has("judge_condition") | |
| const metricRows = await con.runAndReadAll(` | |
| SELECT | |
| r.evaluation_id, | |
| r.metric_id, | |
| r.model_route_id, | |
| r.score, | |
| ${hasCanonical ? "r.score_canonical" : "NULL"} AS score_canonical, | |
| ${viewHasStatus ? "r.comparability_status" : "NULL"} AS comparability_status, | |
| ${viewHasJudge ? "r.judge_condition" : "NULL"} AS judge_condition, | |
| r.is_verified_evaluator | |
| FROM read_parquet(${fileRef("eval_results_view.parquet")}) r | |
| WHERE r.score IS NOT NULL | |
| AND r.model_route_id IS NOT NULL | |
| ${viewHasHeadline ? "AND r.is_headline" : ""} | |
| QUALIFY row_number() OVER ( | |
| PARTITION BY r.evaluation_id, r.model_route_id, r.metric_id | |
| ORDER BY | |
| ${hasPosition ? "(r.position IS NULL), r.position," : ""} | |
| ${hasCanonical ? "r.score_canonical DESC NULLS LAST," : ""} | |
| r.score DESC, | |
| r.metric_summary_id | |
| ) = 1 | |
| ORDER BY r.evaluation_id, r.model_route_id, r.metric_id | |
| `) | |
| // 1b. The judge readings the pick above left behind. The matrix is a | |
| // pivot — one row per model — so a losing judge panel has no row of | |
| // its own here the way it does on a single-metric leaderboard. These | |
| // ride the cell instead, and without them a multi-metric page drops | |
| // every alternate reading silently. One row per (eval, model, metric, | |
| // panel); a snapshot with no headline or no judge axis has none. | |
| const judgeAlternateRows = | |
| viewHasHeadline && viewHasJudge | |
| ? await con.runAndReadAll(` | |
| SELECT | |
| r.evaluation_id, | |
| r.metric_id, | |
| r.model_route_id, | |
| r.judge_condition, | |
| ${hasCanonical ? "COALESCE(any_value(r.score_canonical), any_value(r.score))" : "any_value(r.score)"} AS score | |
| FROM read_parquet(${fileRef("eval_results_view.parquet")}) r | |
| WHERE r.score IS NOT NULL | |
| AND r.model_route_id IS NOT NULL | |
| AND r.judge_condition IS NOT NULL | |
| AND NOT r.is_headline | |
| GROUP BY 1,2,3,4 | |
| ORDER BY 1,2,3,4 | |
| `) | |
| : null | |
| // 2. Per-slice (composite_slug, benchmark, model, metric, slice_key, | |
| // score) rows. The upstream pipeline parks slice scores in | |
| // fact_results rather than threading them through eval_results_view, | |
| // so we have to reach in here. We carry composite_slug because some | |
| // benchmarks (e.g. `gpqa`) appear under multiple composites and | |
| // fact_results emits a per-source pseudo-slice (slice_key = | |
| // "artificial analysis", "llm stats", "openeval gpqa", ...) for | |
| // each source family. Joining slices on (composite_slug, | |
| // benchmark_id) keeps each composite's slices in its own lane, | |
| // so HF Open LLM v2's GPQA doesn't inherit Artificial Analysis's | |
| // pseudo-slice, etc. Also drop the self-rollup (slice_key == | |
| // benchmark_id) since that duplicates the eval's overall score. | |
| // Facts pool the producer's way (see pooledFactScoreSql). | |
| const factCols = new Set( | |
| readDuckRows( | |
| await con.runAndReadAll( | |
| `SELECT column_name FROM (DESCRIBE SELECT * FROM read_parquet(${fileRef("fact_results.parquet")}))`, | |
| ), | |
| ).map((r) => r.column_name), | |
| ) | |
| // `metric_key` is the EFFECTIVE metric identity (after the registry's | |
| // rename rules); `metric_id` is the pre-rename id and is not what the | |
| // view keys its columns on, so slice columns built from it would never | |
| // line up with their root metric. `score_canonical` puts a slice cell on | |
| // the same scale as the root cell above it. | |
| const factMetric = factCols.has("metric_key") ? "f.metric_key" : "f.metric_id" | |
| const factScore = pooledFactScoreSql({ | |
| hasCanonical: factCols.has("score_canonical"), | |
| hasRelationship: factCols.has("evaluator_relationship"), | |
| }) | |
| const factModelKey = factModelKeySql("f", factCols.has("model_aggregation_key")) | |
| const factHasJudge = factCols.has("judge_condition") | |
| const factHasStatus = factCols.has("comparability_status") | |
| const factHasHeadline = factCols.has("is_headline") | |
| if (!factHasHeadline) { | |
| console.warn( | |
| "[build-eval-matrices] fact_results has no is_headline column — slice cells fall back to " + | |
| "picking one judge condition per cell (highest judge cardinality first). " + | |
| "Rebake against a snapshot that carries the column for the producer's own pick.", | |
| ) | |
| } | |
| const sliceRows = await con.runAndReadAll(` | |
| SELECT | |
| f.composite_slug, | |
| f.benchmark_id, | |
| f.parent_benchmark_id, | |
| ${factMetric} AS metric_id, | |
| f.slice_key, | |
| f.slice_name, | |
| ${factModelKey} AS model_key, | |
| ${factHasJudge ? "f.judge_condition" : "NULL"} AS judge_condition, | |
| ${factScore} AS score, | |
| ${factHasStatus ? "any_value(f.comparability_status)" : "NULL"} AS comparability_status, | |
| bool_or(f.is_verified_evaluator) AS is_verified_evaluator | |
| FROM read_parquet(${fileRef("fact_results.parquet")}) f | |
| WHERE f.score IS NOT NULL | |
| AND f.slice_key IS NOT NULL | |
| AND ${factMetric} IS NOT NULL | |
| AND f.composite_slug IS NOT NULL | |
| ${factHasHeadline ? "AND f.is_headline" : ""} | |
| -- Drop any slice that's a self-rollup of the eval — slice_key | |
| -- equals the benchmark, the composite, or the parent benchmark | |
| -- after normalising separators (so "global mmlu lite" filters | |
| -- against benchmark_id "global-mmlu-lite", "fibble_arena" | |
| -- against "fibble-arena", "artificial analysis" against | |
| -- composite "artificial-analysis-llms", etc.). | |
| AND regexp_replace(lower(f.slice_key), '[^a-z0-9]+', '', 'g') | |
| != regexp_replace(lower(f.benchmark_id), '[^a-z0-9]+', '', 'g') | |
| AND regexp_replace(lower(f.slice_key), '[^a-z0-9]+', '', 'g') | |
| != regexp_replace(lower(f.composite_slug), '[^a-z0-9]+', '', 'g') | |
| -- Also drop slices whose slug is a strict prefix of the | |
| -- composite_slug (e.g. "artificial analysis" vs | |
| -- composite "artificial-analysis-llms" — the slice is just | |
| -- the source family naming itself, not a real subtask). | |
| AND NOT regexp_replace(lower(f.composite_slug), '[^a-z0-9]+', '', 'g') | |
| LIKE regexp_replace(lower(f.slice_key), '[^a-z0-9]+', '', 'g') || '%' | |
| AND ( | |
| f.parent_benchmark_id IS NULL | |
| OR regexp_replace(lower(f.slice_key), '[^a-z0-9]+', '', 'g') | |
| != regexp_replace(lower(f.parent_benchmark_id), '[^a-z0-9]+', '', 'g') | |
| ) | |
| -- Grouping by judge_condition keeps a model's judge arms apart. Without | |
| -- it the pool mixes a headline reading with the individual judges' | |
| -- readings and reports a number no source ever published. On a | |
| -- snapshot that marks is_headline only one arm reaches here anyway. | |
| GROUP BY 1,2,3,4,5,6,7,8 | |
| ORDER BY 1,2,3,4,5,6,7,8 | |
| `) | |
| // 3. eval → (composite_slug, benchmark_id) mapping so we can join | |
| // slice rows back to the right evaluation_id. composite_slug is | |
| // what disambiguates HF Open LLM v2's GPQA from Artificial | |
| // Analysis's GPQA — both share benchmark_id `gpqa`. Also pull | |
| // leaderboard_metrics so we know each metric's metric_summary_id / | |
| // unit / lower_is_better when synthesising subtask-scope entries. | |
| const evalRows = await con.runAndReadAll(` | |
| SELECT | |
| evaluation_id, | |
| benchmark_id, | |
| parent_benchmark_id, | |
| composite_slug, | |
| leaderboard_metrics | |
| FROM read_parquet(${fileRef("evals_view.parquet")}) | |
| `) | |
| // 4. Map the producer's model AGGREGATION key → model_route_id so the | |
| // per-slice rows (pooled on that key above) can land alongside the | |
| // per-metric rows (model_route_id). `eval_results_view.model_key` IS | |
| // that key: a fold's member ids are not what the view carries, so a | |
| // lookup keyed on `model_id` finds no route for them and drops the | |
| // cell. The model_id map stays as the fallback for snapshots | |
| // predating `model_key`. | |
| const viewHasModelKey = viewCols.has("model_key") | |
| const modelKeyRows = await con.runAndReadAll(` | |
| SELECT | |
| ${viewHasModelKey ? "model_key" : "model_id"} AS model_key, | |
| model_id, | |
| min(model_route_id) AS model_route_id | |
| FROM read_parquet(${fileRef("eval_results_view.parquet")}) | |
| WHERE model_route_id IS NOT NULL | |
| GROUP BY 1, 2 | |
| `) | |
| await con.disconnectSync() | |
| // Index both maps so slice lookups are O(1). The smallest route id wins | |
| // a key served by several rows, so the build is order-independent. | |
| const modelKeyToRoute = new Map() | |
| const modelIdToRoute = new Map() | |
| const rememberRoute = (map, key, route) => { | |
| if (key == null || route == null) return | |
| const previous = map.get(key) | |
| if (previous === undefined || route < previous) map.set(key, route) | |
| } | |
| for (const row of modelKeyRows.getRowObjects().map(normalizeDuck)) { | |
| rememberRoute(modelKeyToRoute, row.model_key, row.model_route_id) | |
| rememberRoute(modelIdToRoute, row.model_id, row.model_route_id) | |
| } | |
| // Group eval rows by evaluation_id, indexed by (composite_slug, | |
| // benchmark_id) for the slice join. Two evals can share a benchmark_id | |
| // across composites (gpqa under both hfopenllm-v2 and | |
| // artificial-analysis-llms), so the composite_slug component is what | |
| // keeps them separated. | |
| const evalsByCompositeBench = new Map() | |
| const evalsById = new Map() | |
| const compositeBenchKey = (composite, bench) => | |
| `${composite ?? ""}|${bench ?? ""}` | |
| for (const row of evalRows.getRowObjects().map(normalizeDuck)) { | |
| evalsById.set(row.evaluation_id, row) | |
| const bid = row.benchmark_id ?? null | |
| const composite = row.composite_slug ?? null | |
| if (bid && composite) { | |
| const key = compositeBenchKey(composite, bid) | |
| if (!evalsByCompositeBench.has(key)) evalsByCompositeBench.set(key, []) | |
| evalsByCompositeBench.get(key).push(row.evaluation_id) | |
| } | |
| } | |
| // Bucket metric rows by evaluation_id and within that by model. | |
| // out[evalId].rows[modelRoute].values = { column_key: score } | |
| const out = {} | |
| const ensureEval = (evalId) => { | |
| if (!out[evalId]) { | |
| out[evalId] = { | |
| leaderboard_rows: new Map(), // route_id → values | |
| subtask_metric_keys: new Set(), // tracks which subtask cols we've seen | |
| subtask_metrics: [], | |
| } | |
| } | |
| return out[evalId] | |
| } | |
| // modelEntry holds parallel maps keyed by column_key: `values` (score), | |
| // `verified` (the per-cell verified-evaluator flag), | |
| // `comparability_status` (the cell's comparability verdict, so a matrix | |
| // cell can render "not assessable" rather than a silence that reads as | |
| // "checked, nothing found"), `judge_condition` (the panel behind the cell's | |
| // number) and `judge_alternates` (the same model's other panels for that | |
| // metric, which the pivot has no row to show). | |
| const ensureModelEntry = (bucket, route) => { | |
| let modelEntry = bucket.leaderboard_rows.get(route) | |
| if (!modelEntry) { | |
| modelEntry = { | |
| values: {}, | |
| verified: {}, | |
| comparability_status: {}, | |
| judge_condition: {}, | |
| judge_alternates: {}, | |
| judgePick: {}, | |
| } | |
| bucket.leaderboard_rows.set(route, modelEntry) | |
| } | |
| return modelEntry | |
| } | |
| for (const row of metricRows.getRowObjects().map(normalizeDuck)) { | |
| const bucket = ensureEval(row.evaluation_id) | |
| const modelEntry = ensureModelEntry(bucket, row.model_route_id) | |
| modelEntry.values[row.metric_id] = Number( | |
| row.score_canonical != null ? row.score_canonical : row.score, | |
| ) | |
| if (row.is_verified_evaluator != null) { | |
| modelEntry.verified[row.metric_id] = Boolean(row.is_verified_evaluator) | |
| } | |
| if (row.comparability_status != null) { | |
| modelEntry.comparability_status[row.metric_id] = String(row.comparability_status) | |
| } | |
| if (row.judge_condition != null) { | |
| modelEntry.judge_condition[row.metric_id] = String(row.judge_condition) | |
| } | |
| } | |
| for (const row of judgeAlternateRows?.getRowObjects().map(normalizeDuck) ?? []) { | |
| const bucket = out[row.evaluation_id] | |
| // Only cells the pivot actually renders: an alternate whose model or | |
| // metric never produced a headline cell has nothing to hang off. | |
| const modelEntry = bucket?.leaderboard_rows.get(row.model_route_id) | |
| if (!modelEntry || !(row.metric_id in modelEntry.values)) continue | |
| const score = Number(row.score) | |
| const list = (modelEntry.judge_alternates[row.metric_id] ??= []) | |
| list.push({ | |
| judge_condition: String(row.judge_condition), | |
| score: Number.isFinite(score) ? score : null, | |
| }) | |
| } | |
| // Plant slice scores. Each (metric_id, slice_key) becomes a column | |
| // keyed "<metric_id>::<slice_key>" so it slots into values{} alongside | |
| // root metrics. The matching subtask leaderboard metric metadata is | |
| // emitted in subtask_metrics for the runtime to splice into the eval's | |
| // leaderboard_metrics array. | |
| for (const row of sliceRows.getRowObjects().map(normalizeDuck)) { | |
| const evalIds = evalsByCompositeBench.get( | |
| compositeBenchKey(row.composite_slug, row.benchmark_id), | |
| ) | |
| if (!evalIds) continue | |
| const route = | |
| modelKeyToRoute.get(row.model_key) ?? modelIdToRoute.get(row.model_key) | |
| if (!route) continue | |
| const sliceKey = String(row.slice_key) | |
| const sliceName = row.slice_name ? String(row.slice_name) : sliceKey | |
| const metricId = String(row.metric_id) | |
| const columnKey = `${metricId}::${sliceKey}` | |
| const score = Number(row.score) | |
| if (!Number.isFinite(score)) continue | |
| for (const evalId of evalIds) { | |
| const bucket = ensureEval(evalId) | |
| const modelEntry = ensureModelEntry(bucket, route) | |
| if (winsJudgePick(modelEntry.judgePick[columnKey], row.judge_condition ?? null)) { | |
| modelEntry.judgePick[columnKey] = row.judge_condition ?? null | |
| modelEntry.values[columnKey] = score | |
| if (row.judge_condition != null) { | |
| modelEntry.judge_condition[columnKey] = String(row.judge_condition) | |
| } | |
| if (row.is_verified_evaluator != null) { | |
| modelEntry.verified[columnKey] = Boolean(row.is_verified_evaluator) | |
| } | |
| if (row.comparability_status != null) { | |
| modelEntry.comparability_status[columnKey] = String(row.comparability_status) | |
| } | |
| } | |
| if (!bucket.subtask_metric_keys.has(columnKey)) { | |
| bucket.subtask_metric_keys.add(columnKey) | |
| // Look up the parent eval's metric metadata so the subtask-scope | |
| // entry inherits unit / lower_is_better. Fall back to defaults | |
| // when the registry doesn't carry the metric. | |
| const evalMeta = evalsById.get(evalId) | |
| const rootMetric = (evalMeta?.leaderboard_metrics ?? []).find( | |
| (m) => m.metric_id === metricId, | |
| ) | |
| bucket.subtask_metrics.push({ | |
| column_key: columnKey, | |
| metric_summary_id: rootMetric?.metric_summary_id ?? `${evalId}%3A${metricId}`, | |
| metric_id: metricId, | |
| metric_name: rootMetric?.metric_name ?? metricId, | |
| display_name: rootMetric?.display_name ?? metricId, | |
| canonical_display_name: rootMetric?.canonical_display_name ?? null, | |
| lower_is_better: rootMetric?.lower_is_better ?? false, | |
| unit: rootMetric?.unit ?? null, | |
| scope: "subtask", | |
| subtask_key: sliceKey, | |
| subtask_name: sliceName, | |
| }) | |
| } | |
| } | |
| } | |
| // Materialise: convert internal Maps to JSON-friendly arrays. Drop | |
| // evals that ended up with a single root metric and no subtasks since | |
| // the runtime can already render those through the existing path. | |
| const finalEvals = {} | |
| for (const [evalId, bucket] of Object.entries(out)) { | |
| const rows = [] | |
| for (const [routeId, entry] of bucket.leaderboard_rows) { | |
| rows.push({ | |
| model_route_id: routeId, | |
| values: entry.values, | |
| verified: entry.verified, | |
| comparability_status: entry.comparability_status, | |
| // Judge maps are empty on all but a handful of benchmarks; omitting | |
| // them keeps the artifact the size it was everywhere else. | |
| ...(Object.keys(entry.judge_condition).length > 0 | |
| ? { judge_condition: entry.judge_condition } | |
| : {}), | |
| ...(Object.keys(entry.judge_alternates).length > 0 | |
| ? { judge_alternates: entry.judge_alternates } | |
| : {}), | |
| }) | |
| } | |
| // Skip evals where every model has at most one metric and no | |
| // subtask data — adds no information beyond the existing summary. | |
| const hasMultiMetric = rows.some((r) => Object.keys(r.values).length > 1) | |
| if (!hasMultiMetric && bucket.subtask_metrics.length === 0) continue | |
| finalEvals[evalId] = { | |
| leaderboard_rows: rows, | |
| subtask_metrics: bucket.subtask_metrics, | |
| } | |
| } | |
| const payload = { | |
| snapshot_id: snapshotId, | |
| generated_at: new Date().toISOString(), | |
| evals: finalEvals, | |
| } | |
| await fs.mkdir(path.dirname(OUT_PATH), { recursive: true }) | |
| await fs.writeFile(OUT_PATH, JSON.stringify(payload)) | |
| const sizeMb = ( | |
| Buffer.byteLength(JSON.stringify(payload), "utf8") / 1024 / 1024 | |
| ).toFixed(2) | |
| console.log( | |
| `[build-eval-matrices] wrote ${Object.keys(finalEvals).length} evals to ${path.relative(ROOT, OUT_PATH)} (${sizeMb} MB) in ${Date.now() - t0}ms`, | |
| ) | |
| } | |
| // Only run the build when invoked as a script; importing the module (tests | |
| // of the pure helpers above) must not touch the warehouse or the output file. | |
| if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { | |
| main().catch((err) => { | |
| console.error("[build-eval-matrices] failed:", err) | |
| process.exit(1) | |
| }) | |
| } | |