Spaces:
Running
Running
| // One task: what is forecast, how systems score, run days, inputs, case studies, files. | |
| // Every task renders from the same template; nothing here knows a particular target. | |
| import { anchorStrip, fanChart, historyChart, horizonChart } from "./charts.js"; | |
| import { fileUrl, load } from "./data.js"; | |
| import { | |
| bySystem, dayLabel, empty, fmt, h, leaderboard, md, more, redraws, sectionHead, segmented, | |
| sourceLink, system, systemCell, timeIn, | |
| } from "./util.js"; | |
| const MODES = [["live", "Live"], ["mock", "Offline replay"]]; | |
| const RUN_DAYS_SHOWN = 10; | |
| export function areaName(index, key) { | |
| const [domainId, leafId] = (key || "").split("/"); | |
| const domain = index.taxonomy.domains.find((d) => d.id === domainId); | |
| const leaf = domain?.leaves.find((l) => l.id === leafId); | |
| return { domain, leaf }; | |
| } | |
| function pips(done, total) { | |
| return h("span", { class: "pips", "aria-hidden": "true" }, d3.range(total).map((i) => h("i", { class: i < done ? "on" : null }))); | |
| } | |
| function head(index, t) { | |
| const s = t.spec; | |
| const { domain, leaf } = areaName(index, s.taxonomy); | |
| const tz = s.schedule.timezone; | |
| const cadence = s.schedule.every_days === 1 ? "every day" : `every ${s.schedule.every_days} days`; | |
| const zone = new Intl.DateTimeFormat("en-US", { timeZone: tz, timeZoneName: "short" }) | |
| .formatToParts(new Date(t.status.next_run)).find((p) => p.type === "timeZoneName")?.value; | |
| const fact = (label, value) => h("span", {}, h("b", {}, label), value); | |
| const sections = [["target", "Target"], ["results", "Results"], ["run-days", "Run days"], ["inputs", "Inputs"], | |
| ...(t.case_studies.length ? [["cases", "Case studies"]] : []), ["files", "Task files"]]; | |
| return h("header", { class: "page-head" }, | |
| h("div", { class: "crumbs" }, h("a", { href: "#/tasks" }, "Tasks"), "›", | |
| domain ? h("a", { href: `#/tasks?area=${domain.id}` }, domain.name) : null, leaf ? "›" : null, | |
| leaf ? h("a", { href: `#/tasks?area=${s.taxonomy}` }, leaf.name) : null), | |
| h("h1", {}, s.title), | |
| h("p", { class: "lead" }, s.question), | |
| h("div", { class: "facts" }, | |
| fact("Region", s.region), fact("Unit", s.unit), | |
| fact("Horizon", `${d3.min(s.horizons)}–${d3.max(s.horizons)} days ahead`), | |
| fact("Forecast", `${s.quantiles.length} quantiles`), | |
| fact("Due", `${s.schedule.cutoff} ${zone}, ${cadence}`), | |
| fact("Live since", `${dayLabel(t.status.live_since, "day")} · ${t.status.run_days} run days`), | |
| fact("Next run", timeIn(t.status.next_run, tz, true))), | |
| h("nav", { class: "anchors", "aria-label": "Sections" }, sections.map(([id, text]) => | |
| h("a", { href: `#${id}`, onclick: (e) => { e.preventDefault(); document.getElementById(id)?.scrollIntoView({ behavior: "smooth" }); } }, text)))); | |
| } | |
| function targetSection(t) { | |
| const s = t.spec; | |
| const chart = h("div", { class: "chart" }); | |
| redraws.push(() => historyChart(chart, { truth: t.truth, runDays: t.run_days.map((d) => d.run_date), unit: s.unit, title: s.title })); | |
| return h("section", { class: "section" }, | |
| sectionHead("target", "What is forecast"), | |
| h("div", { class: "card split" }, | |
| h("div", { class: "target-text" }, | |
| h("p", { class: "def-main" }, s.summary), | |
| h("dl", { class: "def" }, | |
| h("dt", {}, "Definition"), h("dd", {}, s.y.definition), | |
| h("dt", {}, "Settled on"), h("dd", {}, s.y.settlement), | |
| h("dt", {}, "Source"), h("dd", {}, sourceLink(s.y.source)))), | |
| h("div", {}, chart, h("p", { class: "note" }, `First published values, last ${t.truth.length} days; blue ticks mark run days.`)))); | |
| } | |
| function resultsSection(t, state, rerender) { | |
| const rows = t.leaderboard[state.mode] || []; | |
| const n = rows.find((r) => !r.stopped)?.n; | |
| const horizon = h("div", { class: "chart" }); | |
| const byHorizon = t.by_horizon[state.mode] || []; | |
| const order = [...new Set(byHorizon.map((r) => r.system))].sort(bySystem); | |
| if (byHorizon.length) redraws.push(() => horizonChart(horizon, byHorizon, order)); | |
| return h("section", { class: "section" }, | |
| sectionHead("results", "Results", | |
| state.mode === "live" | |
| ? "Live forecasts, scored once the values are published. Every row uses the same (run day, target date) pairs, except agents no longer run, listed last on their own run days; 1 is the seasonal naive forecast, lower is better." | |
| : "Agents replaying each run day offline from the materials the live agents saved, next to the same TSFM and expert forecasts.", | |
| segmented(MODES, state.mode, (mode) => { state.mode = mode; rerender(); }, "Mode")), | |
| h("div", { class: "card" }, | |
| leaderboard(rows, t.spec.unit), | |
| n ? h("p", { class: "note", style: { marginTop: "10px" } }, `${n} common (run day, target date) pairs.`) : null, | |
| byHorizon.length ? more("By horizon", horizon, h("p", { class: "note" }, "Later horizons resolve later, so their points rest on fewer run days.")) : null), | |
| state.mode === "live" ? continualCard(t.continual, "Self-improvement: continual versus fresh", | |
| "A continual agent continues, each run day, from a snapshot of the sandbox and native session of its previous run day, sees its earlier submissions and is asked to improve. It runs beside the fresh agent of the same harness and model; on the same pairs, the difference is what carried-over experience adds.", | |
| ["Fresh", "Continual"], "No continual forecast has been scored yet.") : null, | |
| state.mode === "live" ? continualCard(t.continual_tsfm, "Using the TSFMs: continual + TSFMs versus continual", | |
| "A second continual chain of the same agent is also told, every run day, to download and run the TSFMs this benchmark evaluates, on covariates it finds itself, and to adjust their forecasts into its own. On the same pairs, its difference from the continual agent is what that instruction adds.", | |
| ["Continual", "+ TSFMs"], "No continual + TSFMs forecast has been scored yet.") : null); | |
| } | |
| /** A continual arm against its twin, on the live pairs both resolved: continual agents | |
| * against the fresh agent of the same harness and model, continual + TSFMs agents against | |
| * the continual agent. `p.fresh` is the twin, `p.continual` the arm's agent. */ | |
| function continualCard(arm, title, text, [twinHead, armHead], none) { | |
| if (!arm?.length) return null; | |
| const pairs = arm.filter((p) => p.rows.length === 2); | |
| const score = (rows, id) => rows.find((r) => r.system === id); | |
| return h("div", { class: "card" }, | |
| h("div", { class: "card-title" }, title), | |
| h("p", { class: "secondary small", style: { marginBottom: "10px" } }, text), | |
| pairs.length | |
| ? h("div", { class: "table-wrap" }, h("table", {}, | |
| h("thead", {}, h("tr", {}, h("th", {}, "Agent"), h("th", { class: "num" }, twinHead), h("th", { class: "num" }, armHead), | |
| h("th", { class: "num" }, "Difference"), h("th", { class: "num" }, "Pairs"))), | |
| h("tbody", {}, pairs.map((p) => { | |
| const twin = score(p.rows, p.fresh), cont = score(p.rows, p.continual); | |
| const diff = cont.scaled_crps - twin.scaled_crps; | |
| return h("tr", {}, h("td", {}, systemCell(p.continual.replace(/-continual(-tsfm)?$/, ""))), | |
| h("td", { class: "num" }, fmt.ratio(twin.scaled_crps)), h("td", { class: "num" }, fmt.ratio(cont.scaled_crps)), | |
| h("td", { class: "num", title: `${armHead} minus ${twinHead} scaled CRPS; negative means ${armHead} did better` }, | |
| `${diff > 0 ? "+" : ""}${fmt.ratio(diff)}`), | |
| h("td", { class: "num" }, String(twin.n))); | |
| })))) | |
| : h("p", { class: "note" }, none)); | |
| } | |
| /** The forecasts a mode shows: live, or the replaying agents next to the live non-agents. */ | |
| export function inMode(forecasts, mode) { | |
| return forecasts.filter((f) => (mode === "live" ? f.mode === "live" : f.mode === "mock" || system(f.system).kind !== "agent")) | |
| .sort((a, b) => bySystem(a.system, b.system)); | |
| } | |
| function runDaysSection(t, state) { | |
| const shown = state.allDays ? t.run_days : t.run_days.slice(0, RUN_DAYS_SHOWN); | |
| // Columns: every non-reference system the mode shows (replaying agents plus the live others). | |
| const columns = [...new Set(t.run_days.flatMap((d) => [...d.submitted[state.mode], ...d.submitted.live.filter((id) => system(id).kind !== "agent")]))] | |
| .filter((id) => system(id).kind !== "reference" && (state.mode === "live" || system(id).kind !== "agent" || t.run_days.some((d) => d.submitted.mock.includes(id)))) | |
| .sort(bySystem); | |
| const latest = t.run_days[0]; | |
| const chart = h("div", { class: "chart" }); | |
| const legend = h("div", { class: "legend" }); | |
| if (latest) { | |
| load(`tasks/${t.id}/${latest.run_date}.json`).then((day) => { | |
| const forecasts = inMode(day.forecasts, state.mode); | |
| const focus = forecasts.find((f) => system(f.system).kind === "agent")?.system; | |
| const draw = () => fanChart(chart, { title: t.spec.title, unit: t.spec.unit, runDate: day.run_date, truth: day.truth, forecasts, | |
| lookback: 21, visible: new Set(forecasts.map((f) => f.system)), focus, height: 320 }); | |
| legend.replaceChildren(...forecasts.map((f) => h("span", { class: "chip" }, systemCell(f.system)))); | |
| redraws.push(draw); | |
| draw(); | |
| }); | |
| } | |
| const table = h("div", { class: "table-wrap" }, h("table", {}, | |
| h("thead", {}, h("tr", {}, h("th", {}, "Run day"), h("th", {}, "Resolved"), | |
| columns.map((id) => h("th", { class: "num wrap-head", title: system(id).label }, system(id).label)), | |
| h("th", { class: "num" }, "Materials"))), | |
| h("tbody", {}, shown.map((d, i) => { | |
| const scores = { ...d.scores.live, ...(state.mode === "mock" ? d.scores.mock : {}) }; | |
| if (state.mode === "mock") for (const id of Object.keys(d.scores.live)) if (system(id).kind === "agent" && !(id in d.scores.mock)) delete scores[id]; | |
| const values = columns.map((id) => scores[id]?.scaled_crps); | |
| const best = d3.min(values.filter((v, i) => v != null && system(columns[i]).kind !== "reference")); | |
| const files = d3.sum(Object.values(d.materials), (m) => m.files); | |
| const open = () => { location.hash = `#/t/${t.id}/${d.run_date}${state.mode === "mock" ? "?mode=mock" : ""}`; }; | |
| return [h("tr", { class: "link", onclick: open }, | |
| h("td", { class: "nowrap" }, h("a", { href: `#/t/${t.id}/${d.run_date}`, onclick: (e) => e.stopPropagation() }, dayLabel(d.run_date, "dow"))), | |
| h("td", { class: "nowrap" }, pips(d.resolved, d.target_dates), h("span", { class: "small secondary" }, `${d.resolved}/${d.target_dates}`)), | |
| columns.map((id, i) => { | |
| const submitted = (system(id).kind === "agent" ? d.submitted[state.mode] : d.submitted.live).includes(id); | |
| return h("td", { class: `num${values[i] != null && values[i] === best ? " best" : ""}` }, | |
| values[i] != null ? fmt.ratio(values[i]) : h("span", { class: "muted" }, submitted ? "pending" : "—")); | |
| }), | |
| h("td", { class: "num" }, files ? String(files) : "—"))]; | |
| })))); | |
| return h("section", { class: "section" }, | |
| sectionHead("run-days", "Run days", | |
| `Scaled CRPS of each system on each run day, over that day's resolved target dates (${state.mode === "live" ? "live" : "offline replay"}). Open a day for its forecasts, agent runs and materials.`), | |
| latest ? h("div", { class: "card" }, | |
| h("div", { class: "card-title" }, `Latest forecasts · made ${dayLabel(latest.run_date, "long")}`), | |
| legend, chart, | |
| h("p", { class: "note" }, "Ink: published values. Lines: each system's median; band: the first agent's 10–90% interval. Open the run day to compare systems one by one.")) : null, | |
| h("div", { class: "card" }, t.run_days.length ? table : empty("No run day yet."), | |
| t.run_days.length > RUN_DAYS_SHOWN | |
| ? h("div", { style: { marginTop: "10px" } }, h("button", { type: "button", class: "link-button", onclick: () => { state.allDays = !state.allDays; state.rerender(); } }, | |
| state.allDays ? "Show fewer" : `Show all ${t.run_days.length} run days`)) | |
| : null)); | |
| } | |
| function inputsSection(t) { | |
| const s = t.spec; | |
| const b = s.baseline; | |
| const x = s.x; | |
| const latest = t.status.latest_run; | |
| const notes = t.latest_x?.covariate_notes || {}; | |
| const noteRows = Object.entries(notes).map(([key, value]) => | |
| [h("dt", {}, key), h("dd", {}, typeof value === "string" ? value : Object.entries(value).map(([k, v]) => `${k.replaceAll("_", " ")}: ${v}`).join(" · "))]); | |
| const files = ["series.csv", "task.json", "MANIFEST.json"]; | |
| return h("section", { class: "section" }, | |
| sectionHead("inputs", "Inputs", "What the comparison systems use. Agents receive none of it: only the instruction, the run date and the open internet."), | |
| h("div", { class: "grid-2" }, | |
| b ? h("div", { class: "card" }, | |
| h("div", { class: "eyebrow" }, b.reference ? "Reference forecast" : "Expert forecast"), | |
| h("h3", { style: { fontSize: "16px", margin: "6px 0 8px" } }, b.name), | |
| h("p", { class: "secondary small" }, b.method), | |
| h("dl", { class: "def", style: { marginTop: "12px" } }, | |
| h("dt", {}, "Form"), h("dd", {}, b.kind === "point" ? "Point forecast (scored by absolute error)" : "Quantile forecast"), | |
| h("dt", {}, "Agents see it"), h("dd", {}, b.visible_to_agents ? "Yes: it is published while the agents run" : "No: it is published after the cutoff"), | |
| h("dt", {}, "Source"), h("dd", {}, sourceLink(b.source)))) | |
| : h("div", { class: "card" }, | |
| h("div", { class: "eyebrow" }, "Expert forecast"), | |
| h("p", { class: "secondary small", style: { margin: "6px 0 0" } }, "No public expert or official forecast of this series exists, so systems are compared with each other and with the seasonal naive reference.")), | |
| h("div", { class: "card" }, | |
| h("div", { class: "eyebrow" }, "Curated covariates · input to the TSFM baselines"), | |
| h("p", { class: "secondary small", style: { margin: "6px 0 10px" } }, | |
| md(`Chosen by the task's author and snapshotted before each run ($X_d$). History from ${dayLabel(x.history_start, "long")}.`)), | |
| h("div", { class: "covariates" }, x.groups.map((g) => h("div", { class: "covariate" }, | |
| h("h4", {}, g.name), | |
| g.what ? h("p", { class: "what" }, g.what) : null, | |
| g.why ? h("p", { class: "why" }, g.why) : null, | |
| h("p", { class: "src" }, g.url ? h("a", { href: g.url, target: "_blank", rel: "noopener" }, g.source) : g.source)))), | |
| more("Columns, notes and downloads", | |
| h("dl", { class: "def" }, | |
| h("dt", {}, "Columns"), h("dd", {}, h("div", { class: "chips" }, x.groups.flatMap((g) => g.columns.map((c) => h("code", {}, c))))), | |
| ...noteRows.flat(), | |
| h("dt", {}, md("Also in $X_d$")), h("dd", {}, x.raw.join("; "))), | |
| latest ? h("div", { class: "downloads", style: { marginTop: "10px" } }, | |
| h("span", { class: "note" }, md(`$X_d$ of ${dayLabel(latest, "day")}:`)), | |
| files.map((name) => h("a", { href: fileUrl(`runs/${t.id}/${latest}/x/${name}`), target: "_blank", rel: "noopener" }, `${name} ↓`))) : null)))); | |
| } | |
| const STEP_KIND = { model: "Model", reference: "Reference", combine: "Combine", uncertainty: "Uncertainty" }; | |
| function caseBody(t, cs, day) { | |
| const value = (f, date) => f.values.find((v) => v.target_date === date); | |
| const actualOf = (date) => day.truth.find((d) => d.date === date)?.value; | |
| const actual = actualOf(cs.focus_date); | |
| const components = Object.fromEntries(cs.agents.map((a) => [a.system, a.components])); | |
| const live = day.forecasts.filter((f) => f.mode === "live").sort((a, b) => bySystem(a.system, b.system)); | |
| const replay = day.forecasts.filter((f) => f.mode === "mock").sort((a, b) => bySystem(a.system, b.system)); | |
| const strip = h("div", { class: "chart" }); | |
| const rows = live.map((f) => { | |
| const v = value(f, cs.focus_date); | |
| return { system: f.system, point: v.point, q10: v.quantiles?.["0.1"], q90: v.quantiles?.["0.9"], components: components[f.system] }; | |
| }); | |
| if (actual != null) redraws.push(() => anchorStrip(strip, { rows, actual, unit: t.spec.unit })); | |
| const resolved = [...new Set(live.flatMap((f) => f.values.filter((v) => v.crps != null).map((v) => v.target_date)))].sort(); | |
| const best = Object.fromEntries(resolved.map((d) => [d, d3.min([...live, ...replay], (f) => value(f, d)?.crps)])); | |
| const scoreRow = (f, suffix) => h("tr", {}, | |
| h("td", {}, systemCell(f.system), suffix ? h("span", { class: "cell-sub" }, suffix) : null), | |
| resolved.map((d) => { | |
| const c = value(f, d)?.crps; | |
| return h("td", { class: `num${c != null && c === best[d] ? " best" : ""}` }, fmt.value(c)); | |
| })); | |
| return h("div", { class: "case-body" }, | |
| h("div", { class: "chips" }, cs.context.map((c) => h("span", { class: "chip", title: c.note }, `${c.label}: ${c.value}`))), | |
| actual != null ? h("div", {}, h("div", { class: "card-title" }, `Where each forecast for ${dayLabel(cs.focus_date, "dow")} landed`), strip) : null, | |
| h("div", { class: "case-agents" }, cs.agents.map((a) => h("div", { class: "card" }, | |
| h("h3", { class: "card-title" }, systemCell(a.system)), | |
| h("div", { class: "chips", style: { marginBottom: "8px" } }, [ | |
| `${a.effort.minutes} min`, `${a.effort.commands} shell commands`, `${a.effort.searches} web searches`, | |
| a.effort.images ? `${a.effort.images} charts viewed` : null, | |
| ].filter(Boolean).map((text) => h("span", { class: "chip" }, text))), | |
| h("p", { style: { fontWeight: 600, marginBottom: "10px" } }, a.headline), | |
| h("ol", { class: "case-steps" }, a.steps.map((step) => h("li", {}, | |
| h("strong", {}, `${STEP_KIND[step.kind] || step.kind}: ${step.title}. `), h("span", { class: "secondary" }, step.text), | |
| step.formula ? h("div", { class: "formula" }, step.formula) : null, | |
| step.facts ? h("div", { class: "chips", style: { marginTop: "4px" } }, step.facts.map((fact) => h("span", { class: "chip" }, fact))) : null)))))), | |
| resolved.length ? h("div", {}, h("div", { class: "card-title" }, `CRPS (${t.spec.unit}) per resolved target date`), | |
| h("div", { class: "table-wrap" }, h("table", {}, | |
| h("thead", {}, h("tr", {}, h("th", {}, "System"), resolved.map((d) => h("th", { class: "num" }, dayLabel(d, "dow"))))), | |
| h("tbody", {}, | |
| h("tr", { class: "group" }, h("td", { colspan: resolved.length + 1 }, "Live")), live.map((f) => scoreRow(f)), | |
| replay.length ? [h("tr", { class: "group" }, h("td", { colspan: resolved.length + 1 }, "Offline replay of the same day")), replay.map((f) => scoreRow(f, "replay"))] : null, | |
| h("tr", {}, h("td", { class: "muted" }, "Actual"), resolved.map((d) => h("td", { class: "num muted" }, fmt.value(actualOf(d))))))))) : null, | |
| h("div", {}, h("div", { class: "card-title" }, "What this day shows"), h("ul", { class: "secondary" }, cs.takeaways.map((text) => h("li", {}, text))))); | |
| } | |
| function casesSection(t) { | |
| if (!t.case_studies.length) return null; | |
| return h("section", { class: "section" }, | |
| sectionHead("cases", "Case studies", "Single run days written up: what the agents found, how they built their forecasts, and how it went."), | |
| h("div", { class: "cases" }, [...t.case_studies].reverse().map((cs) => { | |
| const body = h("div"); | |
| const details = h("details", { class: "card case" }, | |
| h("summary", {}, h("span", { class: "eyebrow" }, `Run day ${dayLabel(cs.run_date, "long")}`), h("h3", {}, cs.title), | |
| h("p", { class: "secondary small" }, cs.summary), h("span", { class: "open" }, "Read the case study ▸")), | |
| body); | |
| details.addEventListener("toggle", async () => { | |
| if (!details.open || body.childElementCount) return; | |
| const day = await load(`tasks/${t.id}/${cs.run_date}.json`); | |
| body.replaceChildren(caseBody(t, cs, day)); | |
| redraws.forEach((draw) => draw()); | |
| }, { once: false }); | |
| return details; | |
| }))); | |
| } | |
| function filesSection(t) { | |
| const link = (href, text, note) => h("div", {}, h("a", { href, target: "_blank", rel: "noopener" }, `${text} ↗`), h("span", { class: "note" }, note)); | |
| return h("section", { class: "section" }, | |
| sectionHead("files", "Task files"), | |
| h("div", { class: "card files" }, | |
| link(t.links.live_task, "Live Task", " · the ALE Task agents run every run day, with its data-fetching script"), | |
| link(t.links.mock_tasks, "Mock Tasks", " · one offline replay Task per run day"), | |
| link(t.links.results, "Run records", " · forecasts, actual values and every agent trajectory"))); | |
| } | |
| export async function taskPage(app, index, id, params) { | |
| const t = await load(`tasks/${id}.json`); | |
| const state = { mode: params.get("mode") === "mock" ? "mock" : "live", allDays: false }; | |
| const results = h("div"); | |
| const runDays = h("div"); | |
| state.rerender = () => { | |
| redraws.length = 0; | |
| results.replaceChildren(resultsSection(t, state, state.rerender)); | |
| runDays.replaceChildren(runDaysSection(t, state)); | |
| staticDraws.forEach((d) => redraws.push(d)); | |
| redraws.forEach((draw) => draw()); | |
| history.replaceState(null, "", `#/t/${id}${state.mode === "mock" ? "?mode=mock" : ""}`); | |
| }; | |
| const target = targetSection(t); | |
| const staticDraws = [...redraws]; | |
| results.replaceChildren(resultsSection(t, state, state.rerender)); | |
| runDays.replaceChildren(runDaysSection(t, state)); | |
| app.replaceChildren(head(index, t), target, results, runDays, inputsSection(t), casesSection(t), filesSection(t)); | |
| } | |