// A task, whatever its environment's format: /t///. The adapter that reads the environment // (app/envs/contract.py) builds its sections out of blocks (blocks.js draws them, and a renderer module draws what no // block covers), says how it can run (its run options) and which other tasks are this very one (a raw row and its // Harbor conversion), so a Harbor folder, a MiMo task and a NeMo Gym row get the same page: contents on the left, the // sections, then the run panel and the facts. Answers stay out: adapters withhold them, and the page names them only. import { $, $$, api, esc, fmt, sk, spinner, toast, ago, dur, money, statusPill, rewardBadge, rewardText, visibilityBadge, emptyState, confetti, PUBLIC_NOTE, PRIVATE_NOTE, nav as navigate, setMeta } from "./util.js"; import { icon } from "./icons.js"; import { getSession, refreshActive } from "./session.js"; import { picker } from "./picker.js"; import { fileViewer } from "./files.js"; import { blocks, inline, loadRenderers } from "./blocks.js"; import { typeset } from "./math.js"; import { supportPanel } from "./support.js"; import { rewardDistribution } from "./reward-stats.js"; let models = null; const getModels = () => (models ||= api("/api/models").catch((e) => { models = null; throw e; })); const enc = (s) => s.split("/").map(encodeURIComponent).join("/"); const envApi = (spec) => `/api/env/${enc(spec)}`; export const taskHref = (spec, ref) => `/t/${enc(spec)}/${enc(ref)}`; export const compareHref = (spec, ref, ids) => `/compare/${enc(spec)}/${enc(ref)}?r=${ids.map(encodeURIComponent).join(",")}`; const subsetLabel = (s) => String(s || "").split("/").join(" · "); const COMPARE_MAX = 4; let observer = null, viewers = [], keys = null; export function unmount() { observer?.disconnect(); observer = null; viewers.forEach((v) => v.destroy()); viewers = []; if (keys) removeEventListener("keydown", keys); keys = null; } export async function mount(el, { spec, path: ref, qs }) { const panel = (lines) => `
${sk.box(16, "width:140px")}
${sk.lines(...lines)}
`; el.innerHTML = `
${sk.head()}
${panel([96, 100, 88, 94, 70, 92, 60])}${panel([90, 70, 84])}${panel([80, 60])}
`; let v, renderers; try { v = await api(`${envApi(spec)}/task?ref=${encodeURIComponent(ref)}`, { retry: 2 }); renderers = await loadRenderers(v.sections); } catch (e) { el.innerHTML = `
${emptyState(e.status === 404 ? "search" : "alert", e.status === 404 ? "No such task" : "Couldn't open this task", esc(e.message), `${icon("arrowLeft", 15)}Open the environment`)}
`; return; } if (!el.isConnected) return; if (v.ref !== ref) { // an older address of this task (a row number): the page lives at its own ref ref = v.ref; history.replaceState(null, "", `${taskHref(spec, ref)}${qs.toString() ? `?${qs}` : ""}`); } const [org, name] = spec.split("/"); const c = v.collection, nav = v.nav; setMeta({ title: `${v.title} · ${name}`, description: `${v.title}: a task in ${spec} (${v.env.framework}). What the agent is asked, how it's graded, what it runs in, and its files.` }); const fileBlocks = v.sections.flatMap((s) => s.blocks.filter((b) => b.type === "files")); const ctx = { files: [], paths: new Set(fileBlocks.flatMap((b) => b.tree.map((f) => f.path))), renderers, spec, ref }; const sections = [...v.sections.map((s) => ({ ...s, html: blocks(s.blocks, ctx), count: s.blocks.find((b) => b.type === "files")?.tree.filter((f) => !f.dir).length })), { id: "rollouts", title: "Rollouts", icon: "play", note: "yours, and everyone's public ones", html: `
${sk.lines(70, 50)}
` }]; const where = nav?.subset ? `${subsetLabel(nav.subset)} #${fmt.format(nav.index)}` : v.ref || "(root)"; const same = (v.links || []).filter((l) => l.rel === "same"), related = (v.links || []).filter((l) => l.rel !== "same"); const kind = v.color ? `${v.icon ? icon(v.icon, 13) : ""}${esc(v.framework || v.env.framework)}` : `${esc(v.env.framework)}`; el.innerHTML = `
${c ? `${icon(c.icon, 13)}${esc(c.group)}` : ""} ${kind}${(v.chips || []).map((x) => `${esc(x)}`).join("")}

${esc(v.title)}

${icon("database", 14)}${esc(org)}/${esc(name)} ${v.id && v.id !== v.title && v.id !== where ? `${icon("flag", 14)}${esc(v.id)}` : ""} ${v.summary ? `${icon("target", 14)}${esc(v.summary)}` : ""} ${nav ? `${icon("list", 14)}${nav.label ? `${esc(nav.label)} ` : ""}${fmt.format(nav.index)}${nav.total != null ? ` of ${fmt.format(nav.total)}` : ""} ${icon("chevronRight", 14, "flip")} ${icon("chevronRight", 14)}` : ""} ${v.hub ? `${icon("external", 14)}${/\/viewer\//.test(v.hub) ? "In the Hub's viewer" : "On the Hub"}` : ""}
${same.length || related.length ? `` : ""}
${supportPanel(v.support)}
${sections.map((s) => `

${icon(s.icon || "list", 15)}${esc(s.title)}

${s.note ? `${esc(s.note)}` : ""}
${s.html}
`).join("")}
`; Object.values(renderers).forEach((r) => r.wire?.(el, ctx)); // files: one viewer per files block, the first keeps its open file and lines in the URL (?f=…&L=…) ctx.files.forEach((b, i) => { const raw = b.raw && b.hub ? b.hub.replace("/blob/", "/resolve/") : null; viewers.push(fileViewer($(`[data-files="${i}"]`, el), { files: b.tree, initial: i === 0 ? qs.get("f") : null, line: i === 0 ? qs.get("L") : null, load: (rel) => api(`${envApi(spec)}/file?ref=${encodeURIComponent(ref)}&f=${encodeURIComponent(rel)}`), listDir: async (dir) => (await api(`${envApi(spec)}/folder?ref=${encodeURIComponent(ref)}&f=${encodeURIComponent(dir)}`)).entries, hubUrl: (rel) => (b.hub ? b.hub + enc(rel) : null), rawUrl: (rel) => (raw ? raw + enc(rel) : null), onChange: i === 0 ? (rel, lines) => { const q = new URLSearchParams(location.search.slice(1) || ""); q.set("f", rel); lines ? q.set("L", lines) : q.delete("L"); history.replaceState(null, "", `${location.pathname}?${q}`); } : () => {}, })); }); if (qs.get("f") && viewers.length) requestAnimationFrame(() => $(".bl-files", el)?.closest(".tp-sec")?.scrollIntoView({ block: "start" })); const another = async () => { try { const r = await api(`${envApi(spec)}/random${nav?.subset ? `?subset=${encodeURIComponent(nav.subset)}` : ""}`); navigate(taskHref(spec, r.ref)); } catch (err) { toast(err.message); } }; const picks = new Set(); el.addEventListener("click", (e) => { const j = e.target.closest("[data-jump]"); if (j) { e.preventDefault(); $(`#sec-${CSS.escape(j.dataset.jump)}`, el)?.scrollIntoView({ behavior: "smooth", block: "start" }); return; } const of = e.target.closest("[data-open-file]"); if (of && viewers[0]) { viewers[0].open(of.dataset.openFile); $(".bl-files", el)?.closest(".tp-sec")?.scrollIntoView({ behavior: "smooth", block: "start" }); return; } if (e.target.closest("#copy")) navigator.clipboard?.writeText(location.href).then(() => toast("Link copied")); if (e.target.closest("#random")) another(); if (e.target.closest("[data-cmp-clear]")) { picks.clear(); syncPicks(el, spec, ref, picks); } }); el.addEventListener("change", (e) => { const cb = e.target.closest("[data-cmp]"); if (!cb) return; cb.checked ? picks.add(cb.dataset.cmp) : picks.delete(cb.dataset.cmp); syncPicks(el, spec, ref, picks); }); // ← and → step through the tasks of a list that has an order if (nav) { keys = (e) => { if (e.metaKey || e.ctrlKey || e.altKey || /INPUT|TEXTAREA|SELECT/.test(document.activeElement?.tagName) || document.activeElement?.isContentEditable || !$("#modal")?.hidden) return; const to = e.key === "ArrowLeft" ? nav.prev : e.key === "ArrowRight" ? nav.next : null; if (to != null) navigate(taskHref(spec, to)); }; addEventListener("keydown", keys); } runCard($("#run-card", el), v, spec, ref); taskRuns($("#t-runs", el), spec, ref, v, () => syncPicks(el, spec, ref, picks)); typeset($(".tp-head", el), $(".tp-body", el)); // math in the title and the prompt (KaTeX loads only if there is some) // the contents list follows the section in view const links = new Map([...el.querySelectorAll("[data-jump]")].map((a) => [`sec-${a.dataset.jump}`, a])); const seen = new Map(); observer = new IntersectionObserver((es) => { es.forEach((x) => seen.set(x.target.id, x.isIntersecting)); const first = [...links.keys()].find((id) => seen.get(id)); links.forEach((a, id) => a.classList.toggle("on", id === first)); }, { rootMargin: "-80px 0px -55% 0px" }); el.querySelectorAll(".tp-sec").forEach((s) => observer.observe(s)); } function glance(v) { const rows = v.glance || []; const withheld = v.withheld || [], cut = v.truncated || []; if (!rows.length && !withheld.length && !cut.length) return ""; return `

${icon("info", 14)}At a glance

${rows.length ? `
${rows.map(([a, b]) => `
${esc(a)}
${typeof b === "number" ? fmt.format(b) : inline(b)}
`).join("")}
` : ""} ${withheld.length ? `

${icon("shield", 12)} Withheld, as they hold the answer: ${withheld.slice(0, 12).map((k) => `${esc(k)}`).join(", ")}${withheld.length > 12 ? ` and ${withheld.length - 12} more` : ""}.

` : ""} ${cut.length ? `

Cut short by the dataset viewer: ${cut.map((k) => `${esc(k)}`).join(", ")}.

` : ""}
`; } // ── running it ─────────────────────────────────────────────────────────────── // The run panel, from the task's run options: one per runner that can run it, the default first. Each says whether // it can run this task (and why not), what it does, and the inputs it takes beyond the agent and the model. The model // comes from HF Inference Providers or, where the runner allows, your own OpenAI-compatible endpoint. Settings worth // keeping live in this browser; an endpoint's key stays in this tab only. const LS = { get: (k, d) => { try { return JSON.parse(localStorage.getItem(k)) ?? d; } catch { return d; } }, set: (k, v) => { try { localStorage.setItem(k, JSON.stringify(v)); } catch { /* private window */ } } }; const SS = { get: (k) => { try { return sessionStorage.getItem(k) || ""; } catch { return ""; } }, set: (k, v) => { try { sessionStorage.setItem(k, v); } catch { /* private window */ } } }; const BILLING = `huggingface.co/settings/billing`; const optValue = (x) => (x && typeof x === "object" ? x.value : x); const optLabel = (x) => (x && typeof x === "object" ? x.label ?? x.value : x); const tok = (n) => (n >= 1e6 ? (n / 1e6).toFixed(1) + "M" : n >= 1e3 ? Math.round(n / 1e3) + "k" : String(Math.round(n))); export async function runCard(box, v, spec, ref) { const opts = v.run?.options || []; const head = `

${icon("play", 14)}${opts.some((o) => o.ok || o.href) ? "Run a rollout" : "Run it"}

`; // nothing runs here: no promise of one if (!opts.length) { box.innerHTML = `

${icon("play", 14)}Run it

${inline(v.run?.note || "This task can't be run here.")}

`; return; } const saved = LS.get("run-runner", {})[spec]; let pick = opts.find((o) => o.runner === saved && o.ok) || opts.find((o) => o.default && o.ok) || opts.find((o) => o.ok) || opts[0]; const st = { source: LS.get("source", "hf"), probe: null, visibility: v.restricted ? "private" : "public", pickers: {} }; const s = getSession(); const choose = () => (opts.length > 1 ? `
Runner${opts.map((o) => ``).join("")}
` : ""); const about = (o) => `${o.about ? `

${inline(o.about)}

` : ""}${opts.length === 1 ? `

${icon("box", 12)} ${esc(o.label)}

` : ""} ${o.notes.map((n) => `

${icon("info", 12)} ${inline(n)}

`).join("")} ${o.warnings.map((w) => `
${icon("alert")}${inline(w)}
`).join("")}`; const render = async () => { const o = pick; if (o.href) { box.innerHTML = `${head}
${choose()}${about(o)}${icon("play", 15)}Open it
`; return; } if (!o.ok) { const external = v.support?.capabilities?.some((c) => c.id === "run" && c.state === "external"); const guide = v.sections?.find((sec) => sec.id === "run"); box.innerHTML = `${head}
${choose()}${about(o)}
${icon(external ? "info" : "alert")}${esc(o.why || "No runnable form is available here.")}
${guide ? `` : ""}
`; return; } if (!s.user) { box.innerHTML = `${head}
${choose()}${about(o)}

The ${o.sandbox ? "sandbox and the " : ""}model's tokens are billed to your Hugging Face account.

`; return; } box.innerHTML = `${head}
${spinner()}Loading models from Inference Providers…
`; let cat; try { cat = await getModels(); } catch (e) { box.innerHTML = `${head}
${icon("alert")}Couldn't reach HF Inference Providers: ${esc(e.message)}
`; $("#run-retry", box).addEventListener("click", render); return; } if (pick !== o) return; // another runner was picked meanwhile const prefs = LS.get("run-prefs", {}); const agents = cat.agents.filter((a) => !o.harnesses || o.harnesses.includes(a.id)); const agent = agents.some((a) => a.id === prefs.agent) ? prefs.agent : agents.some((a) => a.id === cat.default_agent) ? cat.default_agent : agents[0]?.id; const adv = LS.get(`run-adv:${o.runner}`, {}); const ep = LS.get("byo", { base_url: "", model: "", price_in: "", price_out: "" }); if (!o.endpoint) st.source = "hf"; const main = o.fields.filter((f) => !f.advanced), more = o.fields.filter((f) => f.advanced); const bill = s.billing || {}; const noCredit = o.sandbox && (bill.can_pay === false || bill.refused_at); const gaps = s.missing_scopes || []; box.innerHTML = `${head}
${choose()}${about(o)} ${o.endpoint ? `
` : ""} ${agents.length > 1 ? `` : ``}
${agents.length > 1 ? "Model" : "Agent model"} HF Inference Providers, $ per 1M tokens in / out
${o.endpoint ? `
Price per 1M tokens optional, only for the cost shown

Works with a LiteLLM proxy, vLLM, OpenAI, Together, OpenRouter and the like. The ${o.sandbox ? "sandbox still runs" : "rollout still runs"} on your Hugging Face account, so you sign in with HF either way. The key is used for this rollout only and never stored on the server, but the agent's own processes in the sandbox can see it: use a key you can revoke.

` : ""} ${main.map((f) => fieldHtml(f, adv)).join("")} ${more.length ? `
${icon("chevronRight", 14, "chev")}Advanced settings
${more.map((f) => fieldHtml(f, adv)).join("")}
${more.some((f) => f.help) ? `

${more.filter((f) => f.help).map((f) => `${esc(f.label)}: ${esc(f.help)}.`).join(" ")}

` : ""}
` : ""} ${o.estimate ? `
` : ""}
Visibility${["public", "private"].map((x) => ``).join("")}

${gaps.length ? `
${icon("alert")}Your sign-in may be missing ${esc(gaps.join(", "))}. If the rollout fails to start, sign in again with a write token.
` : ""} ${noCredit ? `
${icon("alert")}${bill.refused_at ? `Your last rollout couldn't start because your Hugging Face account had no prepaid credit for sandboxes. Add credit at ${BILLING}, then run again.` : `Your Hugging Face account can't pay for sandboxes yet, so a rollout would stop before it starts. Add prepaid credit at ${BILLING} first.`} Nothing is charged for a rollout that can't start.
` : ""}

Keeps running if you close this page; find it under My rollouts. ${o.sandbox ? "The sandbox is" : "The model's tokens are"} billed to ${esc(s.user.name)} on Hugging Face.

`; const pk = picker($("#run-model", box), { models: cat.models, value: prefs.model || cat.default_model, onChange: () => update(), note: "Every model here supports tool calling. Prices are per million tokens, input / output." }); st.pickers = {}; for (const f of o.fields.filter((x) => x.type === "model")) { const pool = (cat[f.pool] || []).map((m) => ({ ...m, featured: true })); if (!pool.length) continue; st.pickers[f.key] = picker($(`#run-f-${CSS.escape(f.key)}`, box), { models: pool, value: adv[f.key] || pool[0].id, onChange: () => update(), groupLabel: "Tested judges", note: f.pool === "vision_judges" ? "Tested on a real render with Xiaomi's rubric. Judges disagree, so compare scores only between runs graded by the same judge." : "Tested on this dataset's real rubric prompt: each returned a usable verdict on every check." }); } const val = (id) => { const e = $(id, box); return e ? e.value.trim() : ""; }; const num = (id) => { const x = val(id); return x === "" ? null : Number(x); }; const endpoint = () => ({ base_url: val("#ep-url"), api_key: val("#ep-key") || null, model: val("#ep-model"), price_in: num("#ep-pin"), price_out: num("#ep-pout") }); const fields = () => fieldValues(box, o.fields, st.pickers); const save = () => { if (o.endpoint) { const e = endpoint(); LS.set("byo", { base_url: e.base_url, model: e.model, price_in: val("#ep-pin"), price_out: val("#ep-pout") }); SS.set("byo-key", val("#ep-key")); } LS.set(`run-adv:${o.runner}`, fields()); LS.set("run-prefs", { ...LS.get("run-prefs", {}), agent: val("#run-agent"), model: pk.value }); }; const status = (kind, html) => { $("#ep-status", box).innerHTML = html ? `
${icon(kind === "ok" ? "check" : "alert")}${html}
` : ""; }; function update() { save(); $$("[data-src]", box).forEach((b) => b.setAttribute("aria-pressed", String(b.dataset.src === st.source))); $$("[data-pane]", box).forEach((p) => (p.hidden = p.dataset.pane !== st.source)); $$("[data-vis]", box).forEach((b) => b.setAttribute("aria-pressed", String(b.dataset.vis === st.visibility))); $("#run-vis-note", box).textContent = v.restricted ? "This dataset is private or gated, so its rollouts stay private." : st.visibility === "public" ? PUBLIC_NOTE : PRIVATE_NOTE; const fv = fields(); const changed = more.filter((f) => fv[f.key] != null && fv[f.key] !== f.default).map((f) => `${f.label.toLowerCase()} ${optLabel((f.options || []).find((x) => optValue(x) === fv[f.key])) ?? fv[f.key]}`); if ($("#adv-sum", box)) $("#adv-sum", box).textContent = changed.join(" · "); if (o.estimate) { const e = o.estimate; let price; if (st.source === "byo") { const x = endpoint(); price = x.price_in != null || x.price_out != null ? [x.price_in || 0, x.price_out || 0] : null; } else price = [pk.model.input, pk.model.output]; const minutes = fv.timeout_min ? Math.min(fv.timeout_min, e.minutes || 0) : e.minutes; const sandbox = o.sandbox ? (minutes / 60) * ((cat.prices || {})[e.flavor] ?? cat.sandbox_price_per_hour ?? 0.01) : 0; const model = price ? (e.tokens_in * price[0] + e.tokens_out * price[1]) / 1e6 : null; $("#run-est", box).innerHTML = `
Model · ${tok(e.tokens_in)} in, ${tok(e.tokens_out)} out${model == null ? "your provider's price" : money(model)}
${o.sandbox ? `
Sandbox · ~${minutes} min${money(sandbox)}
` : ""}
A typical rollout${model == null ? `${money(sandbox)} + tokens` : `~${money(model + sandbox)}`}
${e.tokens_in > 5e5 && st.source === "hf" ? `
Agents re-read their context every step, so input tokens dominate. A cheaper model makes a big difference here.
` : ""}`; } const go = $("#run-go", box); if (go && !go.dataset.busy) { const ready = st.source === "hf" || (st.probe?.ok && (!o.sandbox || st.probe.tools)); go.disabled = !ready; go.title = ready ? "" : "Test the connection first"; } } box.oninput = (e) => { if (e.target.closest(".byo") && !e.target.matches("#ep-pin, #ep-pout")) st.probe = null; update(); }; box.onchange = update; box.onclick = async (e) => { const r = e.target.closest("#run-runner [data-r]"); if (r) { pick = opts.find((x) => x.runner === r.dataset.r) || pick; LS.set("run-runner", { ...LS.get("run-runner", {}), [spec]: pick.runner }); render(); return; } const vb = e.target.closest("[data-vis]"); if (vb && !vb.disabled) { const was = st.visibility; st.visibility = vb.dataset.vis; update(); if (was === "private" && st.visibility === "public") { const rc = vb.getBoundingClientRect(); confetti(rc.left + rc.width / 2, rc.top); toast("Thank you for sharing with the community"); } return; } const b = e.target.closest("[data-src]"); if (b) { st.source = b.dataset.src; LS.set("source", st.source); update(); return; } if (e.target.closest("#ep-load")) { const btn = $("#ep-load", box); btn.disabled = true; try { const res = await api("/api/endpoints/models", { method: "POST", body: { base_url: val("#ep-url"), api_key: val("#ep-key") || null } }); $("#ep-models", box).innerHTML = res.models.map((m) => `