ritwika96's picture
Sync pipeline status
ea917ba verified
Raw History Blame Contribute Delete
12.8 kB
const FAKES = ["return None", "return 0", "return ''", "return []", "return False", "return first argument"];
const REASONS = {
api_error: ["The API call failed", "Beam never answered (network error, timeout or rate limit). These are retried on the next run."],
unparseable: ["No readable JSON", "Beam's reply didn't contain the problem in the required format."],
missing_fields: ["Missing pieces", "The reply lacked a title, prompt, solution or test list."],
solution_syntax: ["Solution doesn't parse", "The reference solution has a Python syntax error."],
entry_point_missing: ["Function not defined", "The solution never defines the function the tests call."],
disallowed_import: ["Unsafe import", "The solution imports a module outside the pure standard-library allowlist."],
too_few_valid_tests: ["Too few tests", "Fewer than six well-formed assert statements that call the function."],
timeout: ["Too slow", "Running the tests took longer than the time limit."],
solution_load_error: ["Solution crashes on load", "Defining the solution raised an error before any test ran."],
solution_fails_tests: ["Fails its own tests", "Beam's solution and Beam's tests disagree, so at least one of them is wrong."],
tests_too_weak: ["Tests too weak", "A fake answer like return None passed every test."],
duplicate: ["Duplicate", "Same problem statement as one already kept."],
};
const $ = (id) => document.getElementById(id);
const reduceMotion = window.matchMedia("(prefers-reduced-motion: reduce)").matches;
const fmt = (n) => Number(n || 0).toLocaleString("en-US");
const el = (tag, cls, text) => {
const e = document.createElement(tag);
if (cls) e.className = cls;
if (text != null) e.textContent = text;
return e;
};
function ago(iso) {
const s = Math.max(0, (Date.now() - new Date(iso).getTime()) / 1000);
if (s < 90) return "just now";
if (s < 5400) return `${Math.round(s / 60)} minutes ago`;
if (s < 129600) return `${Math.round(s / 3600)} hours ago`;
return `${Math.round(s / 86400)} days ago`;
}
function md(text) {
return DOMPurify.sanitize(marked.parse(text || ""));
}
function highlightIn(root) {
if (!window.hljs) return;
root.querySelectorAll("pre code").forEach((b) => {
if (!b.className) b.className = "language-python";
hljs.highlightElement(b);
});
}
/* ---------- header + funnel + ledger ---------- */
function renderStatus(status) {
const generated = status.funnel[0].count;
const kept = status.funnel[status.funnel.length - 1].count;
$("bar-status").textContent = `Updated ${ago(status.updated_at)}`;
const hs = $("headline-status");
hs.innerHTML = "";
if (!generated) {
hs.textContent = "No problems generated yet. Run the generator, then sync this page.";
return;
}
hs.append(el("strong", null, `${fmt(kept)} of ${fmt(generated)}`), ` problems Beam has written so far passed every check.`);
const funnel = $("funnel");
funnel.innerHTML = "";
for (const stage of status.funnel) {
const li = el("li");
const top = el("div", "f-top");
top.append(el("span", "f-label", stage.label), el("span", "f-count", fmt(stage.count)));
const bar = el("div", "f-bar");
const fill = el("span");
fill.style.width = `${generated ? (100 * stage.count) / generated : 0}%`;
bar.append(fill);
const body = el("div");
body.append(top, bar);
const lostEntries = Object.entries(stage.lost);
const lost = el("p", lostEntries.length ? "f-lost" : "f-lost none");
lost.textContent = lostEntries.length
? lostEntries.map(([r, n]) => `${fmt(n)} dropped: ${(REASONS[r] || [r])[0].toLowerCase()}`).join("; ")
: stage.key === "generated" ? "Every reply Beam sent back" : "None dropped here";
li.append(body, lost);
funnel.append(li);
}
const t = status.tokens || {};
const perRow = generated ? (t.completion || 0) / generated : 0;
const rows = [
["Beam output tokens", fmt(t.completion), `${fmt(t.reasoning)} of them spent thinking`],
["Prompt tokens", fmt(t.prompt), null],
["Output tokens per problem", fmt(Math.round(perRow)), perRow ? `about ${fmt(Math.floor(1e6 / perRow))} problems per million output tokens` : null],
["Typical call", status.median_seconds_per_call != null ? `${status.median_seconds_per_call}s` : "–", null],
["Served by", Object.entries(status.providers).map(([p, n]) => `${p} (${fmt(n)})`).join(", ") || "–", null],
];
const dl = el("dl");
for (const [k, v, note] of rows) {
const dd = el("dd", null, v);
if (note) dd.append(el("small", null, note));
dl.append(el("dt", null, k), dd);
}
$("ledger").replaceChildren(dl);
}
/* ---------- coverage ---------- */
function renderCoverage(status) {
const topics = Object.keys(status.coverage);
const half = Math.ceil(topics.length / 2);
const wrap = $("coverage");
wrap.innerHTML = "";
for (const chunk of [topics.slice(0, half), topics.slice(half)]) {
const col = el("div", "cov-col");
col.append(el("span"));
for (const d of status.difficulties) col.append(el("span", "cov-h", d));
for (const topic of chunk) {
const label = el("span", "cov-t", topic);
label.title = topic;
col.append(label);
for (const d of status.difficulties) {
const n = status.coverage[topic][d];
const c = el("span", n ? "cov-c has" : "cov-c", n ? String(n) : "");
c.setAttribute("aria-label", `${topic}, ${d}: ${n} problems`);
col.append(c);
}
}
wrap.append(col);
}
}
/* ---------- rejections ---------- */
function renderRejected(status) {
const box = $("rejected");
const entries = Object.entries(status.reasons);
if (!entries.length) {
box.replaceChildren(el("p", "rej-empty", "Nothing rejected yet. Every problem Beam has written so far passed all the checks."));
return;
}
const ul = el("ul", "rej-list");
for (const [r, n] of entries) {
const [why, desc] = REASONS[r] || [r, ""];
const li = el("li");
const text = el("div");
text.append(el("div", "rej-why", why), el("div", "rej-desc", desc));
li.append(el("span", "rej-n", fmt(n)), text);
ul.append(li);
}
box.replaceChildren(ul);
}
/* ---------- specimen ---------- */
let specTimers = [];
let specIndex = -1;
function signatureOf(p) {
if (p.signature) return p.signature;
const m = p.solution.match(new RegExp(`def ${p.entry_point}\\([^)]*\\)[^:]*:`));
return m ? m[0] : `def ${p.entry_point}(...)`;
}
function showSpecimen(problems) {
specTimers.forEach(clearTimeout);
specTimers = [];
if (!problems.length) {
$("specimen").replaceChildren(el("p", "spec-note", "The first verified problem will appear here."));
return;
}
specIndex = problems.length === 1 ? 0 : (specIndex + 1 + Math.floor(Math.random() * (problems.length - 1))) % problems.length;
const p = problems[specIndex];
const tests = [...p.public_tests, ...p.hidden_tests];
$("spec-title").textContent = p.title;
$("spec-tags").textContent = `${p.difficulty}, ${p.topic}`;
$("spec-sig").textContent = signatureOf(p);
$("spec-next").hidden = problems.length < 2;
const cells = $("spec-cells");
cells.innerHTML = "";
tests.forEach((t, i) => {
const li = el("li", null, "✓");
li.title = t;
li.setAttribute("aria-label", `Test ${i + 1} passed`);
cells.append(li);
});
const fakes = $("spec-fakes");
fakes.innerHTML = "";
FAKES.forEach((f) => fakes.append(el("li", null, f)));
const note = $("spec-pass-note");
note.textContent = "";
const cellEls = [...cells.children];
const fakeEls = [...fakes.children];
const finish = () => (note.textContent = `${tests.length} of ${tests.length} tests pass. ${p.public_tests.length} are shown to the solver, ${p.hidden_tests.length} are hidden for scoring.`);
if (reduceMotion) {
cellEls.forEach((c) => c.classList.add("on"));
fakeEls.forEach((f) => f.classList.add("caught"));
finish();
return;
}
cellEls.forEach((c, i) => specTimers.push(setTimeout(() => c.classList.add("on"), 300 + i * 85)));
const after = 300 + cellEls.length * 85 + 250;
specTimers.push(setTimeout(finish, after));
fakeEls.forEach((f, i) => specTimers.push(setTimeout(() => f.classList.add("caught"), after + 200 + i * 160)));
}
/* ---------- browser ---------- */
const state = { q: "", diff: "all", current: null };
function renderSeg(difficulties) {
const seg = $("diff-seg");
for (const d of ["all", ...difficulties]) {
const b = el("button", null, d === "all" ? "All" : d[0].toUpperCase() + d.slice(1));
b.type = "button";
b.setAttribute("role", "radio");
b.setAttribute("aria-checked", String(d === state.diff));
b.onclick = () => {
state.diff = d;
seg.querySelectorAll("button").forEach((x) => x.setAttribute("aria-checked", String(x === b)));
renderList();
};
seg.append(b);
}
}
let PROBLEMS = [];
function renderList() {
const q = state.q.trim().toLowerCase();
const items = PROBLEMS.filter(
(p) => (state.diff === "all" || p.difficulty === state.diff) && (!q || `${p.title} ${p.topic}`.toLowerCase().includes(q))
);
const ul = $("plist");
ul.innerHTML = "";
if (!items.length) {
ul.append(el("li", "none", PROBLEMS.length ? "No problems match. Clear the search or pick another difficulty." : "No verified problems yet."));
return;
}
for (const p of items) {
const li = el("li");
const b = el("button");
b.type = "button";
b.setAttribute("aria-current", String(p.id === state.current));
b.append(el("span", "p-title", p.title), el("span", "p-meta", `${p.difficulty}, ${p.topic}, ${p.num_tests} tests`));
b.onclick = () => showDetail(p);
li.append(b);
ul.append(li);
}
if (!items.some((p) => p.id === state.current)) showDetail(items[0]);
}
function testList(tests) {
const ul = el("ul", "tests");
tests.forEach((t) => ul.append(el("li", null, t)));
return ul;
}
function showDetail(p) {
state.current = p.id;
$("plist").querySelectorAll("button").forEach((b) => b.setAttribute("aria-current", String(b.querySelector(".p-title")?.textContent === p.title)));
const d = $("detail");
d.innerHTML = "";
d.append(el("p", "d-meta", `${p.difficulty}, ${p.topic}. Written by ${p.source_model} at temperature ${p.temperature}.`));
const prose = el("div", "prose");
prose.innerHTML = md(p.prompt);
d.append(prose);
const tabs = el("div", "tabs");
tabs.setAttribute("role", "tablist");
const panel = el("div", "tabpanel");
panel.setAttribute("role", "tabpanel");
const views = {
"Reference solution": () => {
const pre = el("pre");
pre.append(el("code", "language-python", p.solution));
return pre;
},
[`Tests shown to the solver (${p.public_tests.length})`]: () => testList(p.public_tests),
[`Hidden tests (${p.hidden_tests.length})`]: () => testList(p.hidden_tests),
};
Object.keys(views).forEach((name, i) => {
const b = el("button", null, name);
b.type = "button";
b.setAttribute("role", "tab");
b.setAttribute("aria-selected", String(i === 0));
b.onclick = () => {
tabs.querySelectorAll("button").forEach((x) => x.setAttribute("aria-selected", String(x === b)));
panel.replaceChildren(views[name]());
highlightIn(panel);
};
tabs.append(b);
});
panel.append(Object.values(views)[0]());
d.append(tabs, panel);
highlightIn(d);
}
/* ---------- footer + boot ---------- */
function renderFoot(status) {
const f = $("foot");
const p = el("p");
p.append(
"Generated with ",
Object.assign(el("a", null, status.model), { href: "https://huggingface.co/reflection/Beam-1.0-501B-A23B" }),
" for the Build with Beam challenge. Solutions run in a separate Python process with CPU, memory and time limits, and may import only pure standard-library modules."
);
f.replaceChildren(p);
}
async function boot() {
try {
const bust = `?t=${Date.now()}`;
const [status, problems] = await Promise.all([
fetch(`data/status.json${bust}`).then((r) => r.json()),
fetch(`data/problems.json${bust}`).then((r) => r.json()),
]);
PROBLEMS = problems;
renderStatus(status);
renderCoverage(status);
renderRejected(status);
renderFoot(status);
showSpecimen(problems);
$("spec-next").onclick = () => showSpecimen(problems);
renderSeg(status.difficulties);
$("q").addEventListener("input", (e) => {
state.q = e.target.value;
renderList();
});
renderList();
} catch (err) {
$("bar-status").textContent = "Couldn't load the run data";
document.querySelector("main").replaceChildren(
el("p", "load-error", `The status files in data/ didn't load (${err.message}). Run beamcode.space_sync to regenerate them.`)
);
}
}
document.addEventListener("DOMContentLoaded", boot);