Spaces:
Running on Zero
Running on Zero
| // GraphTrace Explorer. Every string from the API is inserted as text, never as HTML. | |
| function h(tag, attrs = {}, ...kids) { | |
| const n = document.createElement(tag); | |
| for (const [k, v] of Object.entries(attrs || {})) { | |
| if (v == null || v === false) continue; | |
| if (k === "class") n.className = v; | |
| else if (k.startsWith("on")) n.addEventListener(k.slice(2), v); | |
| else n.setAttribute(k, v === true ? "" : v); | |
| } | |
| for (const kid of kids.flat()) if (kid != null && kid !== false) n.append(kid instanceof Node ? kid : document.createTextNode(String(kid))); | |
| return n; | |
| } | |
| const NS = "http://www.w3.org/2000/svg"; | |
| function s(tag, attrs = {}, parent) { | |
| const n = document.createElementNS(NS, tag); | |
| for (const [k, v] of Object.entries(attrs)) n.setAttribute(k, v); | |
| if (parent) parent.appendChild(n); | |
| return n; | |
| } | |
| const icon = (name) => h("i", { class: `ph ph-${name}`, "aria-hidden": "true" }); | |
| const pct = (v) => (v == null || Number.isNaN(v) ? "n/a" : `${(100 * v).toFixed(1)}`); | |
| const app = document.getElementById("app"); | |
| const TYPES = ["Paper", "Method", "Dataset", "Task", "Metric"]; | |
| const typeColour = (t) => `var(--series-${TYPES.indexOf(t) + 1 || 6})`; | |
| const CATS = { single_hop: "Single hop", multi_hop: "Multi hop", false_premise: "False premise", conflict_or_insufficient: "Conflict / insufficient" }; | |
| // ---------- theme ---------- | |
| const root = document.documentElement; | |
| try { const t = localStorage.getItem("gt-theme"); if (t) root.dataset.theme = t; } catch {} | |
| document.getElementById("theme-toggle").addEventListener("click", () => { | |
| const dark = root.dataset.theme ? root.dataset.theme === "dark" : matchMedia("(prefers-color-scheme: dark)").matches; | |
| root.dataset.theme = dark ? "light" : "dark"; | |
| try { localStorage.setItem("gt-theme", root.dataset.theme); } catch {} | |
| }); | |
| // ---------- data ---------- | |
| async function api(path, opts) { | |
| const r = await fetch(`/api${path}`, opts); | |
| if (!r.ok) { | |
| const body = await r.text(); | |
| let detail = body; | |
| try { detail = JSON.parse(body).detail ?? body; } catch {} | |
| const err = new Error(typeof detail === "string" ? detail : JSON.stringify(detail)); | |
| err.status = r.status; | |
| throw err; | |
| } | |
| return r.json(); | |
| } | |
| const maybe = (p) => p.catch((e) => (e.status === 404 ? null : Promise.reject(e))); | |
| function emptyState(iconName, title, text, commands) { | |
| return h("div", { class: "empty" }, icon(iconName), h("b", {}, title), text ? h("span", {}, text) : null, commands ? h("pre", {}, commands) : null); | |
| } | |
| function errorState(e) { | |
| return h("div", { class: "empty error" }, icon("warning-circle"), h("b", {}, "Could not load this view"), h("span", {}, String(e.message || e))); | |
| } | |
| // ---------- routing: #/ #/bench[/<item>] #/graph[/<node>] #/try[/<qid>] ---------- | |
| function parseHash() { | |
| const parts = location.hash.replace(/^#/, "").split("?")[0].split("/").filter(Boolean).map(decodeURIComponent); | |
| return { view: parts[0] || "overview", id: parts[1] || null }; | |
| } | |
| async function route() { | |
| closeDrawer(); | |
| const { view, id } = parseHash(); | |
| document.querySelectorAll(".tabs a").forEach((a) => (a.dataset.view === view ? a.setAttribute("aria-current", "page") : a.removeAttribute("aria-current"))); | |
| app.replaceChildren(h("div", { class: "skeleton", style: "height:140px" })); | |
| try { | |
| if (view === "bench") await renderBench(id); | |
| else if (view === "graph") await renderGraph(id); | |
| else if (view === "try") await renderTry(id); | |
| else await renderOverview(); | |
| } catch (e) { | |
| app.replaceChildren(errorState(e)); | |
| } | |
| } | |
| // ---------- overview ---------- | |
| function stats(pairs) { | |
| return h("div", { class: "stats" }, pairs.map(([k, v]) => h("div", { class: "stat" }, h("div", { class: "v" }, v ?? "n/a"), h("div", { class: "k" }, k)))); | |
| } | |
| function bars(entries, colourOf) { | |
| const max = Math.max(1, ...entries.map(([, v]) => v)); | |
| return h("div", { class: "bars" }, entries.map(([k, v]) => h("div", { class: "row" }, | |
| h("span", {}, k), h("span", {}, h("div", { class: "bar", style: `width:${(100 * v) / max}%;${colourOf ? `background:${colourOf(k)}` : ""}` })), h("span", { class: "num" }, v)))); | |
| } | |
| function ciCell(c) { | |
| if (!c || c.mean == null || Number.isNaN(c.mean)) return h("td", { class: "num" }, "n/a"); | |
| return h("td", { class: "num" }, pct(c.mean), c.low != null ? h("span", { class: "ci" }, `${pct(c.low)}-${pct(c.high)}`) : null); | |
| } | |
| async function renderOverview() { | |
| const health = await api("/health"); | |
| const [corpus, graph, items, runs] = await Promise.all([ | |
| maybe(api("/corpus/stats")), health.graph ? api("/graph/stats") : null, health.benchmark ? maybe(api("/bench/items")) : null, api("/runs"), | |
| ]); | |
| const out = [h("div", { class: "page-head" }, | |
| h("h1", {}, "GraphTrace-Bench"), | |
| h("p", {}, "Can an LLM answer multi-step questions about AI research through a verifiable knowledge path, and recognise when the evidence is missing, conflicting or based on a false premise? Every edge in the graph carries the exact quote it was extracted from."))]; | |
| // Corpus | |
| out.push(h("section", { class: "section" }, h("h2", {}, "Corpus"), | |
| corpus ? [ | |
| h("p", { class: "lede" }, `QASPER papers (CC-BY-4.0) selected for cross-paper connectivity through shared benchmark datasets.`), | |
| stats([["Papers", corpus.stats.selected_papers], ["Chunks", corpus.stats.chunks ?? null], ["Datasets in selection", corpus.stats.datasets_in_selection], | |
| ["Shared datasets", corpus.stats.datasets_shared_within_selection], ["Paper pairs sharing a dataset", corpus.stats.paper_pairs_sharing_a_dataset], ["Components", corpus.stats.connected_components]]), | |
| corpus.stats.top_datasets?.length ? h("div", { class: "panel", style: "margin-top:16px" }, h("h3", { style: "margin-bottom:10px" }, "Benchmarks mentioned by the most selected papers"), | |
| bars(corpus.stats.top_datasets.slice(0, 10).map(([name, n]) => [name, n]))) : null, | |
| ] : emptyState("books", "Corpus not prepared", null, "uv run graphtrace corpus"))); | |
| // Graph | |
| out.push(h("section", { class: "section" }, h("h2", {}, "Knowledge graph"), | |
| graph ? [ | |
| h("p", { class: "lede" }, "Closed schema. Every edge is backed by a quote anchored at exact offsets in its source chunk; unanchored extractions are rejected."), | |
| stats([["Nodes", graph.nodes], ["Edges", graph.edges], ["Distinct facts", graph.distinct_facts], ["Result records", graph.results], ["Components", graph.components], ["Largest component", graph.largest_component_nodes]]), | |
| h("div", { class: "grid-2", style: "margin-top:16px" }, | |
| h("div", { class: "panel" }, h("h3", { style: "margin-bottom:10px" }, "Nodes by type"), bars(TYPES.filter((t) => graph.nodes_by_type[t]).map((t) => [t, graph.nodes_by_type[t]]), typeColour)), | |
| h("div", { class: "panel" }, h("h3", { style: "margin-bottom:10px" }, "Edges by relation"), bars(Object.entries(graph.edges_by_relation).sort((a, b) => b[1] - a[1])))), | |
| ] : emptyState("graph", "No graph extracted yet", "Extraction calls the configured model on every chunk. Estimate the cost first.", "uv run graphtrace extract\nuv run graphtrace build-graph"))); | |
| // Benchmark | |
| let benchBody; | |
| if (items && items.length) { | |
| const byCat = {}, byStatus = {}; | |
| for (const it of items) { byCat[it.category] = (byCat[it.category] || 0) + 1; byStatus[it.review_status] = (byStatus[it.review_status] || 0) + 1; } | |
| benchBody = [ | |
| h("p", { class: "lede" }, "Generated from the graph, then reviewed by a person against the source text. Only accepted items enter the released benchmark."), | |
| h("div", { class: "grid-2" }, | |
| h("div", { class: "panel" }, h("h3", { style: "margin-bottom:10px" }, "Items by category"), bars(Object.keys(CATS).filter((c) => byCat[c]).map((c) => [CATS[c], byCat[c]]))), | |
| h("div", { class: "panel" }, h("h3", { style: "margin-bottom:10px" }, "Review status"), bars(["accepted", "edited", "rejected", "unreviewed"].filter((k) => byStatus[k]).map((k) => [k, byStatus[k]])))), | |
| ]; | |
| } else { | |
| benchBody = emptyState("list-checks", "No benchmark items yet", "Items are generated from the extracted graph and then reviewed by hand.", "uv run graphtrace generate-bench\nuv run graphtrace review --reviewer <initials>\nuv run graphtrace freeze"); | |
| } | |
| out.push(h("section", { class: "section" }, h("h2", {}, "Benchmark"), benchBody)); | |
| // Results | |
| const evaluated = runs.filter((r) => r.has_metrics); | |
| const res = h("section", { class: "section" }, h("h2", {}, "Results")); | |
| if (!evaluated.length) { | |
| res.append(emptyState("hourglass-medium", "No evaluated runs yet", "Methods are compared with the same base model, output schema and context budget.", "uv run graphtrace estimate-cost configs/experiments/main.yaml\nuv run graphtrace run configs/experiments/main.yaml\nuv run graphtrace evaluate configs/experiments/main.yaml --judge")); | |
| } else { | |
| for (const r of evaluated) { | |
| const m = await api(`/runs/${encodeURIComponent(r.name)}/metrics`); | |
| const cols = [["correct", "Accuracy"], ["path_all_valid", "Path valid"], ["path_completeness", "Path complete"], ["grounding_precision", "Grounding prec."], ["citation_rate", "Cites sources"], ["false_premise_rejected", "False premise rejected"], ["conflict_flagged", "Conflict flagged"]]; | |
| res.append(h("h3", { style: "margin:14px 0 4px" }, r.name, r.smoke_test ? h("span", { class: "badge warn", style: "margin-left:8px" }, icon("warning"), "Smoke test on unreviewed items: not a result") : null)); | |
| if (m.judge_status) res.append(h("p", { style: "color:var(--text-3);font-size:12px;margin-bottom:8px" }, `Evaluation judge: ${m.judge_status}`)); | |
| const table = (title, block) => block && Object.keys(block).length ? h("div", { class: "table-wrap", style: "margin-top:10px" }, h("table", {}, | |
| h("thead", {}, h("tr", {}, h("th", {}, title), h("th", { class: "num" }, "n"), h("th", { class: "num" }, "Errors"), cols.map(([, t]) => h("th", { class: "num" }, t)), h("th", { class: "num" }, "ECE"), h("th", { class: "num" }, "AUROC"))), | |
| h("tbody", {}, Object.entries(block).map(([name, a]) => h("tr", {}, h("td", {}, name), h("td", { class: "num" }, a.n), h("td", { class: "num" }, a.errors ?? 0), cols.map(([k]) => ciCell(a[k])), | |
| h("td", { class: "num" }, m.calibration?.[name]?.ece?.toFixed(3) ?? "n/a"), h("td", { class: "num" }, m.calibration?.[name]?.auroc?.toFixed(3) ?? "n/a")))))) : null; | |
| res.append(table(`Headline (${(m.headline_categories || []).map((c) => CATS[c] || c).join(", ") || "all"})`, m.headline || m.methods)); | |
| res.append(table("Conflict / insufficient items (reported separately)", m.conflict_insufficient)); | |
| const cats = [...new Set(Object.values(m.by_category).flatMap((v) => Object.keys(v)))]; | |
| res.append(h("div", { class: "table-wrap", style: "margin-top:10px" }, h("table", {}, | |
| h("thead", {}, h("tr", {}, h("th", {}, "Accuracy by category"), cats.map((c) => h("th", { class: "num" }, CATS[c] || c)))), | |
| h("tbody", {}, Object.entries(m.by_category).map(([name, by]) => h("tr", {}, h("td", {}, name), cats.map((c) => ciCell(by[c]?.correct)))))))); | |
| } | |
| } | |
| out.push(res); | |
| if (!health.graph) out.push(h("p", { style: "color:var(--text-3);font-size:12px;margin-top:24px" }, "The Benchmark and Graph views fill in once the graph has been extracted. ", h("a", { href: "#/try" }, "Try it"), " runs the retrieval and verification stages live on the corpus today, with no model calls.")); | |
| app.replaceChildren(...out); | |
| } | |
| // ---------- shared bits ---------- | |
| function ent(name, type) { | |
| return h("span", { class: "ent" }, type ? h("span", { class: "type-dot", style: `background:${typeColour(type)}`, title: type }) : null, name); | |
| } | |
| function tripleView(sName, rel, oName, sType, oType) { | |
| return h("div", { class: "triple" }, ent(sName, sType), h("span", { class: "rel" }, rel), icon("arrow-right"), ent(oName, oType)); | |
| } | |
| function sourceLink(chunkId, quoteRange) { | |
| return h("button", { type: "button", class: "src-link", onclick: () => openSource(chunkId, quoteRange) }, icon("file-text"), " ", chunkId); | |
| } | |
| // ---------- benchmark ---------- | |
| let benchFilter = { cat: "all", status: "all", text: "" }; | |
| async function renderBench(itemId) { | |
| const [health, items] = await Promise.all([api("/health"), maybe(api("/bench/items"))]); | |
| if (!items || !items.length) { | |
| app.replaceChildren(emptyState("list-checks", "No benchmark items yet", "Generate candidates from the extracted graph, then review them.", "uv run graphtrace generate-bench\nuv run graphtrace review --reviewer <initials>")); | |
| return; | |
| } | |
| if (!itemId) itemId = items[0].item_id; | |
| const list = h("div", { class: "qitems", role: "list" }); | |
| const count = h("div", { class: "qcount" }); | |
| const search = h("input", { type: "search", placeholder: "Filter questions", "aria-label": "Filter questions", value: benchFilter.text }); | |
| const catSel = h("select", { "aria-label": "Category" }, h("option", { value: "all" }, "All categories"), Object.entries(CATS).map(([k, v]) => h("option", { value: k, selected: benchFilter.cat === k }, v))); | |
| const stSel = h("select", { "aria-label": "Review status" }, ["all", "unreviewed", "accepted", "edited", "rejected"].map((k) => h("option", { value: k, selected: benchFilter.status === k }, k === "all" ? "Any status" : k))); | |
| const fill = () => { | |
| const t = benchFilter.text.toLowerCase(); | |
| const shown = items.filter((i) => (benchFilter.cat === "all" || i.category === benchFilter.cat) && (benchFilter.status === "all" || i.review_status === benchFilter.status) && (!t || i.question.toLowerCase().includes(t))); | |
| list.replaceChildren(...shown.map((i) => h("button", { class: "qitem", role: "listitem", "aria-current": String(i.item_id === itemId), onclick: () => (location.hash = `#/bench/${encodeURIComponent(i.item_id)}`) }, | |
| h("div", { class: "q" }, i.question), h("div", { class: "t" }, `${CATS[i.category] || i.category}, ${i.difficulty} hop(s), ${i.review_status}`)))); | |
| count.textContent = `${shown.length} of ${items.length} items`; | |
| }; | |
| search.addEventListener("input", () => { benchFilter.text = search.value; fill(); }); | |
| catSel.addEventListener("change", () => { benchFilter.cat = catSel.value; fill(); }); | |
| stSel.addEventListener("change", () => { benchFilter.status = stSel.value; fill(); }); | |
| fill(); | |
| const detail = h("div", {}, h("div", { class: "skeleton", style: "height:200px" })); | |
| app.replaceChildren(h("div", { class: "browser" }, h("div", { class: "qlist" }, h("div", { class: "qlist-tools" }, search, h("div", { class: "btn-row" }, catSel, stSel)), list, count), detail)); | |
| const [d, runs] = await Promise.all([api(`/bench/items/${encodeURIComponent(itemId)}`), api("/runs")]); | |
| detail.replaceChildren(...itemDetail(d, health, runs)); | |
| } | |
| function statusBadge(st) { | |
| const m = { accepted: ["good", "check-circle"], edited: ["good", "pencil-simple"], rejected: ["bad", "x-circle"], unreviewed: ["warn", "hourglass-medium"] }[st] || ["neutral", "circle"]; | |
| return h("span", { class: `badge ${m[0]}` }, icon(m[1]), st); | |
| } | |
| function itemDetail(d, health, runs) { | |
| const it = d.item; | |
| const gold = it.gold_decision === "answer" ? it.gold_answer.join(", ") : it.gold_decision === "abstain" ? "Abstain: the evidence does not answer this" : "Flag the conflict between sources"; | |
| const head = h("div", { class: "qhead" }, | |
| h("h1", {}, it.question), | |
| h("div", { class: "cites" }, h("span", { class: "chip" }, CATS[it.category] || it.category), h("span", { class: "chip" }, `${it.difficulty} hop(s)`), h("span", { class: "chip mono" }, it.template_id), statusBadge(it.review_status)), | |
| h("dl", { class: "kv", style: "margin-top:6px" }, h("dt", {}, "Gold"), h("dd", {}, h("b", {}, gold)), it.annotation_notes ? [h("dt", {}, "Notes"), h("dd", {}, it.annotation_notes)] : null, | |
| it.reviewer ? [h("dt", {}, "Reviewed by"), h("dd", {}, `${it.reviewer}${it.review_notes ? `: ${it.review_notes}` : ""}`)] : null)); | |
| const steps = d.path.length ? d.path.map((p) => h("div", { class: "step" }, | |
| p.kind === "edge" ? tripleView(p.subject_name, p.relation, p.object_name) : tripleView(p.method_name, `reports ${p.value}`, p.dataset_name), | |
| h("blockquote", { class: "q" }, `"${p.quote}"`), | |
| h("div", { style: "display:flex;gap:12px;flex-wrap:wrap;align-items:center" }, sourceLink(p.source_chunk_id, [p.start_char, p.end_char]), | |
| p.paper_title ? h("span", { style: "color:var(--text-3);font-size:12px" }, p.paper_title) : null, | |
| p.confidence != null ? h("span", { class: "badge neutral" }, `support ${p.confidence.toFixed(2)}`) : null))) | |
| : [h("p", { style: "color:var(--text-2)" }, "No required path: the correct behaviour is to notice that the graph and sources do not support the question.")]; | |
| const out = [head, h("section", { class: "section" }, h("h2", { style: "margin-bottom:6px" }, "Required path and its evidence"), h("div", { class: "panel" }, steps))]; | |
| if (health.review_enabled) out.push(h("section", { class: "section" }, h("h2", { style: "margin-bottom:6px" }, "Review against the source text"), reviewForm(it))); | |
| if (runs.length) { | |
| const sel = h("select", { "aria-label": "Run" }, runs.map((r) => h("option", { value: r.name }, r.name))); | |
| const host = h("div", { class: "answers" }); | |
| const load = async () => { | |
| host.replaceChildren(h("div", { class: "skeleton", style: "height:120px" })); | |
| try { | |
| const r = await api(`/runs/${encodeURIComponent(sel.value)}/items/${encodeURIComponent(it.item_id)}`); | |
| const entries = Object.entries(r.outputs); | |
| host.replaceChildren(...(entries.length ? entries.map(([m, o]) => outputCard(m, o)) : [emptyState("hourglass-medium", "This run has no prediction for this item.")])); | |
| } catch (e) { host.replaceChildren(errorState(e)); } | |
| }; | |
| sel.addEventListener("change", load); | |
| out.push(h("section", { class: "section" }, h("div", { class: "toolbar" }, h("h2", {}, "Method outputs"), sel), host)); | |
| load(); | |
| } | |
| return out; | |
| } | |
| function outputCard(method, o) { | |
| const pc = o.path_check || { checks: [] }; | |
| const decision = { answer: ["neutral", "chat-text", "Answered"], abstain: ["warn", "hand", "Abstained"], flag_conflict: ["warn", "warning-diamond", "Flagged conflict"] }[o.decision] || ["neutral", "circle", o.decision]; | |
| return h("article", { class: "answer-card" }, | |
| h("header", {}, h("span", { class: "sys" }, method), h("span", { class: `badge ${decision[0]}` }, icon(decision[1]), decision[2])), | |
| o.error ? h("div", { class: "callout warn" }, icon("warning"), `Run error: ${o.error}`) : null, | |
| h("div", { class: "answer-text" }, o.answer?.length ? o.answer.join(", ") : "(no answer)"), | |
| h("div", { class: "usage" }, h("span", {}, `confidence ${o.confidence?.toFixed?.(2) ?? "n/a"}`), h("span", {}, `${o.context_tokens} ctx tokens`)), | |
| pc.checks.length ? h("div", { class: "stmt" }, h("b", { style: "font-size:12.5px" }, "Claimed path"), | |
| pc.checks.map((c) => h("div", { class: "cites" }, tripleView(c.triple[0], c.triple[1], c.triple[2]), | |
| c.valid || (c.exists && c.supported) ? h("span", { class: "badge good" }, icon("check-circle"), "in graph, supported") : h("span", { class: "badge bad" }, icon("x-circle"), c.problem || "invalid"))), | |
| h("div", { class: "cites" }, | |
| h("span", { class: `badge ${pc.connected ? "good" : "bad"}` }, pc.connected ? "connected" : "not connected"), | |
| h("span", { class: `badge ${pc.touches_question ? "good" : "warn"}` }, pc.touches_question ? "starts at the question" : "does not start at the question"), | |
| h("span", { class: `badge ${pc.covers_answer ? "good" : "warn"}` }, pc.covers_answer ? "reaches the answer" : "does not reach the answer"))) | |
| : h("div", { class: "stmt" }, h("span", { style: "color:var(--text-3);font-size:12.5px" }, "No path claimed.")), | |
| o.source_chunk_ids?.length ? h("div", { class: "cites" }, o.source_chunk_ids.map((c) => sourceLink(c))) : null, | |
| o.verifier ? h("details", { class: "more" }, h("summary", {}, "Verifier decision"), h("pre", { style: "white-space:pre-wrap;font:12px var(--mono);margin:0" }, JSON.stringify(o.verifier, null, 2))) : null); | |
| } | |
| function reviewForm(it) { | |
| let reviewer = ""; | |
| try { reviewer = localStorage.getItem("gt-reviewer") || ""; } catch {} | |
| const who = h("input", { value: reviewer, required: true, autocomplete: "off" }); | |
| const notes = h("textarea", { rows: 2 }); | |
| const question = h("input", { value: it.question }); | |
| const decision = h("select", {}, ["answer", "abstain", "flag_conflict"].map((k) => h("option", { value: k, selected: it.gold_decision === k }, k))); | |
| const answer = h("input", { value: it.gold_answer.join("; ") }); | |
| const msg = h("div", { role: "status", style: "font-size:12.5px;color:var(--text-2)" }); | |
| const send = async (status) => { | |
| if (!who.value.trim()) { msg.textContent = "Enter your initials first."; who.focus(); return; } | |
| try { localStorage.setItem("gt-reviewer", who.value.trim()); } catch {} | |
| const body = { status, reviewer: who.value.trim(), notes: notes.value }; | |
| if (status === "edited") Object.assign(body, { question: question.value, gold_decision: decision.value, gold_answer: answer.value.split(";").map((x) => x.trim()).filter(Boolean) }); | |
| try { | |
| await api(`/bench/items/${encodeURIComponent(it.item_id)}/review`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) }); | |
| msg.textContent = `Saved as ${status}.`; | |
| route(); | |
| } catch (e) { msg.textContent = `Not saved: ${e.message}`; } | |
| }; | |
| return h("div", { class: "panel review-form" }, | |
| h("p", { style: "color:var(--text-2);font-size:13px" }, "Open each quote's source above and check that the passage really states the edge. Accept only items whose gold answer follows from the sources."), | |
| h("label", {}, "Reviewer initials", who), h("label", {}, "Notes", notes), | |
| h("details", { class: "more" }, h("summary", {}, "Edit before accepting"), h("div", { class: "review-form" }, h("label", {}, "Question", question), h("label", {}, "Gold decision", decision), h("label", {}, "Gold answer (separate with ;)", answer), | |
| h("div", { class: "btn-row" }, h("button", { type: "button", class: "btn", onclick: () => send("edited") }, icon("pencil-simple"), "Save edit")))), | |
| h("div", { class: "btn-row" }, h("button", { type: "button", class: "btn primary", onclick: () => send("accepted") }, icon("check"), "Accept"), h("button", { type: "button", class: "btn", onclick: () => send("rejected") }, icon("x"), "Reject")), | |
| msg); | |
| } | |
| // ---------- graph ---------- | |
| async function renderGraph(nodeId) { | |
| const health = await api("/health"); | |
| if (!health.graph) { | |
| app.replaceChildren(emptyState("graph", "No graph extracted yet", "Extraction runs the configured model over every corpus chunk and keeps only edges whose quote is found in the source.", "uv run graphtrace extract\nuv run graphtrace build-graph")); | |
| return; | |
| } | |
| const search = h("input", { type: "search", placeholder: "Search entities: a method, dataset, task, metric or paper", "aria-label": "Search entities", style: "width:100%" }); | |
| const results = h("div", { class: "qitems", role: "list" }); | |
| const detail = h("div", {}); | |
| let timer = null; | |
| search.addEventListener("input", () => { | |
| clearTimeout(timer); | |
| timer = setTimeout(async () => { | |
| const q = search.value.trim(); | |
| if (!q) { results.replaceChildren(); return; } | |
| try { | |
| const hits = await api(`/graph/search?q=${encodeURIComponent(q)}`); | |
| results.replaceChildren(...(hits.length ? hits.map((n) => h("button", { class: "qitem", role: "listitem", onclick: () => (location.hash = `#/graph/${encodeURIComponent(n.node_id)}`) }, | |
| h("div", { class: "q" }, h("span", { class: "type-dot", style: `background:${typeColour(n.type)};margin-right:6px` }), n.name), h("div", { class: "t" }, `${n.type}, ${n.mentions} mention(s)`))) | |
| : [h("div", { class: "qcount" }, "No matching entity.")])); | |
| } catch (e) { results.replaceChildren(errorState(e)); } | |
| }, 180); | |
| }); | |
| app.replaceChildren(h("div", { class: "browser" }, h("div", { class: "qlist" }, h("div", { class: "qlist-tools" }, search), results), detail)); | |
| if (!nodeId) { | |
| const g = await api("/graph/stats"); | |
| detail.replaceChildren(h("div", { class: "page-head" }, h("h1", {}, "Explore the graph"), h("p", {}, "Search for an entity to see its neighbourhood. Every edge links to the quote and source chunk it was extracted from.")), | |
| g.top_degree?.length ? h("div", { class: "panel" }, h("h3", { style: "margin-bottom:10px" }, "Most connected entities"), bars(g.top_degree)) : null); | |
| return; | |
| } | |
| const nb = await api(`/graph/neighbourhood?node_id=${encodeURIComponent(nodeId)}&hops=1`); | |
| const nodes = Object.fromEntries(nb.nodes.map((n) => [n.node_id, n])); | |
| const center = nodes[nb.center]; | |
| const byRel = {}; | |
| for (const e of nb.edges) (byRel[e.relation] ||= []).push(e); | |
| detail.replaceChildren( | |
| h("div", { class: "qhead" }, h("h1", {}, center.name), h("div", { class: "cites" }, h("span", { class: "chip" }, h("span", { class: "type-dot", style: `background:${typeColour(center.type)}` }), center.type), | |
| center.aliases?.length ? h("span", { class: "chip" }, `also: ${center.aliases.join(", ")}`) : null, h("span", { class: "chip" }, `${nb.edges.length} edge(s)`))), | |
| h("div", { class: "panel", style: "margin-top:16px" }, radial(nb, nodes), h("div", { class: "legend" }, TYPES.map((t) => h("span", {}, h("span", { class: "type-dot", style: `background:${typeColour(t)}` }), t)))), | |
| ...Object.entries(byRel).map(([rel, es]) => h("section", { class: "section" }, h("h2", { style: "margin-bottom:6px" }, rel, h("span", { style: "color:var(--text-3);font-weight:400;margin-left:8px" }, es.length)), | |
| h("div", { class: "panel" }, es.map((e) => h("div", { class: "step" }, tripleView(e.subject_name, e.relation, e.object_name, nodes[e.subject]?.type, nodes[e.object]?.type), | |
| h("blockquote", { class: "q" }, `"${e.quote}"`), h("div", { style: "display:flex;gap:12px;flex-wrap:wrap" }, sourceLink(e.source_chunk_id, [e.start_char, e.end_char]), e.paper_title ? h("span", { style: "color:var(--text-3);font-size:12px" }, e.paper_title) : null))))))); | |
| } | |
| function radial(nb, nodes) { | |
| // Centre entity in the middle, neighbours evenly on a ring; capped so labels stay readable. | |
| const W = 640, H = 360, cx = W / 2, cy = H / 2, R = 140; | |
| const others = nb.nodes.filter((n) => n.node_id !== nb.center).slice(0, 24); | |
| const pos = { [nb.center]: [cx, cy] }; | |
| others.forEach((n, i) => { const a = (2 * Math.PI * i) / others.length - Math.PI / 2; pos[n.node_id] = [cx + R * Math.cos(a), cy + R * Math.sin(a)]; }); | |
| const svg = s("svg", { viewBox: `0 0 ${W} ${H}`, class: "graph-svg", role: "img", "aria-label": `Neighbourhood of ${nodes[nb.center].name}` }); | |
| for (const e of nb.edges) if (pos[e.subject] && pos[e.object]) s("line", { x1: pos[e.subject][0], y1: pos[e.subject][1], x2: pos[e.object][0], y2: pos[e.object][1], class: "edge" }, svg); | |
| for (const n of [nodes[nb.center], ...others]) { | |
| const [x, y] = pos[n.node_id]; | |
| const g = s("g", { class: `node${n.node_id === nb.center ? " center" : ""}`, tabindex: "0", role: "link" }, svg); | |
| s("circle", { cx: x, cy: y, r: n.node_id === nb.center ? 10 : 7, fill: typeColour(n.type) }, g); | |
| const label = n.name.length > 26 ? `${n.name.slice(0, 25)}...` : n.name; | |
| const t = s("text", { x, y: y + (y < cy ? -14 : 22), "text-anchor": "middle" }, g); | |
| t.textContent = label; | |
| const go = () => (location.hash = `#/graph/${encodeURIComponent(n.node_id)}`); | |
| g.addEventListener("click", go); | |
| g.addEventListener("keydown", (ev) => { if (ev.key === "Enter") go(); }); | |
| } | |
| return svg; | |
| } | |
| // ---------- try it: live retrieval and claim checking, no language model ---------- | |
| const tryState = { custom: "" }; | |
| async function renderTry(qid) { | |
| const [health, qs] = await Promise.all([api("/health"), api("/demo/questions")]); | |
| if (!health.corpus) { | |
| app.replaceChildren(emptyState("books", "Corpus not prepared", null, "uv run graphtrace corpus")); | |
| return; | |
| } | |
| const isCustom = qid === "custom"; | |
| if (!qid && qs.length) qid = qs[0].qid; | |
| const own = h("textarea", { id: "own-q", rows: 2, maxlength: 400, placeholder: "e.g. Which datasets are used to evaluate sentence embeddings?" }, tryState.custom); | |
| const ownForm = h("form", { | |
| class: "review-form", | |
| onsubmit: (e) => { | |
| e.preventDefault(); | |
| tryState.custom = own.value.trim(); | |
| if (!tryState.custom) return; | |
| if (location.hash === "#/try/custom") route(); | |
| else location.hash = "#/try/custom"; | |
| }, | |
| }, h("label", { for: "own-q" }, "Your own question"), own, h("div", { class: "btn-row" }, h("button", { type: "submit", class: "btn" }, icon("magnifying-glass"), "Retrieve"))); | |
| const list = h("div", { class: "qitems", role: "list" }, qs.map((q) => h("button", { class: "qitem", role: "listitem", "aria-current": String(!isCustom && q.qid === qid), onclick: () => (location.hash = `#/try/${encodeURIComponent(q.qid)}`) }, | |
| h("div", { class: "q" }, q.question), h("div", { class: "t" }, q.title)))); | |
| const detail = h("div", {}); | |
| app.replaceChildren( | |
| h("div", { class: "page-head" }, h("h1", {}, "Try the retrieval and verification stages"), | |
| h("p", {}, "Both run live on the 120-paper corpus without a language model: BM25 retrieval into the 2,048-token context the text-RAG method sees, and the NLI verifier that checks claims against their source passages.")), | |
| h("div", { class: "browser" }, h("div", { class: "qlist" }, h("div", { class: "qlist-tools" }, ownForm), list, | |
| h("div", { class: "qcount" }, qs.length ? `${qs.length} real QASPER questions about corpus papers, chosen by a fixed rule (graphtrace/demo.py)` : "No predefined questions on this server.")), detail)); | |
| const q = isCustom ? (tryState.custom ? { question: tryState.custom } : null) : qs.find((x) => x.qid === qid); | |
| if (!q) { detail.replaceChildren(emptyState("chat-text", "Pick a question", "Choose a predefined question or type your own.")); return; } | |
| detail.replaceChildren(h("div", { class: "skeleton", style: "height:200px" })); | |
| let r; | |
| try { | |
| r = await api("/demo/retrieve", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ question: q.question, gold_chunk_id: q.evidence?.chunk_id ?? null }) }); | |
| } catch (e) { detail.replaceChildren(errorState(e)); return; } | |
| detail.replaceChildren(...tryDetail(q, r, health)); | |
| } | |
| function tryDetail(q, r, health) { | |
| const ev = q.evidence; | |
| const out = [h("div", { class: "qhead" }, h("h1", {}, q.question), | |
| q.title ? h("div", { class: "paper" }, q.title, " ", h("span", { class: "mono" }, `(${q.doc_id})`)) : h("div", { class: "paper" }, "Your question"), | |
| q.answers?.length ? h("div", { class: "gold" }, h("span", {}, "Annotator answers"), q.answers.map((a) => h("span", { class: "chip gold-ans" }, a))) : null)]; | |
| if (ev) out.push(h("section", { class: "section" }, h("h2", { style: "margin-bottom:6px" }, "Annotator evidence"), | |
| h("div", { class: "panel step" }, h("blockquote", { class: "q" }, `"${ev.text}"`), sourceLink(ev.chunk_id, [ev.start, ev.end])))); | |
| const g = r.gold; | |
| const passages = r.passages.map((p) => h("div", { class: "step" }, | |
| h("div", { class: "cites" }, h("span", { class: "cite", style: "cursor:default" }, p.handle), h("b", { style: "font-weight:500" }, p.section), | |
| g && p.chunk_id === g.chunk_id ? h("span", { class: "badge good" }, icon("star"), "gold evidence") : null), | |
| h("div", { style: "color:var(--text-2);font-size:12.5px" }, p.title), | |
| h("div", { class: "usage" }, h("span", {}, `rank ${p.rank}`), h("span", {}, `BM25 ${p.bm25.toFixed(2)}`), h("span", {}, `${p.n_tokens} tokens`)), | |
| sourceLink(p.chunk_id))); | |
| out.push(h("section", { class: "section" }, h("h2", {}, "Context the text-RAG method would see"), | |
| h("p", { class: "lede" }, `BM25 over all ${r.corpus_chunks.toLocaleString("en")} corpus chunks, packed in rank order until the ${r.budget.toLocaleString("en")}-token budget is full. This is the text_rag method's own context builder.`), | |
| stats([["Passages", r.passages.length], ["Context tokens", `${r.context_tokens} / ${r.budget}`], ...(g ? [["Gold evidence rank", g.rank], ["Gold in context", g.in_context ? "yes" : "no"]] : [])]), | |
| h("div", { class: "panel", style: "margin-top:14px" }, passages.length ? passages : h("p", { style: "color:var(--text-2)" }, "No passage fits the budget.")))); | |
| // Claim checker: the annotator evidence by default, against the gold chunk (or the top passage). | |
| const options = [...(g && !r.passages.some((p) => p.chunk_id === g.chunk_id) ? [{ chunk_id: g.chunk_id, label: "Gold evidence chunk (not retrieved)" }] : []), | |
| ...r.passages.map((p) => ({ chunk_id: p.chunk_id, label: `${p.handle} ${p.section}, ${p.title}` }))]; | |
| const defaultIds = new Set([g?.chunk_id ?? r.passages[0]?.chunk_id].filter(Boolean)); | |
| const claim = h("textarea", { id: "claim", rows: 3, maxlength: 600 }, ev?.text ?? ""); | |
| const boxes = options.map((o) => h("label", { style: "display:flex;gap:8px;align-items:start;font-size:13px;color:var(--text)" }, | |
| h("input", { type: "checkbox", value: o.chunk_id, checked: defaultIds.has(o.chunk_id) }), h("span", {}, o.label))); | |
| const result = h("div", { role: "status", "aria-live": "polite" }); | |
| const run = async (e) => { | |
| e.preventDefault(); | |
| const ids = boxes.map((b) => b.querySelector("input")).filter((i) => i.checked).map((i) => i.value); | |
| if (!ids.length) { result.replaceChildren(h("div", { class: "callout warn" }, icon("warning"), "Select at least one passage.")); return; } | |
| if (ids.length > 3) { result.replaceChildren(h("div", { class: "callout warn" }, icon("warning"), "Select at most three passages.")); return; } | |
| result.replaceChildren(h("div", { class: "skeleton", style: "height:90px" })); | |
| try { | |
| const c = await api("/demo/check", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ claim: claim.value, chunk_ids: ids }) }); | |
| result.replaceChildren(checkResult(c)); | |
| } catch (err) { result.replaceChildren(errorState(err)); } | |
| }; | |
| const form = h("form", { class: "panel review-form", onsubmit: run }, | |
| h("label", { for: "claim" }, "Claim"), claim, | |
| h("fieldset", { style: "border:0;padding:0;margin:0;display:grid;gap:6px" }, h("legend", { style: "font-size:12.5px;color:var(--text-2);margin-bottom:6px" }, "Check it against (up to three passages)"), boxes), | |
| h("div", { class: "btn-row" }, h("button", { type: "submit", class: "btn primary", disabled: !health.nli }, icon("seal-check"), "Check claim")), | |
| !health.nli ? h("div", { class: "callout warn" }, icon("warning"), "The NLI verifier is not installed on this server.") : null, | |
| result); | |
| out.push(h("section", { class: "section" }, h("h2", {}, "Check a claim against its sources"), | |
| h("p", { class: "lede" }, "Exact anchoring accepts only text that occurs verbatim in the passage (after whitespace and typographic normalisation), the rule every extracted edge must pass. The NLI score is the best three-sentence window; 0.5 or above counts as supported, otherwise a contradiction score of 0.5 or above in any selected passage labels it contradicted. These are the verifier's real outputs, mistakes included. Edit the claim to see both checks react."), | |
| form)); | |
| return out; | |
| } | |
| function checkResult(c) { | |
| const lab = { entailed: ["good", "seal-check", "Supported"], contradicted: ["bad", "x-circle", "Contradicted"], not_enough_evidence: ["warn", "question", "Not supported"] }[c.nli?.label] || ["neutral", "circle", c.nli?.label ?? "n/a"]; | |
| return h("div", { style: "display:grid;gap:12px;margin-top:6px" }, | |
| c.nli ? h("div", { class: "cites" }, h("span", { class: `badge ${lab[0]}` }, icon(lab[1]), lab[2]), | |
| h("span", { class: "usage" }, h("span", {}, `entailment ${c.nli.entail.toFixed(3)}`), h("span", {}, `contradiction ${c.nli.contradict.toFixed(3)}`), c.nli.joint ? h("span", {}, `joint ${c.nli.joint.entail.toFixed(3)}`) : null, h("span", {}, c.nli.model))) : null, | |
| c.passages.map((p) => h("div", { class: "step" }, | |
| h("div", { class: "cites" }, h("b", { style: "font-weight:500" }, p.section), h("span", { style: "color:var(--text-3);font-size:12px" }, p.title)), | |
| h("div", { class: "cites" }, | |
| p.anchor ? h("span", { class: "badge good" }, icon("anchor-simple"), `found verbatim at ${p.anchor.start}-${p.anchor.end}`) : h("span", { class: "badge neutral" }, icon("anchor-simple"), "not found verbatim"), | |
| p.nli ? h("span", { class: "badge neutral" }, `entailment ${p.nli.entail.toFixed(3)}, contradiction ${p.nli.contradict.toFixed(3)}`) : null), | |
| p.nli?.window ? h("blockquote", { class: "q" }, p.nli.window) : null, | |
| h("div", { class: "btn-row" }, | |
| p.anchor ? h("button", { type: "button", class: "src-link", onclick: () => openSource(p.chunk_id, [p.anchor.start, p.anchor.end]) }, icon("file-text"), " show the anchored claim") : null, | |
| p.nli?.window_start != null ? h("button", { type: "button", class: "src-link", onclick: () => openSource(p.chunk_id, [p.nli.window_start, p.nli.window_end]) }, icon("file-text"), " show the best window") : sourceLink(p.chunk_id))))); | |
| } | |
| // ---------- source drawer ---------- | |
| const drawer = document.getElementById("drawer"); | |
| function closeDrawer() { drawer.classList.remove("open"); drawer.setAttribute("aria-hidden", "true"); } | |
| document.getElementById("drawer-close").addEventListener("click", closeDrawer); | |
| document.addEventListener("keydown", (e) => { if (e.key === "Escape") closeDrawer(); }); | |
| async function openSource(chunkId, range) { | |
| let c; | |
| try { c = await api(`/chunks/${encodeURIComponent(chunkId)}`); } catch (e) { document.getElementById("drawer-body").replaceChildren(errorState(e)); drawer.classList.add("open"); return; } | |
| document.getElementById("drawer-kicker").textContent = chunkId; | |
| document.getElementById("drawer-title").textContent = c.section_heading; | |
| const text = c.text; | |
| const body = h("div", { class: "source" }); | |
| const [a, b] = range && range[0] != null ? range : [-1, -1]; | |
| const pieces = a >= 0 ? [[0, a, false], [a, b, true], [b, text.length, false]] : [[0, text.length, false]]; | |
| for (const [x, y, marked] of pieces) { | |
| for (const part of text.slice(x, y).split(/(§ [^\n]*)/)) { | |
| if (!part) continue; | |
| let node = part.startsWith("§ ") ? h("span", { class: "sec" }, part.slice(2)) : document.createTextNode(part); | |
| if (marked) node = h("mark", {}, node); | |
| body.append(node); | |
| } | |
| } | |
| document.getElementById("drawer-body").replaceChildren( | |
| h("div", { class: "callout info" }, icon("file-text"), c.paper_title || c.doc_id), | |
| a >= 0 ? h("div", { style: "color:var(--text-3);font-size:12px" }, "The highlighted span is shown at its exact character offsets in the source chunk.") : null, | |
| body); | |
| drawer.classList.add("open"); | |
| drawer.setAttribute("aria-hidden", "false"); | |
| document.getElementById("drawer-close").focus({ preventScroll: true }); | |
| body.querySelector("mark")?.scrollIntoView({ block: "center" }); | |
| } | |
| window.addEventListener("hashchange", route); | |
| route(); | |