// GraphTrace Explorer. Every string from the API is inserted as text, never as HTML. function h(tag, attrs = {}, ...kids) { const n = document.createElement(tag); for (const [k, v] of Object.entries(attrs || {})) { if (v == null || v === false) continue; if (k === "class") n.className = v; else if (k.startsWith("on")) n.addEventListener(k.slice(2), v); else n.setAttribute(k, v === true ? "" : v); } for (const kid of kids.flat()) if (kid != null && kid !== false) n.append(kid instanceof Node ? kid : document.createTextNode(String(kid))); return n; } const NS = "http://www.w3.org/2000/svg"; function s(tag, attrs = {}, parent) { const n = document.createElementNS(NS, tag); for (const [k, v] of Object.entries(attrs)) n.setAttribute(k, v); if (parent) parent.appendChild(n); return n; } const icon = (name) => h("i", { class: `ph ph-${name}`, "aria-hidden": "true" }); const pct = (v) => (v == null || Number.isNaN(v) ? "n/a" : `${(100 * v).toFixed(1)}`); const app = document.getElementById("app"); const TYPES = ["Paper", "Method", "Dataset", "Task", "Metric"]; const typeColour = (t) => `var(--series-${TYPES.indexOf(t) + 1 || 6})`; const CATS = { single_hop: "Single hop", multi_hop: "Multi hop", false_premise: "False premise", conflict_or_insufficient: "Conflict / insufficient" }; // ---------- theme ---------- const root = document.documentElement; try { const t = localStorage.getItem("gt-theme"); if (t) root.dataset.theme = t; } catch {} document.getElementById("theme-toggle").addEventListener("click", () => { const dark = root.dataset.theme ? root.dataset.theme === "dark" : matchMedia("(prefers-color-scheme: dark)").matches; root.dataset.theme = dark ? "light" : "dark"; try { localStorage.setItem("gt-theme", root.dataset.theme); } catch {} }); // ---------- data ---------- async function api(path, opts) { const r = await fetch(`/api${path}`, opts); if (!r.ok) { const body = await r.text(); let detail = body; try { detail = JSON.parse(body).detail ?? body; } catch {} const err = new Error(typeof detail === "string" ? detail : JSON.stringify(detail)); err.status = r.status; throw err; } return r.json(); } const maybe = (p) => p.catch((e) => (e.status === 404 ? null : Promise.reject(e))); function emptyState(iconName, title, text, commands) { return h("div", { class: "empty" }, icon(iconName), h("b", {}, title), text ? h("span", {}, text) : null, commands ? h("pre", {}, commands) : null); } function errorState(e) { return h("div", { class: "empty error" }, icon("warning-circle"), h("b", {}, "Could not load this view"), h("span", {}, String(e.message || e))); } // ---------- routing: #/ #/bench[/] #/graph[/] #/try[/] ---------- function parseHash() { const parts = location.hash.replace(/^#/, "").split("?")[0].split("/").filter(Boolean).map(decodeURIComponent); return { view: parts[0] || "overview", id: parts[1] || null }; } async function route() { closeDrawer(); const { view, id } = parseHash(); document.querySelectorAll(".tabs a").forEach((a) => (a.dataset.view === view ? a.setAttribute("aria-current", "page") : a.removeAttribute("aria-current"))); app.replaceChildren(h("div", { class: "skeleton", style: "height:140px" })); try { if (view === "bench") await renderBench(id); else if (view === "graph") await renderGraph(id); else if (view === "try") await renderTry(id); else await renderOverview(); } catch (e) { app.replaceChildren(errorState(e)); } } // ---------- overview ---------- function stats(pairs) { return h("div", { class: "stats" }, pairs.map(([k, v]) => h("div", { class: "stat" }, h("div", { class: "v" }, v ?? "n/a"), h("div", { class: "k" }, k)))); } function bars(entries, colourOf) { const max = Math.max(1, ...entries.map(([, v]) => v)); return h("div", { class: "bars" }, entries.map(([k, v]) => h("div", { class: "row" }, h("span", {}, k), h("span", {}, h("div", { class: "bar", style: `width:${(100 * v) / max}%;${colourOf ? `background:${colourOf(k)}` : ""}` })), h("span", { class: "num" }, v)))); } function ciCell(c) { if (!c || c.mean == null || Number.isNaN(c.mean)) return h("td", { class: "num" }, "n/a"); return h("td", { class: "num" }, pct(c.mean), c.low != null ? h("span", { class: "ci" }, `${pct(c.low)}-${pct(c.high)}`) : null); } async function renderOverview() { const health = await api("/health"); const [corpus, graph, items, runs] = await Promise.all([ maybe(api("/corpus/stats")), health.graph ? api("/graph/stats") : null, health.benchmark ? maybe(api("/bench/items")) : null, api("/runs"), ]); const out = [h("div", { class: "page-head" }, h("h1", {}, "GraphTrace-Bench"), h("p", {}, "Can an LLM answer multi-step questions about AI research through a verifiable knowledge path, and recognise when the evidence is missing, conflicting or based on a false premise? Every edge in the graph carries the exact quote it was extracted from."))]; // Corpus out.push(h("section", { class: "section" }, h("h2", {}, "Corpus"), corpus ? [ h("p", { class: "lede" }, `QASPER papers (CC-BY-4.0) selected for cross-paper connectivity through shared benchmark datasets.`), stats([["Papers", corpus.stats.selected_papers], ["Chunks", corpus.stats.chunks ?? null], ["Datasets in selection", corpus.stats.datasets_in_selection], ["Shared datasets", corpus.stats.datasets_shared_within_selection], ["Paper pairs sharing a dataset", corpus.stats.paper_pairs_sharing_a_dataset], ["Components", corpus.stats.connected_components]]), corpus.stats.top_datasets?.length ? h("div", { class: "panel", style: "margin-top:16px" }, h("h3", { style: "margin-bottom:10px" }, "Benchmarks mentioned by the most selected papers"), bars(corpus.stats.top_datasets.slice(0, 10).map(([name, n]) => [name, n]))) : null, ] : emptyState("books", "Corpus not prepared", null, "uv run graphtrace corpus"))); // Graph out.push(h("section", { class: "section" }, h("h2", {}, "Knowledge graph"), graph ? [ h("p", { class: "lede" }, "Closed schema. Every edge is backed by a quote anchored at exact offsets in its source chunk; unanchored extractions are rejected."), stats([["Nodes", graph.nodes], ["Edges", graph.edges], ["Distinct facts", graph.distinct_facts], ["Result records", graph.results], ["Components", graph.components], ["Largest component", graph.largest_component_nodes]]), h("div", { class: "grid-2", style: "margin-top:16px" }, h("div", { class: "panel" }, h("h3", { style: "margin-bottom:10px" }, "Nodes by type"), bars(TYPES.filter((t) => graph.nodes_by_type[t]).map((t) => [t, graph.nodes_by_type[t]]), typeColour)), h("div", { class: "panel" }, h("h3", { style: "margin-bottom:10px" }, "Edges by relation"), bars(Object.entries(graph.edges_by_relation).sort((a, b) => b[1] - a[1])))), ] : emptyState("graph", "No graph extracted yet", "Extraction calls the configured model on every chunk. Estimate the cost first.", "uv run graphtrace extract\nuv run graphtrace build-graph"))); // Benchmark let benchBody; if (items && items.length) { const byCat = {}, byStatus = {}; for (const it of items) { byCat[it.category] = (byCat[it.category] || 0) + 1; byStatus[it.review_status] = (byStatus[it.review_status] || 0) + 1; } benchBody = [ h("p", { class: "lede" }, "Generated from the graph, then reviewed by a person against the source text. Only accepted items enter the released benchmark."), h("div", { class: "grid-2" }, h("div", { class: "panel" }, h("h3", { style: "margin-bottom:10px" }, "Items by category"), bars(Object.keys(CATS).filter((c) => byCat[c]).map((c) => [CATS[c], byCat[c]]))), h("div", { class: "panel" }, h("h3", { style: "margin-bottom:10px" }, "Review status"), bars(["accepted", "edited", "rejected", "unreviewed"].filter((k) => byStatus[k]).map((k) => [k, byStatus[k]])))), ]; } else { benchBody = emptyState("list-checks", "No benchmark items yet", "Items are generated from the extracted graph and then reviewed by hand.", "uv run graphtrace generate-bench\nuv run graphtrace review --reviewer \nuv run graphtrace freeze"); } out.push(h("section", { class: "section" }, h("h2", {}, "Benchmark"), benchBody)); // Results const evaluated = runs.filter((r) => r.has_metrics); const res = h("section", { class: "section" }, h("h2", {}, "Results")); if (!evaluated.length) { res.append(emptyState("hourglass-medium", "No evaluated runs yet", "Methods are compared with the same base model, output schema and context budget.", "uv run graphtrace estimate-cost configs/experiments/main.yaml\nuv run graphtrace run configs/experiments/main.yaml\nuv run graphtrace evaluate configs/experiments/main.yaml --judge")); } else { for (const r of evaluated) { const m = await api(`/runs/${encodeURIComponent(r.name)}/metrics`); const cols = [["correct", "Accuracy"], ["path_all_valid", "Path valid"], ["path_completeness", "Path complete"], ["grounding_precision", "Grounding prec."], ["citation_rate", "Cites sources"], ["false_premise_rejected", "False premise rejected"], ["conflict_flagged", "Conflict flagged"]]; res.append(h("h3", { style: "margin:14px 0 4px" }, r.name, r.smoke_test ? h("span", { class: "badge warn", style: "margin-left:8px" }, icon("warning"), "Smoke test on unreviewed items: not a result") : null)); if (m.judge_status) res.append(h("p", { style: "color:var(--text-3);font-size:12px;margin-bottom:8px" }, `Evaluation judge: ${m.judge_status}`)); const table = (title, block) => block && Object.keys(block).length ? h("div", { class: "table-wrap", style: "margin-top:10px" }, h("table", {}, h("thead", {}, h("tr", {}, h("th", {}, title), h("th", { class: "num" }, "n"), h("th", { class: "num" }, "Errors"), cols.map(([, t]) => h("th", { class: "num" }, t)), h("th", { class: "num" }, "ECE"), h("th", { class: "num" }, "AUROC"))), h("tbody", {}, Object.entries(block).map(([name, a]) => h("tr", {}, h("td", {}, name), h("td", { class: "num" }, a.n), h("td", { class: "num" }, a.errors ?? 0), cols.map(([k]) => ciCell(a[k])), h("td", { class: "num" }, m.calibration?.[name]?.ece?.toFixed(3) ?? "n/a"), h("td", { class: "num" }, m.calibration?.[name]?.auroc?.toFixed(3) ?? "n/a")))))) : null; res.append(table(`Headline (${(m.headline_categories || []).map((c) => CATS[c] || c).join(", ") || "all"})`, m.headline || m.methods)); res.append(table("Conflict / insufficient items (reported separately)", m.conflict_insufficient)); const cats = [...new Set(Object.values(m.by_category).flatMap((v) => Object.keys(v)))]; res.append(h("div", { class: "table-wrap", style: "margin-top:10px" }, h("table", {}, h("thead", {}, h("tr", {}, h("th", {}, "Accuracy by category"), cats.map((c) => h("th", { class: "num" }, CATS[c] || c)))), h("tbody", {}, Object.entries(m.by_category).map(([name, by]) => h("tr", {}, h("td", {}, name), cats.map((c) => ciCell(by[c]?.correct)))))))); } } out.push(res); if (!health.graph) out.push(h("p", { style: "color:var(--text-3);font-size:12px;margin-top:24px" }, "The Benchmark and Graph views fill in once the graph has been extracted. ", h("a", { href: "#/try" }, "Try it"), " runs the retrieval and verification stages live on the corpus today, with no model calls.")); app.replaceChildren(...out); } // ---------- shared bits ---------- function ent(name, type) { return h("span", { class: "ent" }, type ? h("span", { class: "type-dot", style: `background:${typeColour(type)}`, title: type }) : null, name); } function tripleView(sName, rel, oName, sType, oType) { return h("div", { class: "triple" }, ent(sName, sType), h("span", { class: "rel" }, rel), icon("arrow-right"), ent(oName, oType)); } function sourceLink(chunkId, quoteRange) { return h("button", { type: "button", class: "src-link", onclick: () => openSource(chunkId, quoteRange) }, icon("file-text"), " ", chunkId); } // ---------- benchmark ---------- let benchFilter = { cat: "all", status: "all", text: "" }; async function renderBench(itemId) { const [health, items] = await Promise.all([api("/health"), maybe(api("/bench/items"))]); if (!items || !items.length) { app.replaceChildren(emptyState("list-checks", "No benchmark items yet", "Generate candidates from the extracted graph, then review them.", "uv run graphtrace generate-bench\nuv run graphtrace review --reviewer ")); return; } if (!itemId) itemId = items[0].item_id; const list = h("div", { class: "qitems", role: "list" }); const count = h("div", { class: "qcount" }); const search = h("input", { type: "search", placeholder: "Filter questions", "aria-label": "Filter questions", value: benchFilter.text }); const catSel = h("select", { "aria-label": "Category" }, h("option", { value: "all" }, "All categories"), Object.entries(CATS).map(([k, v]) => h("option", { value: k, selected: benchFilter.cat === k }, v))); const stSel = h("select", { "aria-label": "Review status" }, ["all", "unreviewed", "accepted", "edited", "rejected"].map((k) => h("option", { value: k, selected: benchFilter.status === k }, k === "all" ? "Any status" : k))); const fill = () => { const t = benchFilter.text.toLowerCase(); const shown = items.filter((i) => (benchFilter.cat === "all" || i.category === benchFilter.cat) && (benchFilter.status === "all" || i.review_status === benchFilter.status) && (!t || i.question.toLowerCase().includes(t))); list.replaceChildren(...shown.map((i) => h("button", { class: "qitem", role: "listitem", "aria-current": String(i.item_id === itemId), onclick: () => (location.hash = `#/bench/${encodeURIComponent(i.item_id)}`) }, h("div", { class: "q" }, i.question), h("div", { class: "t" }, `${CATS[i.category] || i.category}, ${i.difficulty} hop(s), ${i.review_status}`)))); count.textContent = `${shown.length} of ${items.length} items`; }; search.addEventListener("input", () => { benchFilter.text = search.value; fill(); }); catSel.addEventListener("change", () => { benchFilter.cat = catSel.value; fill(); }); stSel.addEventListener("change", () => { benchFilter.status = stSel.value; fill(); }); fill(); const detail = h("div", {}, h("div", { class: "skeleton", style: "height:200px" })); app.replaceChildren(h("div", { class: "browser" }, h("div", { class: "qlist" }, h("div", { class: "qlist-tools" }, search, h("div", { class: "btn-row" }, catSel, stSel)), list, count), detail)); const [d, runs] = await Promise.all([api(`/bench/items/${encodeURIComponent(itemId)}`), api("/runs")]); detail.replaceChildren(...itemDetail(d, health, runs)); } function statusBadge(st) { const m = { accepted: ["good", "check-circle"], edited: ["good", "pencil-simple"], rejected: ["bad", "x-circle"], unreviewed: ["warn", "hourglass-medium"] }[st] || ["neutral", "circle"]; return h("span", { class: `badge ${m[0]}` }, icon(m[1]), st); } function itemDetail(d, health, runs) { const it = d.item; const gold = it.gold_decision === "answer" ? it.gold_answer.join(", ") : it.gold_decision === "abstain" ? "Abstain: the evidence does not answer this" : "Flag the conflict between sources"; const head = h("div", { class: "qhead" }, h("h1", {}, it.question), h("div", { class: "cites" }, h("span", { class: "chip" }, CATS[it.category] || it.category), h("span", { class: "chip" }, `${it.difficulty} hop(s)`), h("span", { class: "chip mono" }, it.template_id), statusBadge(it.review_status)), h("dl", { class: "kv", style: "margin-top:6px" }, h("dt", {}, "Gold"), h("dd", {}, h("b", {}, gold)), it.annotation_notes ? [h("dt", {}, "Notes"), h("dd", {}, it.annotation_notes)] : null, it.reviewer ? [h("dt", {}, "Reviewed by"), h("dd", {}, `${it.reviewer}${it.review_notes ? `: ${it.review_notes}` : ""}`)] : null)); const steps = d.path.length ? d.path.map((p) => h("div", { class: "step" }, p.kind === "edge" ? tripleView(p.subject_name, p.relation, p.object_name) : tripleView(p.method_name, `reports ${p.value}`, p.dataset_name), h("blockquote", { class: "q" }, `"${p.quote}"`), h("div", { style: "display:flex;gap:12px;flex-wrap:wrap;align-items:center" }, sourceLink(p.source_chunk_id, [p.start_char, p.end_char]), p.paper_title ? h("span", { style: "color:var(--text-3);font-size:12px" }, p.paper_title) : null, p.confidence != null ? h("span", { class: "badge neutral" }, `support ${p.confidence.toFixed(2)}`) : null))) : [h("p", { style: "color:var(--text-2)" }, "No required path: the correct behaviour is to notice that the graph and sources do not support the question.")]; const out = [head, h("section", { class: "section" }, h("h2", { style: "margin-bottom:6px" }, "Required path and its evidence"), h("div", { class: "panel" }, steps))]; if (health.review_enabled) out.push(h("section", { class: "section" }, h("h2", { style: "margin-bottom:6px" }, "Review against the source text"), reviewForm(it))); if (runs.length) { const sel = h("select", { "aria-label": "Run" }, runs.map((r) => h("option", { value: r.name }, r.name))); const host = h("div", { class: "answers" }); const load = async () => { host.replaceChildren(h("div", { class: "skeleton", style: "height:120px" })); try { const r = await api(`/runs/${encodeURIComponent(sel.value)}/items/${encodeURIComponent(it.item_id)}`); const entries = Object.entries(r.outputs); host.replaceChildren(...(entries.length ? entries.map(([m, o]) => outputCard(m, o)) : [emptyState("hourglass-medium", "This run has no prediction for this item.")])); } catch (e) { host.replaceChildren(errorState(e)); } }; sel.addEventListener("change", load); out.push(h("section", { class: "section" }, h("div", { class: "toolbar" }, h("h2", {}, "Method outputs"), sel), host)); load(); } return out; } function outputCard(method, o) { const pc = o.path_check || { checks: [] }; const decision = { answer: ["neutral", "chat-text", "Answered"], abstain: ["warn", "hand", "Abstained"], flag_conflict: ["warn", "warning-diamond", "Flagged conflict"] }[o.decision] || ["neutral", "circle", o.decision]; return h("article", { class: "answer-card" }, h("header", {}, h("span", { class: "sys" }, method), h("span", { class: `badge ${decision[0]}` }, icon(decision[1]), decision[2])), o.error ? h("div", { class: "callout warn" }, icon("warning"), `Run error: ${o.error}`) : null, h("div", { class: "answer-text" }, o.answer?.length ? o.answer.join(", ") : "(no answer)"), h("div", { class: "usage" }, h("span", {}, `confidence ${o.confidence?.toFixed?.(2) ?? "n/a"}`), h("span", {}, `${o.context_tokens} ctx tokens`)), pc.checks.length ? h("div", { class: "stmt" }, h("b", { style: "font-size:12.5px" }, "Claimed path"), pc.checks.map((c) => h("div", { class: "cites" }, tripleView(c.triple[0], c.triple[1], c.triple[2]), c.valid || (c.exists && c.supported) ? h("span", { class: "badge good" }, icon("check-circle"), "in graph, supported") : h("span", { class: "badge bad" }, icon("x-circle"), c.problem || "invalid"))), h("div", { class: "cites" }, h("span", { class: `badge ${pc.connected ? "good" : "bad"}` }, pc.connected ? "connected" : "not connected"), h("span", { class: `badge ${pc.touches_question ? "good" : "warn"}` }, pc.touches_question ? "starts at the question" : "does not start at the question"), h("span", { class: `badge ${pc.covers_answer ? "good" : "warn"}` }, pc.covers_answer ? "reaches the answer" : "does not reach the answer"))) : h("div", { class: "stmt" }, h("span", { style: "color:var(--text-3);font-size:12.5px" }, "No path claimed.")), o.source_chunk_ids?.length ? h("div", { class: "cites" }, o.source_chunk_ids.map((c) => sourceLink(c))) : null, o.verifier ? h("details", { class: "more" }, h("summary", {}, "Verifier decision"), h("pre", { style: "white-space:pre-wrap;font:12px var(--mono);margin:0" }, JSON.stringify(o.verifier, null, 2))) : null); } function reviewForm(it) { let reviewer = ""; try { reviewer = localStorage.getItem("gt-reviewer") || ""; } catch {} const who = h("input", { value: reviewer, required: true, autocomplete: "off" }); const notes = h("textarea", { rows: 2 }); const question = h("input", { value: it.question }); const decision = h("select", {}, ["answer", "abstain", "flag_conflict"].map((k) => h("option", { value: k, selected: it.gold_decision === k }, k))); const answer = h("input", { value: it.gold_answer.join("; ") }); const msg = h("div", { role: "status", style: "font-size:12.5px;color:var(--text-2)" }); const send = async (status) => { if (!who.value.trim()) { msg.textContent = "Enter your initials first."; who.focus(); return; } try { localStorage.setItem("gt-reviewer", who.value.trim()); } catch {} const body = { status, reviewer: who.value.trim(), notes: notes.value }; if (status === "edited") Object.assign(body, { question: question.value, gold_decision: decision.value, gold_answer: answer.value.split(";").map((x) => x.trim()).filter(Boolean) }); try { await api(`/bench/items/${encodeURIComponent(it.item_id)}/review`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) }); msg.textContent = `Saved as ${status}.`; route(); } catch (e) { msg.textContent = `Not saved: ${e.message}`; } }; return h("div", { class: "panel review-form" }, h("p", { style: "color:var(--text-2);font-size:13px" }, "Open each quote's source above and check that the passage really states the edge. Accept only items whose gold answer follows from the sources."), h("label", {}, "Reviewer initials", who), h("label", {}, "Notes", notes), h("details", { class: "more" }, h("summary", {}, "Edit before accepting"), h("div", { class: "review-form" }, h("label", {}, "Question", question), h("label", {}, "Gold decision", decision), h("label", {}, "Gold answer (separate with ;)", answer), h("div", { class: "btn-row" }, h("button", { type: "button", class: "btn", onclick: () => send("edited") }, icon("pencil-simple"), "Save edit")))), h("div", { class: "btn-row" }, h("button", { type: "button", class: "btn primary", onclick: () => send("accepted") }, icon("check"), "Accept"), h("button", { type: "button", class: "btn", onclick: () => send("rejected") }, icon("x"), "Reject")), msg); } // ---------- graph ---------- async function renderGraph(nodeId) { const health = await api("/health"); if (!health.graph) { app.replaceChildren(emptyState("graph", "No graph extracted yet", "Extraction runs the configured model over every corpus chunk and keeps only edges whose quote is found in the source.", "uv run graphtrace extract\nuv run graphtrace build-graph")); return; } const search = h("input", { type: "search", placeholder: "Search entities: a method, dataset, task, metric or paper", "aria-label": "Search entities", style: "width:100%" }); const results = h("div", { class: "qitems", role: "list" }); const detail = h("div", {}); let timer = null; search.addEventListener("input", () => { clearTimeout(timer); timer = setTimeout(async () => { const q = search.value.trim(); if (!q) { results.replaceChildren(); return; } try { const hits = await api(`/graph/search?q=${encodeURIComponent(q)}`); results.replaceChildren(...(hits.length ? hits.map((n) => h("button", { class: "qitem", role: "listitem", onclick: () => (location.hash = `#/graph/${encodeURIComponent(n.node_id)}`) }, h("div", { class: "q" }, h("span", { class: "type-dot", style: `background:${typeColour(n.type)};margin-right:6px` }), n.name), h("div", { class: "t" }, `${n.type}, ${n.mentions} mention(s)`))) : [h("div", { class: "qcount" }, "No matching entity.")])); } catch (e) { results.replaceChildren(errorState(e)); } }, 180); }); app.replaceChildren(h("div", { class: "browser" }, h("div", { class: "qlist" }, h("div", { class: "qlist-tools" }, search), results), detail)); if (!nodeId) { const g = await api("/graph/stats"); detail.replaceChildren(h("div", { class: "page-head" }, h("h1", {}, "Explore the graph"), h("p", {}, "Search for an entity to see its neighbourhood. Every edge links to the quote and source chunk it was extracted from.")), g.top_degree?.length ? h("div", { class: "panel" }, h("h3", { style: "margin-bottom:10px" }, "Most connected entities"), bars(g.top_degree)) : null); return; } const nb = await api(`/graph/neighbourhood?node_id=${encodeURIComponent(nodeId)}&hops=1`); const nodes = Object.fromEntries(nb.nodes.map((n) => [n.node_id, n])); const center = nodes[nb.center]; const byRel = {}; for (const e of nb.edges) (byRel[e.relation] ||= []).push(e); detail.replaceChildren( h("div", { class: "qhead" }, h("h1", {}, center.name), h("div", { class: "cites" }, h("span", { class: "chip" }, h("span", { class: "type-dot", style: `background:${typeColour(center.type)}` }), center.type), center.aliases?.length ? h("span", { class: "chip" }, `also: ${center.aliases.join(", ")}`) : null, h("span", { class: "chip" }, `${nb.edges.length} edge(s)`))), h("div", { class: "panel", style: "margin-top:16px" }, radial(nb, nodes), h("div", { class: "legend" }, TYPES.map((t) => h("span", {}, h("span", { class: "type-dot", style: `background:${typeColour(t)}` }), t)))), ...Object.entries(byRel).map(([rel, es]) => h("section", { class: "section" }, h("h2", { style: "margin-bottom:6px" }, rel, h("span", { style: "color:var(--text-3);font-weight:400;margin-left:8px" }, es.length)), h("div", { class: "panel" }, es.map((e) => h("div", { class: "step" }, tripleView(e.subject_name, e.relation, e.object_name, nodes[e.subject]?.type, nodes[e.object]?.type), h("blockquote", { class: "q" }, `"${e.quote}"`), h("div", { style: "display:flex;gap:12px;flex-wrap:wrap" }, sourceLink(e.source_chunk_id, [e.start_char, e.end_char]), e.paper_title ? h("span", { style: "color:var(--text-3);font-size:12px" }, e.paper_title) : null))))))); } function radial(nb, nodes) { // Centre entity in the middle, neighbours evenly on a ring; capped so labels stay readable. const W = 640, H = 360, cx = W / 2, cy = H / 2, R = 140; const others = nb.nodes.filter((n) => n.node_id !== nb.center).slice(0, 24); const pos = { [nb.center]: [cx, cy] }; others.forEach((n, i) => { const a = (2 * Math.PI * i) / others.length - Math.PI / 2; pos[n.node_id] = [cx + R * Math.cos(a), cy + R * Math.sin(a)]; }); const svg = s("svg", { viewBox: `0 0 ${W} ${H}`, class: "graph-svg", role: "img", "aria-label": `Neighbourhood of ${nodes[nb.center].name}` }); for (const e of nb.edges) if (pos[e.subject] && pos[e.object]) s("line", { x1: pos[e.subject][0], y1: pos[e.subject][1], x2: pos[e.object][0], y2: pos[e.object][1], class: "edge" }, svg); for (const n of [nodes[nb.center], ...others]) { const [x, y] = pos[n.node_id]; const g = s("g", { class: `node${n.node_id === nb.center ? " center" : ""}`, tabindex: "0", role: "link" }, svg); s("circle", { cx: x, cy: y, r: n.node_id === nb.center ? 10 : 7, fill: typeColour(n.type) }, g); const label = n.name.length > 26 ? `${n.name.slice(0, 25)}...` : n.name; const t = s("text", { x, y: y + (y < cy ? -14 : 22), "text-anchor": "middle" }, g); t.textContent = label; const go = () => (location.hash = `#/graph/${encodeURIComponent(n.node_id)}`); g.addEventListener("click", go); g.addEventListener("keydown", (ev) => { if (ev.key === "Enter") go(); }); } return svg; } // ---------- try it: live retrieval and claim checking, no language model ---------- const tryState = { custom: "" }; async function renderTry(qid) { const [health, qs] = await Promise.all([api("/health"), api("/demo/questions")]); if (!health.corpus) { app.replaceChildren(emptyState("books", "Corpus not prepared", null, "uv run graphtrace corpus")); return; } const isCustom = qid === "custom"; if (!qid && qs.length) qid = qs[0].qid; const own = h("textarea", { id: "own-q", rows: 2, maxlength: 400, placeholder: "e.g. Which datasets are used to evaluate sentence embeddings?" }, tryState.custom); const ownForm = h("form", { class: "review-form", onsubmit: (e) => { e.preventDefault(); tryState.custom = own.value.trim(); if (!tryState.custom) return; if (location.hash === "#/try/custom") route(); else location.hash = "#/try/custom"; }, }, h("label", { for: "own-q" }, "Your own question"), own, h("div", { class: "btn-row" }, h("button", { type: "submit", class: "btn" }, icon("magnifying-glass"), "Retrieve"))); const list = h("div", { class: "qitems", role: "list" }, qs.map((q) => h("button", { class: "qitem", role: "listitem", "aria-current": String(!isCustom && q.qid === qid), onclick: () => (location.hash = `#/try/${encodeURIComponent(q.qid)}`) }, h("div", { class: "q" }, q.question), h("div", { class: "t" }, q.title)))); const detail = h("div", {}); app.replaceChildren( h("div", { class: "page-head" }, h("h1", {}, "Try the retrieval and verification stages"), h("p", {}, "Both run live on the 120-paper corpus without a language model: BM25 retrieval into the 2,048-token context the text-RAG method sees, and the NLI verifier that checks claims against their source passages.")), h("div", { class: "browser" }, h("div", { class: "qlist" }, h("div", { class: "qlist-tools" }, ownForm), list, h("div", { class: "qcount" }, qs.length ? `${qs.length} real QASPER questions about corpus papers, chosen by a fixed rule (graphtrace/demo.py)` : "No predefined questions on this server.")), detail)); const q = isCustom ? (tryState.custom ? { question: tryState.custom } : null) : qs.find((x) => x.qid === qid); if (!q) { detail.replaceChildren(emptyState("chat-text", "Pick a question", "Choose a predefined question or type your own.")); return; } detail.replaceChildren(h("div", { class: "skeleton", style: "height:200px" })); let r; try { r = await api("/demo/retrieve", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ question: q.question, gold_chunk_id: q.evidence?.chunk_id ?? null }) }); } catch (e) { detail.replaceChildren(errorState(e)); return; } detail.replaceChildren(...tryDetail(q, r, health)); } function tryDetail(q, r, health) { const ev = q.evidence; const out = [h("div", { class: "qhead" }, h("h1", {}, q.question), q.title ? h("div", { class: "paper" }, q.title, " ", h("span", { class: "mono" }, `(${q.doc_id})`)) : h("div", { class: "paper" }, "Your question"), q.answers?.length ? h("div", { class: "gold" }, h("span", {}, "Annotator answers"), q.answers.map((a) => h("span", { class: "chip gold-ans" }, a))) : null)]; if (ev) out.push(h("section", { class: "section" }, h("h2", { style: "margin-bottom:6px" }, "Annotator evidence"), h("div", { class: "panel step" }, h("blockquote", { class: "q" }, `"${ev.text}"`), sourceLink(ev.chunk_id, [ev.start, ev.end])))); const g = r.gold; const passages = r.passages.map((p) => h("div", { class: "step" }, h("div", { class: "cites" }, h("span", { class: "cite", style: "cursor:default" }, p.handle), h("b", { style: "font-weight:500" }, p.section), g && p.chunk_id === g.chunk_id ? h("span", { class: "badge good" }, icon("star"), "gold evidence") : null), h("div", { style: "color:var(--text-2);font-size:12.5px" }, p.title), h("div", { class: "usage" }, h("span", {}, `rank ${p.rank}`), h("span", {}, `BM25 ${p.bm25.toFixed(2)}`), h("span", {}, `${p.n_tokens} tokens`)), sourceLink(p.chunk_id))); out.push(h("section", { class: "section" }, h("h2", {}, "Context the text-RAG method would see"), h("p", { class: "lede" }, `BM25 over all ${r.corpus_chunks.toLocaleString("en")} corpus chunks, packed in rank order until the ${r.budget.toLocaleString("en")}-token budget is full. This is the text_rag method's own context builder.`), stats([["Passages", r.passages.length], ["Context tokens", `${r.context_tokens} / ${r.budget}`], ...(g ? [["Gold evidence rank", g.rank], ["Gold in context", g.in_context ? "yes" : "no"]] : [])]), h("div", { class: "panel", style: "margin-top:14px" }, passages.length ? passages : h("p", { style: "color:var(--text-2)" }, "No passage fits the budget.")))); // Claim checker: the annotator evidence by default, against the gold chunk (or the top passage). const options = [...(g && !r.passages.some((p) => p.chunk_id === g.chunk_id) ? [{ chunk_id: g.chunk_id, label: "Gold evidence chunk (not retrieved)" }] : []), ...r.passages.map((p) => ({ chunk_id: p.chunk_id, label: `${p.handle} ${p.section}, ${p.title}` }))]; const defaultIds = new Set([g?.chunk_id ?? r.passages[0]?.chunk_id].filter(Boolean)); const claim = h("textarea", { id: "claim", rows: 3, maxlength: 600 }, ev?.text ?? ""); const boxes = options.map((o) => h("label", { style: "display:flex;gap:8px;align-items:start;font-size:13px;color:var(--text)" }, h("input", { type: "checkbox", value: o.chunk_id, checked: defaultIds.has(o.chunk_id) }), h("span", {}, o.label))); const result = h("div", { role: "status", "aria-live": "polite" }); const run = async (e) => { e.preventDefault(); const ids = boxes.map((b) => b.querySelector("input")).filter((i) => i.checked).map((i) => i.value); if (!ids.length) { result.replaceChildren(h("div", { class: "callout warn" }, icon("warning"), "Select at least one passage.")); return; } if (ids.length > 3) { result.replaceChildren(h("div", { class: "callout warn" }, icon("warning"), "Select at most three passages.")); return; } result.replaceChildren(h("div", { class: "skeleton", style: "height:90px" })); try { const c = await api("/demo/check", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ claim: claim.value, chunk_ids: ids }) }); result.replaceChildren(checkResult(c)); } catch (err) { result.replaceChildren(errorState(err)); } }; const form = h("form", { class: "panel review-form", onsubmit: run }, h("label", { for: "claim" }, "Claim"), claim, h("fieldset", { style: "border:0;padding:0;margin:0;display:grid;gap:6px" }, h("legend", { style: "font-size:12.5px;color:var(--text-2);margin-bottom:6px" }, "Check it against (up to three passages)"), boxes), h("div", { class: "btn-row" }, h("button", { type: "submit", class: "btn primary", disabled: !health.nli }, icon("seal-check"), "Check claim")), !health.nli ? h("div", { class: "callout warn" }, icon("warning"), "The NLI verifier is not installed on this server.") : null, result); out.push(h("section", { class: "section" }, h("h2", {}, "Check a claim against its sources"), h("p", { class: "lede" }, "Exact anchoring accepts only text that occurs verbatim in the passage (after whitespace and typographic normalisation), the rule every extracted edge must pass. The NLI score is the best three-sentence window; 0.5 or above counts as supported, otherwise a contradiction score of 0.5 or above in any selected passage labels it contradicted. These are the verifier's real outputs, mistakes included. Edit the claim to see both checks react."), form)); return out; } function checkResult(c) { const lab = { entailed: ["good", "seal-check", "Supported"], contradicted: ["bad", "x-circle", "Contradicted"], not_enough_evidence: ["warn", "question", "Not supported"] }[c.nli?.label] || ["neutral", "circle", c.nli?.label ?? "n/a"]; return h("div", { style: "display:grid;gap:12px;margin-top:6px" }, c.nli ? h("div", { class: "cites" }, h("span", { class: `badge ${lab[0]}` }, icon(lab[1]), lab[2]), h("span", { class: "usage" }, h("span", {}, `entailment ${c.nli.entail.toFixed(3)}`), h("span", {}, `contradiction ${c.nli.contradict.toFixed(3)}`), c.nli.joint ? h("span", {}, `joint ${c.nli.joint.entail.toFixed(3)}`) : null, h("span", {}, c.nli.model))) : null, c.passages.map((p) => h("div", { class: "step" }, h("div", { class: "cites" }, h("b", { style: "font-weight:500" }, p.section), h("span", { style: "color:var(--text-3);font-size:12px" }, p.title)), h("div", { class: "cites" }, p.anchor ? h("span", { class: "badge good" }, icon("anchor-simple"), `found verbatim at ${p.anchor.start}-${p.anchor.end}`) : h("span", { class: "badge neutral" }, icon("anchor-simple"), "not found verbatim"), p.nli ? h("span", { class: "badge neutral" }, `entailment ${p.nli.entail.toFixed(3)}, contradiction ${p.nli.contradict.toFixed(3)}`) : null), p.nli?.window ? h("blockquote", { class: "q" }, p.nli.window) : null, h("div", { class: "btn-row" }, p.anchor ? h("button", { type: "button", class: "src-link", onclick: () => openSource(p.chunk_id, [p.anchor.start, p.anchor.end]) }, icon("file-text"), " show the anchored claim") : null, p.nli?.window_start != null ? h("button", { type: "button", class: "src-link", onclick: () => openSource(p.chunk_id, [p.nli.window_start, p.nli.window_end]) }, icon("file-text"), " show the best window") : sourceLink(p.chunk_id))))); } // ---------- source drawer ---------- const drawer = document.getElementById("drawer"); function closeDrawer() { drawer.classList.remove("open"); drawer.setAttribute("aria-hidden", "true"); } document.getElementById("drawer-close").addEventListener("click", closeDrawer); document.addEventListener("keydown", (e) => { if (e.key === "Escape") closeDrawer(); }); async function openSource(chunkId, range) { let c; try { c = await api(`/chunks/${encodeURIComponent(chunkId)}`); } catch (e) { document.getElementById("drawer-body").replaceChildren(errorState(e)); drawer.classList.add("open"); return; } document.getElementById("drawer-kicker").textContent = chunkId; document.getElementById("drawer-title").textContent = c.section_heading; const text = c.text; const body = h("div", { class: "source" }); const [a, b] = range && range[0] != null ? range : [-1, -1]; const pieces = a >= 0 ? [[0, a, false], [a, b, true], [b, text.length, false]] : [[0, text.length, false]]; for (const [x, y, marked] of pieces) { for (const part of text.slice(x, y).split(/(§ [^\n]*)/)) { if (!part) continue; let node = part.startsWith("§ ") ? h("span", { class: "sec" }, part.slice(2)) : document.createTextNode(part); if (marked) node = h("mark", {}, node); body.append(node); } } document.getElementById("drawer-body").replaceChildren( h("div", { class: "callout info" }, icon("file-text"), c.paper_title || c.doc_id), a >= 0 ? h("div", { style: "color:var(--text-3);font-size:12px" }, "The highlighted span is shown at its exact character offsets in the source chunk.") : null, body); drawer.classList.add("open"); drawer.setAttribute("aria-hidden", "false"); document.getElementById("drawer-close").focus({ preventScroll: true }); body.querySelector("mark")?.scrollIntoView({ block: "center" }); } window.addEventListener("hashchange", route); route();