Spaces:
Running
Running
File size: 11,754 Bytes
c0ff0d3 ff18450 c0ff0d3 ff18450 c0ff0d3 ff18450 c0ff0d3 ff18450 c0ff0d3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 | const METHODS = {
hybrid: { label: "Hybrid E5 + rerank", color: "#4f46e5" },
graph: { label: "Retained E5 + selected graph", color: "#0f766e" },
paths: { label: "Retained E5 + graph paths", color: "#c25b16" },
both: { label: "Retained E5 + graph + paths", color: "#9a6700" }
};
const state = { benchmark: [], byId: {}, replay: null, sources: {}, corpus: {}, filtered: [] };
const $ = id => document.getElementById(id);
const esc = value => String(value ?? "").replace(/[&<>"']/g, c => ({"&":"&","<":"<",">":">",'"':""","'":"'"}[c]));
const parseJsonl = text => text.split(/\r?\n/).filter(Boolean).map(line => JSON.parse(line));
function parseCsv(text) {
const lines = text.trim().split(/\r?\n/);
const headers = lines.shift().split(",");
return lines.map(line => {
const cells = line.match(/("(?:[^"]|"")*"|[^,]*)(?:,|$)/g).map(v => v.replace(/,$/, "").replace(/^"|"$/g, "").replace(/""/g, '"'));
return Object.fromEntries(headers.map((h, i) => [h, cells[i] || ""]));
});
}
async function loadCorpus(year) {
if (state.corpus[year]) return state.corpus[year];
const response = await fetch(`corpus/eval_corpus_${year}.jsonl`);
if (!response.ok) throw new Error(`Could not load the ${year} evidence corpus.`);
const rows = parseJsonl(await response.text());
state.corpus[year] = Object.fromEntries(rows.map(row => [row.object_id, row]));
return state.corpus[year];
}
function goldStatus(question, candidate) {
const evidence = question.evidence || [];
if (evidence.some(e => e.object_id === candidate.object_id)) return { css: "gold", label: "Exact gold evidence" };
if (evidence.some(e => String(e.page) === String(candidate.page))) return { css: "page", label: "Gold page, different object" };
return { css: "miss", label: "Not annotated gold" };
}
function provenance(reasons = []) {
const labels = [];
for (const reason of reasons) {
if (reason === "base" || reason.startsWith("source:hybrid")) labels.push("Hybrid");
else if (reason.startsWith("source:without_adjacent_page")) labels.push("Selected graph");
else if (reason.startsWith("query_metric:")) labels.push(`Query metric: ${reason.split(":")[1]}`);
else if (reason.startsWith("query_entity:")) labels.push(`Query entity: ${reason.split(":")[1]}`);
else if (reason.startsWith("path:same_page:")) labels.push("Same page");
else if (reason.startsWith("path:same_metric:")) labels.push(`Same metric: ${reason.split(":").at(-1)}`);
else if (reason.startsWith("path:same_entity:")) labels.push(`Same entity: ${reason.split(":").at(-1)}`);
}
return [...new Set(labels.length ? labels : ["Saved ranking"])];
}
function applyFilters() {
const year = $("year-filter").value;
const type = $("type-filter").value;
const keyword = $("keyword-filter").value.trim().toLowerCase();
state.filtered = state.benchmark.filter(q => state.replay.results[q.question_id])
.filter(q => year === "All years" || String(q.report_year) === year)
.filter(q => type === "All types" || q.question_type === type)
.filter(q => !keyword || q.question.toLowerCase().includes(keyword));
$("question-count").textContent = `${state.filtered.length} questions`;
$("question-select").innerHTML = state.filtered.map(q => `<option value="${esc(q.question_id)}">${esc(q.report_year)} | ${esc(q.question)}</option>`).join("");
render();
}
async function render() {
const qid = $("question-select").value;
if (!qid) {
$("question-summary").innerHTML = "<p>No questions match these filters.</p>";
$("comparison").innerHTML = "";
$("ranking").innerHTML = "";
return;
}
const question = state.byId[qid];
$("question-summary").innerHTML = `<h3>${esc(question.question)}</h3><p><strong>Gold answer:</strong> ${esc(question.answer || "No annotated answer")}</p><p>${esc(question.report_year)} annual report 路 ${esc(question.question_type.replaceAll("_", " "))}</p>`;
$("comparison").innerHTML = Object.entries(METHODS).map(([key, method]) => {
const top = state.replay.results[qid][key].top_k;
const exact = top.filter(c => goldStatus(question, c).css === "gold");
const page = top.filter(c => ["gold", "page"].includes(goldStatus(question, c).css));
const first = exact.length ? exact[0].rank : "-";
return `<article class="metric-block" style="--accent:${method.color}"><h3>${esc(method.label)}</h3><div class="metric-row"><span>Exact objects</span><strong>${exact.length}</strong></div><div class="metric-row"><span>Gold-page objects</span><strong>${page.length}</strong></div><div class="metric-row"><span>First exact rank</span><strong>${first}</strong></div></article>`;
}).join("");
await renderRanking(question);
}
async function renderRanking(question) {
const key = $("method-select").value || "both";
const method = METHODS[key];
const result = state.replay.results[question.question_id][key];
$("ranking").innerHTML = '<div class="loading">Loading evidence text...</div>';
try {
const corpus = await loadCorpus(question.report_year);
const trace = result.trace || {};
const cues = [];
if (trace.years?.length) cues.push(`Year: ${trace.years.join(", ")}`);
if (trace.entities?.length) cues.push(`Entities: ${trace.entities.join(", ")}`);
if (trace.metrics?.length) cues.push(`Metrics: ${trace.metrics.join(", ")}`);
const allLabels = [...new Set(result.top_k.flatMap(c => provenance(c.reasons)))];
$("route-summary").style.setProperty("--accent", method.color);
$("route-summary").innerHTML = `<strong>${esc(method.label)}</strong><span>${esc(cues.join(" 路 ") || "Saved ranking without path cues")}</span><div class="chips">${allLabels.map(x => `<span class="chip">${esc(x)}</span>`).join("")}</div>`;
renderGraph(question, key, result, corpus);
$("ranking").innerHTML = result.top_k.map(candidate => {
const obj = corpus[candidate.object_id] || {};
const status = goldStatus(question, candidate);
const base = state.sources[question.document_id] || "";
const link = base ? `${base}#page=${candidate.page}` : "#";
return `<article class="evidence ${status.css}"><div class="rank">#${candidate.rank}</div><div><div class="match">${esc(status.label)}</div><div class="chips">${provenance(candidate.reasons).map(x => `<span class="chip">${esc(x)}</span>`).join("")}</div></div><div class="meta">Page ${esc(candidate.page)}<br>${esc(candidate.type || obj.object_type || "")}</div><div class="preview"><strong>${esc(candidate.object_id)}</strong><p>${esc((obj.text || "Evidence text unavailable").replace(/\s+/g, " ").slice(0, 520))}</p></div><a class="source" href="${esc(link)}" target="_blank" rel="noopener">PDF</a></article>`;
}).join("");
} catch (error) {
$("graph-view").innerHTML = `<div class="error">${esc(error.message)}</div>`;
$("ranking").innerHTML = `<div class="error">${esc(error.message)}</div>`;
}
}
function renderGraph(question, key, result, corpus) {
const trace = result.trace || {};
const years = trace.years?.length ? trace.years : [question.report_year];
const entities = trace.entities || [];
const metrics = trace.metrics || [];
const cueNodes = [
...years.map(year => `<div class="graph-node cue year"><strong>Year</strong><span>${esc(year)}</span></div>`),
...entities.map(entity => `<div class="graph-node cue entity"><strong>Entity</strong><span>${esc(entity)}</span></div>`),
...metrics.map(metric => `<div class="graph-node cue metric"><strong>Metric</strong><span>${esc(metric)}</span></div>`)
].join("");
const graphRows = result.top_k.slice(0, 5).map(candidate => {
const obj = corpus[candidate.object_id] || {};
const status = goldStatus(question, candidate);
const reasons = candidate.reasons || [];
let source = "BM25 + E5";
let sourceClass = "hybrid";
let relation = "retrieves";
let edgeClass = "hybrid";
if (reasons.some(reason => reason.startsWith("path:same_metric:"))) {
source = `Metric: ${metrics[0] || "query metric"}`;
sourceClass = "metric";
relation = "same-metric";
edgeClass = "metric";
} else if (reasons.some(reason => reason.startsWith("path:same_entity:"))) {
source = `Entity: ${entities[0] || "query entity"}`;
sourceClass = "entity";
relation = "same-entity";
edgeClass = "entity";
} else if (reasons.some(reason => reason.startsWith("path:same_page:"))) {
source = "Page-linked object";
sourceClass = "page";
relation = "same-page";
edgeClass = "page";
} else if (reasons.some(reason => reason.startsWith("source:without_adjacent_page"))) {
source = "Hybrid seed object";
sourceClass = "graph";
relation = "selected-graph expansion";
edgeClass = "graph";
}
const text = (obj.text || candidate.object_id).replace(/\s+/g, " ").slice(0, 115);
return `<div class="graph-path-row"><div class="graph-source ${sourceClass}">${esc(source)}</div><div class="graph-edge ${edgeClass}"><span>${esc(relation)}</span></div><div class="graph-node evidence-node ${status.css}"><strong>#${candidate.rank} 路 Page ${esc(candidate.page)}</strong><span>${esc(text)}</span></div></div>`;
}).join("");
$("graph-view").style.setProperty("--accent", METHODS[key].color);
$("graph-view").innerHTML = `<div class="graph-root graph-node query"><strong>Question</strong><span>${esc(question.question)}</span></div><div class="graph-down" aria-hidden="true"></div><div class="graph-cues">${cueNodes || `<div class="graph-node cue year"><strong>Year</strong><span>${esc(question.report_year)}</span></div>`}</div><div class="graph-section-label">Active node-to-node routes</div><div class="graph-paths">${graphRows}</div><div class="graph-legend"><span class="legend-line hybrid">Hybrid edge</span><span class="legend-line graph">Graph expansion edge</span><span class="legend-line path">Typed graph edge</span><span>Green node = exact gold 路 Amber node = gold page</span></div>`;
}
async function init() {
try {
const [benchmarkRes, replayRes, sourceRes] = await Promise.all([
fetch("finrag_equinor_v1.0.jsonl"), fetch("graph_replay.json"), fetch("source_manifest.csv")
]);
if (!benchmarkRes.ok || !replayRes.ok || !sourceRes.ok) throw new Error("One or more demo data files could not be loaded.");
state.benchmark = parseJsonl(await benchmarkRes.text()).filter(q => q.is_answerable);
state.byId = Object.fromEntries(state.benchmark.map(q => [q.question_id, q]));
state.replay = await replayRes.json();
state.sources = Object.fromEntries(parseCsv(await sourceRes.text()).map(row => [row.document_id, row.download_url]));
const years = [...new Set(state.benchmark.map(q => q.report_year))].sort();
$("year-filter").innerHTML += years.map(y => `<option>${y}</option>`).join("");
const types = [...new Set(state.benchmark.map(q => q.question_type))].sort();
$("type-filter").innerHTML += types.map(t => `<option value="${esc(t)}">${esc(t.replaceAll("_", " "))}</option>`).join("");
$("method-select").innerHTML = Object.entries(METHODS).map(([k, v]) => `<option value="${k}" ${k === "both" ? "selected" : ""}>${esc(v.label)}</option>`).join("");
["year-filter", "type-filter"].forEach(id => $(id).addEventListener("change", applyFilters));
$("keyword-filter").addEventListener("input", applyFilters);
$("question-select").addEventListener("change", render);
$("method-select").addEventListener("change", () => renderRanking(state.byId[$("question-select").value]));
applyFilters();
} catch (error) {
$("question-count").textContent = "Load failed";
$("question-summary").innerHTML = `<div class="error">${esc(error.message)}</div>`;
}
}
init();
|