File size: 11,754 Bytes
c0ff0d3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ff18450
c0ff0d3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
ff18450
 
c0ff0d3
 
 
 
 
 
 
 
ff18450
c0ff0d3
 
 
 
ff18450
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c0ff0d3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
const METHODS = {
  hybrid: { label: "Hybrid E5 + rerank", color: "#4f46e5" },
  graph: { label: "Retained E5 + selected graph", color: "#0f766e" },
  paths: { label: "Retained E5 + graph paths", color: "#c25b16" },
  both: { label: "Retained E5 + graph + paths", color: "#9a6700" }
};

const state = { benchmark: [], byId: {}, replay: null, sources: {}, corpus: {}, filtered: [] };
const $ = id => document.getElementById(id);
const esc = value => String(value ?? "").replace(/[&<>"']/g, c => ({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;"}[c]));
const parseJsonl = text => text.split(/\r?\n/).filter(Boolean).map(line => JSON.parse(line));

function parseCsv(text) {
  const lines = text.trim().split(/\r?\n/);
  const headers = lines.shift().split(",");
  return lines.map(line => {
    const cells = line.match(/("(?:[^"]|"")*"|[^,]*)(?:,|$)/g).map(v => v.replace(/,$/, "").replace(/^"|"$/g, "").replace(/""/g, '"'));
    return Object.fromEntries(headers.map((h, i) => [h, cells[i] || ""]));
  });
}

async function loadCorpus(year) {
  if (state.corpus[year]) return state.corpus[year];
  const response = await fetch(`corpus/eval_corpus_${year}.jsonl`);
  if (!response.ok) throw new Error(`Could not load the ${year} evidence corpus.`);
  const rows = parseJsonl(await response.text());
  state.corpus[year] = Object.fromEntries(rows.map(row => [row.object_id, row]));
  return state.corpus[year];
}

function goldStatus(question, candidate) {
  const evidence = question.evidence || [];
  if (evidence.some(e => e.object_id === candidate.object_id)) return { css: "gold", label: "Exact gold evidence" };
  if (evidence.some(e => String(e.page) === String(candidate.page))) return { css: "page", label: "Gold page, different object" };
  return { css: "miss", label: "Not annotated gold" };
}

function provenance(reasons = []) {
  const labels = [];
  for (const reason of reasons) {
    if (reason === "base" || reason.startsWith("source:hybrid")) labels.push("Hybrid");
    else if (reason.startsWith("source:without_adjacent_page")) labels.push("Selected graph");
    else if (reason.startsWith("query_metric:")) labels.push(`Query metric: ${reason.split(":")[1]}`);
    else if (reason.startsWith("query_entity:")) labels.push(`Query entity: ${reason.split(":")[1]}`);
    else if (reason.startsWith("path:same_page:")) labels.push("Same page");
    else if (reason.startsWith("path:same_metric:")) labels.push(`Same metric: ${reason.split(":").at(-1)}`);
    else if (reason.startsWith("path:same_entity:")) labels.push(`Same entity: ${reason.split(":").at(-1)}`);
  }
  return [...new Set(labels.length ? labels : ["Saved ranking"])];
}

function applyFilters() {
  const year = $("year-filter").value;
  const type = $("type-filter").value;
  const keyword = $("keyword-filter").value.trim().toLowerCase();
  state.filtered = state.benchmark.filter(q => state.replay.results[q.question_id])
    .filter(q => year === "All years" || String(q.report_year) === year)
    .filter(q => type === "All types" || q.question_type === type)
    .filter(q => !keyword || q.question.toLowerCase().includes(keyword));
  $("question-count").textContent = `${state.filtered.length} questions`;
  $("question-select").innerHTML = state.filtered.map(q => `<option value="${esc(q.question_id)}">${esc(q.report_year)} | ${esc(q.question)}</option>`).join("");
  render();
}

async function render() {
  const qid = $("question-select").value;
  if (!qid) {
    $("question-summary").innerHTML = "<p>No questions match these filters.</p>";
    $("comparison").innerHTML = "";
    $("ranking").innerHTML = "";
    return;
  }
  const question = state.byId[qid];
  $("question-summary").innerHTML = `<h3>${esc(question.question)}</h3><p><strong>Gold answer:</strong> ${esc(question.answer || "No annotated answer")}</p><p>${esc(question.report_year)} annual report 路 ${esc(question.question_type.replaceAll("_", " "))}</p>`;
  $("comparison").innerHTML = Object.entries(METHODS).map(([key, method]) => {
    const top = state.replay.results[qid][key].top_k;
    const exact = top.filter(c => goldStatus(question, c).css === "gold");
    const page = top.filter(c => ["gold", "page"].includes(goldStatus(question, c).css));
    const first = exact.length ? exact[0].rank : "-";
    return `<article class="metric-block" style="--accent:${method.color}"><h3>${esc(method.label)}</h3><div class="metric-row"><span>Exact objects</span><strong>${exact.length}</strong></div><div class="metric-row"><span>Gold-page objects</span><strong>${page.length}</strong></div><div class="metric-row"><span>First exact rank</span><strong>${first}</strong></div></article>`;
  }).join("");
  await renderRanking(question);
}

async function renderRanking(question) {
  const key = $("method-select").value || "both";
  const method = METHODS[key];
  const result = state.replay.results[question.question_id][key];
  $("ranking").innerHTML = '<div class="loading">Loading evidence text...</div>';
  try {
    const corpus = await loadCorpus(question.report_year);
    const trace = result.trace || {};
    const cues = [];
    if (trace.years?.length) cues.push(`Year: ${trace.years.join(", ")}`);
    if (trace.entities?.length) cues.push(`Entities: ${trace.entities.join(", ")}`);
    if (trace.metrics?.length) cues.push(`Metrics: ${trace.metrics.join(", ")}`);
    const allLabels = [...new Set(result.top_k.flatMap(c => provenance(c.reasons)))];
    $("route-summary").style.setProperty("--accent", method.color);
    $("route-summary").innerHTML = `<strong>${esc(method.label)}</strong><span>${esc(cues.join(" 路 ") || "Saved ranking without path cues")}</span><div class="chips">${allLabels.map(x => `<span class="chip">${esc(x)}</span>`).join("")}</div>`;
    renderGraph(question, key, result, corpus);
    $("ranking").innerHTML = result.top_k.map(candidate => {
      const obj = corpus[candidate.object_id] || {};
      const status = goldStatus(question, candidate);
      const base = state.sources[question.document_id] || "";
      const link = base ? `${base}#page=${candidate.page}` : "#";
      return `<article class="evidence ${status.css}"><div class="rank">#${candidate.rank}</div><div><div class="match">${esc(status.label)}</div><div class="chips">${provenance(candidate.reasons).map(x => `<span class="chip">${esc(x)}</span>`).join("")}</div></div><div class="meta">Page ${esc(candidate.page)}<br>${esc(candidate.type || obj.object_type || "")}</div><div class="preview"><strong>${esc(candidate.object_id)}</strong><p>${esc((obj.text || "Evidence text unavailable").replace(/\s+/g, " ").slice(0, 520))}</p></div><a class="source" href="${esc(link)}" target="_blank" rel="noopener">PDF</a></article>`;
    }).join("");
  } catch (error) {
    $("graph-view").innerHTML = `<div class="error">${esc(error.message)}</div>`;
    $("ranking").innerHTML = `<div class="error">${esc(error.message)}</div>`;
  }
}

function renderGraph(question, key, result, corpus) {
  const trace = result.trace || {};
  const years = trace.years?.length ? trace.years : [question.report_year];
  const entities = trace.entities || [];
  const metrics = trace.metrics || [];
  const cueNodes = [
    ...years.map(year => `<div class="graph-node cue year"><strong>Year</strong><span>${esc(year)}</span></div>`),
    ...entities.map(entity => `<div class="graph-node cue entity"><strong>Entity</strong><span>${esc(entity)}</span></div>`),
    ...metrics.map(metric => `<div class="graph-node cue metric"><strong>Metric</strong><span>${esc(metric)}</span></div>`)
  ].join("");

  const graphRows = result.top_k.slice(0, 5).map(candidate => {
    const obj = corpus[candidate.object_id] || {};
    const status = goldStatus(question, candidate);
    const reasons = candidate.reasons || [];
    let source = "BM25 + E5";
    let sourceClass = "hybrid";
    let relation = "retrieves";
    let edgeClass = "hybrid";
    if (reasons.some(reason => reason.startsWith("path:same_metric:"))) {
      source = `Metric: ${metrics[0] || "query metric"}`;
      sourceClass = "metric";
      relation = "same-metric";
      edgeClass = "metric";
    } else if (reasons.some(reason => reason.startsWith("path:same_entity:"))) {
      source = `Entity: ${entities[0] || "query entity"}`;
      sourceClass = "entity";
      relation = "same-entity";
      edgeClass = "entity";
    } else if (reasons.some(reason => reason.startsWith("path:same_page:"))) {
      source = "Page-linked object";
      sourceClass = "page";
      relation = "same-page";
      edgeClass = "page";
    } else if (reasons.some(reason => reason.startsWith("source:without_adjacent_page"))) {
      source = "Hybrid seed object";
      sourceClass = "graph";
      relation = "selected-graph expansion";
      edgeClass = "graph";
    }
    const text = (obj.text || candidate.object_id).replace(/\s+/g, " ").slice(0, 115);
    return `<div class="graph-path-row"><div class="graph-source ${sourceClass}">${esc(source)}</div><div class="graph-edge ${edgeClass}"><span>${esc(relation)}</span></div><div class="graph-node evidence-node ${status.css}"><strong>#${candidate.rank} 路 Page ${esc(candidate.page)}</strong><span>${esc(text)}</span></div></div>`;
  }).join("");
  $("graph-view").style.setProperty("--accent", METHODS[key].color);
  $("graph-view").innerHTML = `<div class="graph-root graph-node query"><strong>Question</strong><span>${esc(question.question)}</span></div><div class="graph-down" aria-hidden="true"></div><div class="graph-cues">${cueNodes || `<div class="graph-node cue year"><strong>Year</strong><span>${esc(question.report_year)}</span></div>`}</div><div class="graph-section-label">Active node-to-node routes</div><div class="graph-paths">${graphRows}</div><div class="graph-legend"><span class="legend-line hybrid">Hybrid edge</span><span class="legend-line graph">Graph expansion edge</span><span class="legend-line path">Typed graph edge</span><span>Green node = exact gold 路 Amber node = gold page</span></div>`;
}

async function init() {
  try {
    const [benchmarkRes, replayRes, sourceRes] = await Promise.all([
      fetch("finrag_equinor_v1.0.jsonl"), fetch("graph_replay.json"), fetch("source_manifest.csv")
    ]);
    if (!benchmarkRes.ok || !replayRes.ok || !sourceRes.ok) throw new Error("One or more demo data files could not be loaded.");
    state.benchmark = parseJsonl(await benchmarkRes.text()).filter(q => q.is_answerable);
    state.byId = Object.fromEntries(state.benchmark.map(q => [q.question_id, q]));
    state.replay = await replayRes.json();
    state.sources = Object.fromEntries(parseCsv(await sourceRes.text()).map(row => [row.document_id, row.download_url]));
    const years = [...new Set(state.benchmark.map(q => q.report_year))].sort();
    $("year-filter").innerHTML += years.map(y => `<option>${y}</option>`).join("");
    const types = [...new Set(state.benchmark.map(q => q.question_type))].sort();
    $("type-filter").innerHTML += types.map(t => `<option value="${esc(t)}">${esc(t.replaceAll("_", " "))}</option>`).join("");
    $("method-select").innerHTML = Object.entries(METHODS).map(([k, v]) => `<option value="${k}" ${k === "both" ? "selected" : ""}>${esc(v.label)}</option>`).join("");
    ["year-filter", "type-filter"].forEach(id => $(id).addEventListener("change", applyFilters));
    $("keyword-filter").addEventListener("input", applyFilters);
    $("question-select").addEventListener("change", render);
    $("method-select").addEventListener("change", () => renderRanking(state.byId[$("question-select").value]));
    applyFilters();
  } catch (error) {
    $("question-count").textContent = "Load failed";
    $("question-summary").innerHTML = `<div class="error">${esc(error.message)}</div>`;
  }
}

init();