Spaces:
Running
Running
File size: 12,432 Bytes
41b7006 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 | """Paper-aligned GraphRAG retrieval replay for FinRAG-Equinor."""
import csv
import json
from html import escape
from pathlib import Path
import gradio as gr
ROOT = Path(__file__).parent
METHODS = {
"hybrid": "Hybrid E5 + rerank",
"graph": "Retained E5 + selected graph",
"paths": "Retained E5 + graph paths",
"both": "Retained E5 + graph + paths",
}
COLORS = {"hybrid": "#4f46e5", "graph": "#0f766e", "paths": "#d97706", "both": "#a16207"}
def load_jsonl(path):
with path.open(encoding="utf-8") as handle:
return [json.loads(line) for line in handle if line.strip()]
BENCHMARK = load_jsonl(ROOT / "finrag_equinor_v1.0.jsonl")
BENCHMARK_BY_ID = {row["question_id"]: row for row in BENCHMARK}
with (ROOT / "graph_replay.json").open(encoding="utf-8") as handle:
REPLAY = json.load(handle)
with (ROOT / "source_manifest.csv").open(encoding="utf-8-sig", newline="") as handle:
SOURCES = {row["document_id"]: row["download_url"] for row in csv.DictReader(handle)}
CORPUS = {}
for path in sorted((ROOT / "corpus").glob("eval_corpus_*.jsonl")):
for row in load_jsonl(path):
CORPUS[row["object_id"]] = row
QUESTION_LABELS = {
f'{row["report_year"]} | {row["question"]}': row["question_id"]
for row in BENCHMARK if row["question_id"] in REPLAY["results"]
}
QUESTIONS = list(QUESTION_LABELS)
TYPES = sorted({BENCHMARK_BY_ID[qid]["question_type"] for qid in QUESTION_LABELS.values()})
def pdf_url(document_id, page):
base = SOURCES.get(document_id, "")
return f"{base}#page={page}" if base else ""
def match_label(item, object_id, page):
gold_ids = set(item.get("gold_evidence_object_ids", []))
if not gold_ids:
gold_ids = {e.get("object_id") for e in item.get("evidence", [])}
gold_pages = {str(p) for p in item.get("gold_evidence_pages", [])}
if not gold_pages:
gold_pages = {str(e.get("page")) for e in item.get("evidence", [])}
if object_id in gold_ids:
return "Exact gold evidence", "gold"
if str(page) in gold_pages:
return "Gold page, different object", "page"
return "Not annotated gold", "miss"
def provenance(reasons):
labels = []
for reason in reasons or []:
if reason == "base" or reason.startswith("source:hybrid"):
labels.append("Hybrid")
elif reason.startswith("source:without_adjacent_page"):
labels.append("Selected graph")
elif reason.startswith("query_metric:"):
labels.append("Query metric: " + reason.split(":", 1)[1])
elif reason.startswith("query_entity:"):
labels.append("Query entity: " + reason.split(":", 1)[1])
elif reason.startswith("path:same_page:"):
labels.append("Same page")
elif reason.startswith("path:same_metric:"):
labels.append("Same metric: " + reason.rsplit(":", 1)[1])
elif reason.startswith("path:same_entity:"):
labels.append("Same entity: " + reason.rsplit(":", 1)[1])
return list(dict.fromkeys(labels)) or ["Saved ranking"]
def render_ranking(item, method):
data = REPLAY["results"][item["question_id"]][method]
rows = []
for candidate in data["top_k"]:
obj = CORPUS.get(candidate["object_id"], {})
page = candidate.get("page", obj.get("page"))
label, css = match_label(item, candidate["object_id"], page)
source = " · ".join(provenance(candidate.get("reasons")))
text = " ".join(obj.get("text", "").split())
link = pdf_url(item["document_id"], page)
rows.append(
f'<tr class="{css}"><td>{candidate["rank"]}</td><td>{escape(label)}</td>'
f'<td>{escape(str(page))}</td><td>{escape(str(candidate.get("type") or obj.get("object_type", "")))}</td>'
f'<td><span class="source-tag">{escape(source)}</span></td>'
f'<td>{escape(text[:360])}</td><td><a href="{escape(link, quote=True)}" target="_blank">PDF</a></td></tr>'
)
return (
'<div class="evidence-table"><table><thead><tr><th>Rank</th><th>Evidence match</th>'
'<th>Page</th><th>Type</th><th>Candidate source</th><th>Evidence preview</th><th>Source</th>'
'</tr></thead><tbody>' + "".join(rows) + '</tbody></table></div>'
)
def render_trace(item, method):
result = REPLAY["results"][item["question_id"]][method]
trace = result.get("trace") or {}
candidates = result["top_k"][:5]
relation_counts = {}
for candidate in candidates:
for label in provenance(candidate.get("reasons")):
relation_counts[label] = relation_counts.get(label, 0) + 1
chips = "".join(
f'<span class="trace-chip">{escape(k)} <b>{v}</b></span>' for k, v in relation_counts.items()
) or '<span class="trace-chip">Hybrid ranking only</span>'
cues = []
if trace.get("years"):
cues.append("Year: " + ", ".join(map(str, trace["years"])))
if trace.get("entities"):
cues.append("Entities: " + ", ".join(trace["entities"]))
if trace.get("metrics"):
cues.append("Metrics: " + ", ".join(trace["metrics"]))
cue_text = " · ".join(cues) or "No stored path cues for this method"
color = COLORS[method]
nodes = []
for index, candidate in enumerate(candidates):
obj = CORPUS.get(candidate["object_id"], {})
label, css = match_label(item, candidate["object_id"], candidate.get("page"))
nodes.append(
f'<div class="evidence-node {css}"><b>#{index + 1} · page {candidate.get("page")}</b>'
f'<span>{escape(obj.get("text", "")[:115])}</span><small>{escape(label)}</small></div>'
)
return (
f'<div class="trace-panel" style="--accent:{color}"><div class="trace-root">'
f'<b>{escape(METHODS[method])}</b><span>{escape(cue_text)}</span></div>'
f'<div class="trace-arrow">Candidate provenance ↓</div><div class="trace-chips">{chips}</div>'
f'<div class="trace-nodes">{"".join(nodes)}</div></div>'
)
def show(question_label, method_label):
if question_label not in QUESTION_LABELS:
return "Select a question.", "", ""
method = next((key for key, label in METHODS.items() if label == method_label), "both")
item = BENCHMARK_BY_ID[QUESTION_LABELS[question_label]]
metric = REPLAY["metrics"][method]
answer = item.get("answer") or "No answer is annotated."
summary = (
f'### {item["question"]}\n\n**Gold answer:** {answer}\n\n'
f'**Report:** {item["report_year"]} · **Type:** {item["question_type"].replace("_", " ")} · '
f'**Method:** {METHODS[method]}\n\n'
f'Paper-level metrics: Object R@10 **{metric["object_r10"]:.3f}** · '
f'Page R@10 **{metric["page_r10"]:.3f}** · MRR **{metric["mrr"]:.3f}**'
)
return summary, render_trace(item, method), render_ranking(item, method)
def compare(question_label):
if question_label not in QUESTION_LABELS:
return "Select a question."
item = BENCHMARK_BY_ID[QUESTION_LABELS[question_label]]
blocks = []
for method in METHODS:
top = REPLAY["results"][item["question_id"]][method]["top_k"]
exact = sum(match_label(item, c["object_id"], c.get("page"))[1] == "gold" for c in top)
page = sum(match_label(item, c["object_id"], c.get("page"))[1] in {"gold", "page"} for c in top)
best = next((c["rank"] for c in top if match_label(item, c["object_id"], c.get("page"))[1] == "gold"), "—")
blocks.append(
f'<div class="metric-block" style="border-top-color:{COLORS[method]}"><b>{METHODS[method]}</b>'
f'<span>Exact objects in top-10<strong>{exact}</strong></span>'
f'<span>Objects from gold pages<strong>{page}</strong></span>'
f'<span>First exact rank<strong>{best}</strong></span></div>'
)
return '<div class="comparison">' + "".join(blocks) + '</div>'
def filter_questions(year, question_type, keyword):
keyword = (keyword or "").strip().lower()
labels = []
for label, qid in QUESTION_LABELS.items():
item = BENCHMARK_BY_ID[qid]
if year != "All years" and str(item["report_year"]) != year:
continue
if question_type != "All types" and item["question_type"] != question_type:
continue
if keyword and keyword not in item["question"].lower():
continue
labels.append(label)
return gr.update(choices=labels, value=labels[0] if labels else None), f"**{len(labels)} questions**"
CSS = """
.gradio-container {max-width: 1240px !important;}
.subtitle {color:#52617a;margin-top:-8px;}
footer {display:none!important;}
.comparison {display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px;margin:10px 0 22px;}
.metric-block {border:1px solid #d8dee9;border-top:4px solid;padding:14px;background:#fff;border-radius:6px;}
.metric-block>b {display:block;min-height:42px;color:#18212f;}
.metric-block span {display:flex;justify-content:space-between;gap:10px;padding-top:8px;font-size:13px;color:#52617a;}
.metric-block strong {color:#18212f;font-size:16px;}
.trace-panel {border:1px solid #d8dee9;border-left:5px solid var(--accent);padding:16px;border-radius:6px;background:#fbfcfe;}
.trace-root {display:flex;justify-content:space-between;gap:16px;flex-wrap:wrap;}.trace-root span{color:#52617a;}
.trace-arrow {text-align:center;color:#687386;padding:10px;}.trace-chips{display:flex;gap:7px;flex-wrap:wrap;justify-content:center;}
.trace-chip,.source-tag {background:#eef2f7;border:1px solid #d5dce7;border-radius:4px;padding:4px 7px;font-size:12px;}
.trace-nodes {display:grid;grid-template-columns:repeat(5,minmax(0,1fr));gap:8px;margin-top:15px;}
.evidence-node {border:1px solid #cbd5e1;padding:9px;border-radius:5px;background:white;min-width:0;}
.evidence-node b,.evidence-node span,.evidence-node small{display:block}.evidence-node span{font-size:12px;margin:6px 0;overflow-wrap:anywhere}.evidence-node small{color:#64748b}
.evidence-node.gold{background:#dcfce7;border-color:#86d49d}.evidence-node.page{background:#fef9c3;border-color:#e6d86b}
.evidence-table {overflow-x:auto;}.evidence-table table{border-collapse:collapse;width:100%;font-size:13px;}
.evidence-table th,.evidence-table td{padding:9px;border:1px solid #d8dee9;text-align:left;vertical-align:top;overflow-wrap:anywhere;}
.evidence-table th{background:#f1f5f9}.evidence-table tr.gold td{background:#effcf3}.evidence-table tr.page td{background:#fffced}.evidence-table a{text-decoration:underline}
@media(max-width:800px){.comparison{grid-template-columns:1fr 1fr}.trace-nodes{grid-template-columns:1fr 1fr}}
"""
with gr.Blocks(css=CSS, title="FinRAG GraphRAG Demo") as demo:
gr.Markdown("# FinRAG GraphRAG Evidence Explorer\n<p class='subtitle'>Inspect how saved hybrid, graph-expansion, and graph-path rankings retrieve evidence from long annual reports.</p>")
gr.Markdown("This demo replays the paper's saved outputs. Green marks exact annotated evidence; yellow marks a different object on an annotated page.")
with gr.Row():
year = gr.Dropdown(["All years"] + [str(y) for y in range(2010, 2025)], value="All years", label="Year")
qtype = gr.Dropdown(["All types"] + TYPES, value="All types", label="Question type")
keyword = gr.Textbox(label="Question contains", placeholder="production, revenue, emissions...")
count = gr.Markdown(f"**{len(QUESTIONS)} questions**")
question = gr.Dropdown(QUESTIONS, value=QUESTIONS[0], label="Benchmark question")
for component in (year, qtype, keyword):
component.change(filter_questions, [year, qtype, keyword], [question, count])
gr.Markdown("## Per-question comparison")
compare_button = gr.Button("Compare all methods", variant="primary")
comparison = gr.HTML()
compare_button.click(compare, question, comparison)
gr.Markdown("## Inspect one retrieval route")
method = gr.Radio(list(METHODS.values()), value=METHODS["both"], label="Method")
inspect = gr.Button("Inspect saved Top-10", variant="primary")
summary = gr.Markdown()
trace = gr.HTML()
ranking = gr.HTML()
inspect.click(show, [question, method], [summary, trace, ranking])
gr.Markdown("*Research demo. Reference report and year come from the benchmark. Graph paths are fixed typed expansions, not an autonomous agent.*")
if __name__ == "__main__":
demo.queue(default_concurrency_limit=4).launch(ssr_mode=False)
|