nonnan's picture
Upload GraphRAG evidence explorer
41b7006 verified
Raw History Blame Contribute Delete
12.4 kB
"""Paper-aligned GraphRAG retrieval replay for FinRAG-Equinor."""
import csv
import json
from html import escape
from pathlib import Path
import gradio as gr
ROOT = Path(__file__).parent
METHODS = {
"hybrid": "Hybrid E5 + rerank",
"graph": "Retained E5 + selected graph",
"paths": "Retained E5 + graph paths",
"both": "Retained E5 + graph + paths",
}
COLORS = {"hybrid": "#4f46e5", "graph": "#0f766e", "paths": "#d97706", "both": "#a16207"}
def load_jsonl(path):
with path.open(encoding="utf-8") as handle:
return [json.loads(line) for line in handle if line.strip()]
BENCHMARK = load_jsonl(ROOT / "finrag_equinor_v1.0.jsonl")
BENCHMARK_BY_ID = {row["question_id"]: row for row in BENCHMARK}
with (ROOT / "graph_replay.json").open(encoding="utf-8") as handle:
REPLAY = json.load(handle)
with (ROOT / "source_manifest.csv").open(encoding="utf-8-sig", newline="") as handle:
SOURCES = {row["document_id"]: row["download_url"] for row in csv.DictReader(handle)}
CORPUS = {}
for path in sorted((ROOT / "corpus").glob("eval_corpus_*.jsonl")):
for row in load_jsonl(path):
CORPUS[row["object_id"]] = row
QUESTION_LABELS = {
f'{row["report_year"]} | {row["question"]}': row["question_id"]
for row in BENCHMARK if row["question_id"] in REPLAY["results"]
}
QUESTIONS = list(QUESTION_LABELS)
TYPES = sorted({BENCHMARK_BY_ID[qid]["question_type"] for qid in QUESTION_LABELS.values()})
def pdf_url(document_id, page):
base = SOURCES.get(document_id, "")
return f"{base}#page={page}" if base else ""
def match_label(item, object_id, page):
gold_ids = set(item.get("gold_evidence_object_ids", []))
if not gold_ids:
gold_ids = {e.get("object_id") for e in item.get("evidence", [])}
gold_pages = {str(p) for p in item.get("gold_evidence_pages", [])}
if not gold_pages:
gold_pages = {str(e.get("page")) for e in item.get("evidence", [])}
if object_id in gold_ids:
return "Exact gold evidence", "gold"
if str(page) in gold_pages:
return "Gold page, different object", "page"
return "Not annotated gold", "miss"
def provenance(reasons):
labels = []
for reason in reasons or []:
if reason == "base" or reason.startswith("source:hybrid"):
labels.append("Hybrid")
elif reason.startswith("source:without_adjacent_page"):
labels.append("Selected graph")
elif reason.startswith("query_metric:"):
labels.append("Query metric: " + reason.split(":", 1)[1])
elif reason.startswith("query_entity:"):
labels.append("Query entity: " + reason.split(":", 1)[1])
elif reason.startswith("path:same_page:"):
labels.append("Same page")
elif reason.startswith("path:same_metric:"):
labels.append("Same metric: " + reason.rsplit(":", 1)[1])
elif reason.startswith("path:same_entity:"):
labels.append("Same entity: " + reason.rsplit(":", 1)[1])
return list(dict.fromkeys(labels)) or ["Saved ranking"]
def render_ranking(item, method):
data = REPLAY["results"][item["question_id"]][method]
rows = []
for candidate in data["top_k"]:
obj = CORPUS.get(candidate["object_id"], {})
page = candidate.get("page", obj.get("page"))
label, css = match_label(item, candidate["object_id"], page)
source = " · ".join(provenance(candidate.get("reasons")))
text = " ".join(obj.get("text", "").split())
link = pdf_url(item["document_id"], page)
rows.append(
f'<tr class="{css}"><td>{candidate["rank"]}</td><td>{escape(label)}</td>'
f'<td>{escape(str(page))}</td><td>{escape(str(candidate.get("type") or obj.get("object_type", "")))}</td>'
f'<td><span class="source-tag">{escape(source)}</span></td>'
f'<td>{escape(text[:360])}</td><td><a href="{escape(link, quote=True)}" target="_blank">PDF</a></td></tr>'
)
return (
'<div class="evidence-table"><table><thead><tr><th>Rank</th><th>Evidence match</th>'
'<th>Page</th><th>Type</th><th>Candidate source</th><th>Evidence preview</th><th>Source</th>'
'</tr></thead><tbody>' + "".join(rows) + '</tbody></table></div>'
)
def render_trace(item, method):
result = REPLAY["results"][item["question_id"]][method]
trace = result.get("trace") or {}
candidates = result["top_k"][:5]
relation_counts = {}
for candidate in candidates:
for label in provenance(candidate.get("reasons")):
relation_counts[label] = relation_counts.get(label, 0) + 1
chips = "".join(
f'<span class="trace-chip">{escape(k)} <b>{v}</b></span>' for k, v in relation_counts.items()
) or '<span class="trace-chip">Hybrid ranking only</span>'
cues = []
if trace.get("years"):
cues.append("Year: " + ", ".join(map(str, trace["years"])))
if trace.get("entities"):
cues.append("Entities: " + ", ".join(trace["entities"]))
if trace.get("metrics"):
cues.append("Metrics: " + ", ".join(trace["metrics"]))
cue_text = " · ".join(cues) or "No stored path cues for this method"
color = COLORS[method]
nodes = []
for index, candidate in enumerate(candidates):
obj = CORPUS.get(candidate["object_id"], {})
label, css = match_label(item, candidate["object_id"], candidate.get("page"))
nodes.append(
f'<div class="evidence-node {css}"><b>#{index + 1} · page {candidate.get("page")}</b>'
f'<span>{escape(obj.get("text", "")[:115])}</span><small>{escape(label)}</small></div>'
)
return (
f'<div class="trace-panel" style="--accent:{color}"><div class="trace-root">'
f'<b>{escape(METHODS[method])}</b><span>{escape(cue_text)}</span></div>'
f'<div class="trace-arrow">Candidate provenance ↓</div><div class="trace-chips">{chips}</div>'
f'<div class="trace-nodes">{"".join(nodes)}</div></div>'
)
def show(question_label, method_label):
if question_label not in QUESTION_LABELS:
return "Select a question.", "", ""
method = next((key for key, label in METHODS.items() if label == method_label), "both")
item = BENCHMARK_BY_ID[QUESTION_LABELS[question_label]]
metric = REPLAY["metrics"][method]
answer = item.get("answer") or "No answer is annotated."
summary = (
f'### {item["question"]}\n\n**Gold answer:** {answer}\n\n'
f'**Report:** {item["report_year"]} · **Type:** {item["question_type"].replace("_", " ")} · '
f'**Method:** {METHODS[method]}\n\n'
f'Paper-level metrics: Object R@10 **{metric["object_r10"]:.3f}** · '
f'Page R@10 **{metric["page_r10"]:.3f}** · MRR **{metric["mrr"]:.3f}**'
)
return summary, render_trace(item, method), render_ranking(item, method)
def compare(question_label):
if question_label not in QUESTION_LABELS:
return "Select a question."
item = BENCHMARK_BY_ID[QUESTION_LABELS[question_label]]
blocks = []
for method in METHODS:
top = REPLAY["results"][item["question_id"]][method]["top_k"]
exact = sum(match_label(item, c["object_id"], c.get("page"))[1] == "gold" for c in top)
page = sum(match_label(item, c["object_id"], c.get("page"))[1] in {"gold", "page"} for c in top)
best = next((c["rank"] for c in top if match_label(item, c["object_id"], c.get("page"))[1] == "gold"), "—")
blocks.append(
f'<div class="metric-block" style="border-top-color:{COLORS[method]}"><b>{METHODS[method]}</b>'
f'<span>Exact objects in top-10<strong>{exact}</strong></span>'
f'<span>Objects from gold pages<strong>{page}</strong></span>'
f'<span>First exact rank<strong>{best}</strong></span></div>'
)
return '<div class="comparison">' + "".join(blocks) + '</div>'
def filter_questions(year, question_type, keyword):
keyword = (keyword or "").strip().lower()
labels = []
for label, qid in QUESTION_LABELS.items():
item = BENCHMARK_BY_ID[qid]
if year != "All years" and str(item["report_year"]) != year:
continue
if question_type != "All types" and item["question_type"] != question_type:
continue
if keyword and keyword not in item["question"].lower():
continue
labels.append(label)
return gr.update(choices=labels, value=labels[0] if labels else None), f"**{len(labels)} questions**"
CSS = """
.gradio-container {max-width: 1240px !important;}
.subtitle {color:#52617a;margin-top:-8px;}
footer {display:none!important;}
.comparison {display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px;margin:10px 0 22px;}
.metric-block {border:1px solid #d8dee9;border-top:4px solid;padding:14px;background:#fff;border-radius:6px;}
.metric-block>b {display:block;min-height:42px;color:#18212f;}
.metric-block span {display:flex;justify-content:space-between;gap:10px;padding-top:8px;font-size:13px;color:#52617a;}
.metric-block strong {color:#18212f;font-size:16px;}
.trace-panel {border:1px solid #d8dee9;border-left:5px solid var(--accent);padding:16px;border-radius:6px;background:#fbfcfe;}
.trace-root {display:flex;justify-content:space-between;gap:16px;flex-wrap:wrap;}.trace-root span{color:#52617a;}
.trace-arrow {text-align:center;color:#687386;padding:10px;}.trace-chips{display:flex;gap:7px;flex-wrap:wrap;justify-content:center;}
.trace-chip,.source-tag {background:#eef2f7;border:1px solid #d5dce7;border-radius:4px;padding:4px 7px;font-size:12px;}
.trace-nodes {display:grid;grid-template-columns:repeat(5,minmax(0,1fr));gap:8px;margin-top:15px;}
.evidence-node {border:1px solid #cbd5e1;padding:9px;border-radius:5px;background:white;min-width:0;}
.evidence-node b,.evidence-node span,.evidence-node small{display:block}.evidence-node span{font-size:12px;margin:6px 0;overflow-wrap:anywhere}.evidence-node small{color:#64748b}
.evidence-node.gold{background:#dcfce7;border-color:#86d49d}.evidence-node.page{background:#fef9c3;border-color:#e6d86b}
.evidence-table {overflow-x:auto;}.evidence-table table{border-collapse:collapse;width:100%;font-size:13px;}
.evidence-table th,.evidence-table td{padding:9px;border:1px solid #d8dee9;text-align:left;vertical-align:top;overflow-wrap:anywhere;}
.evidence-table th{background:#f1f5f9}.evidence-table tr.gold td{background:#effcf3}.evidence-table tr.page td{background:#fffced}.evidence-table a{text-decoration:underline}
@media(max-width:800px){.comparison{grid-template-columns:1fr 1fr}.trace-nodes{grid-template-columns:1fr 1fr}}
"""
with gr.Blocks(css=CSS, title="FinRAG GraphRAG Demo") as demo:
gr.Markdown("# FinRAG GraphRAG Evidence Explorer\n<p class='subtitle'>Inspect how saved hybrid, graph-expansion, and graph-path rankings retrieve evidence from long annual reports.</p>")
gr.Markdown("This demo replays the paper's saved outputs. Green marks exact annotated evidence; yellow marks a different object on an annotated page.")
with gr.Row():
year = gr.Dropdown(["All years"] + [str(y) for y in range(2010, 2025)], value="All years", label="Year")
qtype = gr.Dropdown(["All types"] + TYPES, value="All types", label="Question type")
keyword = gr.Textbox(label="Question contains", placeholder="production, revenue, emissions...")
count = gr.Markdown(f"**{len(QUESTIONS)} questions**")
question = gr.Dropdown(QUESTIONS, value=QUESTIONS[0], label="Benchmark question")
for component in (year, qtype, keyword):
component.change(filter_questions, [year, qtype, keyword], [question, count])
gr.Markdown("## Per-question comparison")
compare_button = gr.Button("Compare all methods", variant="primary")
comparison = gr.HTML()
compare_button.click(compare, question, comparison)
gr.Markdown("## Inspect one retrieval route")
method = gr.Radio(list(METHODS.values()), value=METHODS["both"], label="Method")
inspect = gr.Button("Inspect saved Top-10", variant="primary")
summary = gr.Markdown()
trace = gr.HTML()
ranking = gr.HTML()
inspect.click(show, [question, method], [summary, trace, ranking])
gr.Markdown("*Research demo. Reference report and year come from the benchmark. Graph paths are fixed typed expansions, not an autonomous agent.*")
if __name__ == "__main__":
demo.queue(default_concurrency_limit=4).launch(ssr_mode=False)