"""Paper-aligned GraphRAG retrieval replay for FinRAG-Equinor.""" import csv import json from html import escape from pathlib import Path import gradio as gr ROOT = Path(__file__).parent METHODS = { "hybrid": "Hybrid E5 + rerank", "graph": "Retained E5 + selected graph", "paths": "Retained E5 + graph paths", "both": "Retained E5 + graph + paths", } COLORS = {"hybrid": "#4f46e5", "graph": "#0f766e", "paths": "#d97706", "both": "#a16207"} def load_jsonl(path): with path.open(encoding="utf-8") as handle: return [json.loads(line) for line in handle if line.strip()] BENCHMARK = load_jsonl(ROOT / "finrag_equinor_v1.0.jsonl") BENCHMARK_BY_ID = {row["question_id"]: row for row in BENCHMARK} with (ROOT / "graph_replay.json").open(encoding="utf-8") as handle: REPLAY = json.load(handle) with (ROOT / "source_manifest.csv").open(encoding="utf-8-sig", newline="") as handle: SOURCES = {row["document_id"]: row["download_url"] for row in csv.DictReader(handle)} CORPUS = {} for path in sorted((ROOT / "corpus").glob("eval_corpus_*.jsonl")): for row in load_jsonl(path): CORPUS[row["object_id"]] = row QUESTION_LABELS = { f'{row["report_year"]} | {row["question"]}': row["question_id"] for row in BENCHMARK if row["question_id"] in REPLAY["results"] } QUESTIONS = list(QUESTION_LABELS) TYPES = sorted({BENCHMARK_BY_ID[qid]["question_type"] for qid in QUESTION_LABELS.values()}) def pdf_url(document_id, page): base = SOURCES.get(document_id, "") return f"{base}#page={page}" if base else "" def match_label(item, object_id, page): gold_ids = set(item.get("gold_evidence_object_ids", [])) if not gold_ids: gold_ids = {e.get("object_id") for e in item.get("evidence", [])} gold_pages = {str(p) for p in item.get("gold_evidence_pages", [])} if not gold_pages: gold_pages = {str(e.get("page")) for e in item.get("evidence", [])} if object_id in gold_ids: return "Exact gold evidence", "gold" if str(page) in gold_pages: return "Gold page, different object", "page" return "Not annotated gold", "miss" def provenance(reasons): labels = [] for reason in reasons or []: if reason == "base" or reason.startswith("source:hybrid"): labels.append("Hybrid") elif reason.startswith("source:without_adjacent_page"): labels.append("Selected graph") elif reason.startswith("query_metric:"): labels.append("Query metric: " + reason.split(":", 1)[1]) elif reason.startswith("query_entity:"): labels.append("Query entity: " + reason.split(":", 1)[1]) elif reason.startswith("path:same_page:"): labels.append("Same page") elif reason.startswith("path:same_metric:"): labels.append("Same metric: " + reason.rsplit(":", 1)[1]) elif reason.startswith("path:same_entity:"): labels.append("Same entity: " + reason.rsplit(":", 1)[1]) return list(dict.fromkeys(labels)) or ["Saved ranking"] def render_ranking(item, method): data = REPLAY["results"][item["question_id"]][method] rows = [] for candidate in data["top_k"]: obj = CORPUS.get(candidate["object_id"], {}) page = candidate.get("page", obj.get("page")) label, css = match_label(item, candidate["object_id"], page) source = " · ".join(provenance(candidate.get("reasons"))) text = " ".join(obj.get("text", "").split()) link = pdf_url(item["document_id"], page) rows.append( f'{candidate["rank"]}{escape(label)}' f'{escape(str(page))}{escape(str(candidate.get("type") or obj.get("object_type", "")))}' f'{escape(source)}' f'{escape(text[:360])}PDF' ) return ( '
' '' '' + "".join(rows) + '
RankEvidence matchPageTypeCandidate sourceEvidence previewSource
' ) def render_trace(item, method): result = REPLAY["results"][item["question_id"]][method] trace = result.get("trace") or {} candidates = result["top_k"][:5] relation_counts = {} for candidate in candidates: for label in provenance(candidate.get("reasons")): relation_counts[label] = relation_counts.get(label, 0) + 1 chips = "".join( f'{escape(k)} {v}' for k, v in relation_counts.items() ) or 'Hybrid ranking only' cues = [] if trace.get("years"): cues.append("Year: " + ", ".join(map(str, trace["years"]))) if trace.get("entities"): cues.append("Entities: " + ", ".join(trace["entities"])) if trace.get("metrics"): cues.append("Metrics: " + ", ".join(trace["metrics"])) cue_text = " · ".join(cues) or "No stored path cues for this method" color = COLORS[method] nodes = [] for index, candidate in enumerate(candidates): obj = CORPUS.get(candidate["object_id"], {}) label, css = match_label(item, candidate["object_id"], candidate.get("page")) nodes.append( f'
#{index + 1} · page {candidate.get("page")}' f'{escape(obj.get("text", "")[:115])}{escape(label)}
' ) return ( f'
' f'{escape(METHODS[method])}{escape(cue_text)}
' f'
Candidate provenance ↓
{chips}
' f'
{"".join(nodes)}
' ) def show(question_label, method_label): if question_label not in QUESTION_LABELS: return "Select a question.", "", "" method = next((key for key, label in METHODS.items() if label == method_label), "both") item = BENCHMARK_BY_ID[QUESTION_LABELS[question_label]] metric = REPLAY["metrics"][method] answer = item.get("answer") or "No answer is annotated." summary = ( f'### {item["question"]}\n\n**Gold answer:** {answer}\n\n' f'**Report:** {item["report_year"]} · **Type:** {item["question_type"].replace("_", " ")} · ' f'**Method:** {METHODS[method]}\n\n' f'Paper-level metrics: Object R@10 **{metric["object_r10"]:.3f}** · ' f'Page R@10 **{metric["page_r10"]:.3f}** · MRR **{metric["mrr"]:.3f}**' ) return summary, render_trace(item, method), render_ranking(item, method) def compare(question_label): if question_label not in QUESTION_LABELS: return "Select a question." item = BENCHMARK_BY_ID[QUESTION_LABELS[question_label]] blocks = [] for method in METHODS: top = REPLAY["results"][item["question_id"]][method]["top_k"] exact = sum(match_label(item, c["object_id"], c.get("page"))[1] == "gold" for c in top) page = sum(match_label(item, c["object_id"], c.get("page"))[1] in {"gold", "page"} for c in top) best = next((c["rank"] for c in top if match_label(item, c["object_id"], c.get("page"))[1] == "gold"), "—") blocks.append( f'
{METHODS[method]}' f'Exact objects in top-10{exact}' f'Objects from gold pages{page}' f'First exact rank{best}
' ) return '
' + "".join(blocks) + '
' def filter_questions(year, question_type, keyword): keyword = (keyword or "").strip().lower() labels = [] for label, qid in QUESTION_LABELS.items(): item = BENCHMARK_BY_ID[qid] if year != "All years" and str(item["report_year"]) != year: continue if question_type != "All types" and item["question_type"] != question_type: continue if keyword and keyword not in item["question"].lower(): continue labels.append(label) return gr.update(choices=labels, value=labels[0] if labels else None), f"**{len(labels)} questions**" CSS = """ .gradio-container {max-width: 1240px !important;} .subtitle {color:#52617a;margin-top:-8px;} footer {display:none!important;} .comparison {display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:12px;margin:10px 0 22px;} .metric-block {border:1px solid #d8dee9;border-top:4px solid;padding:14px;background:#fff;border-radius:6px;} .metric-block>b {display:block;min-height:42px;color:#18212f;} .metric-block span {display:flex;justify-content:space-between;gap:10px;padding-top:8px;font-size:13px;color:#52617a;} .metric-block strong {color:#18212f;font-size:16px;} .trace-panel {border:1px solid #d8dee9;border-left:5px solid var(--accent);padding:16px;border-radius:6px;background:#fbfcfe;} .trace-root {display:flex;justify-content:space-between;gap:16px;flex-wrap:wrap;}.trace-root span{color:#52617a;} .trace-arrow {text-align:center;color:#687386;padding:10px;}.trace-chips{display:flex;gap:7px;flex-wrap:wrap;justify-content:center;} .trace-chip,.source-tag {background:#eef2f7;border:1px solid #d5dce7;border-radius:4px;padding:4px 7px;font-size:12px;} .trace-nodes {display:grid;grid-template-columns:repeat(5,minmax(0,1fr));gap:8px;margin-top:15px;} .evidence-node {border:1px solid #cbd5e1;padding:9px;border-radius:5px;background:white;min-width:0;} .evidence-node b,.evidence-node span,.evidence-node small{display:block}.evidence-node span{font-size:12px;margin:6px 0;overflow-wrap:anywhere}.evidence-node small{color:#64748b} .evidence-node.gold{background:#dcfce7;border-color:#86d49d}.evidence-node.page{background:#fef9c3;border-color:#e6d86b} .evidence-table {overflow-x:auto;}.evidence-table table{border-collapse:collapse;width:100%;font-size:13px;} .evidence-table th,.evidence-table td{padding:9px;border:1px solid #d8dee9;text-align:left;vertical-align:top;overflow-wrap:anywhere;} .evidence-table th{background:#f1f5f9}.evidence-table tr.gold td{background:#effcf3}.evidence-table tr.page td{background:#fffced}.evidence-table a{text-decoration:underline} @media(max-width:800px){.comparison{grid-template-columns:1fr 1fr}.trace-nodes{grid-template-columns:1fr 1fr}} """ with gr.Blocks(css=CSS, title="FinRAG GraphRAG Demo") as demo: gr.Markdown("# FinRAG GraphRAG Evidence Explorer\n

Inspect how saved hybrid, graph-expansion, and graph-path rankings retrieve evidence from long annual reports.

") gr.Markdown("This demo replays the paper's saved outputs. Green marks exact annotated evidence; yellow marks a different object on an annotated page.") with gr.Row(): year = gr.Dropdown(["All years"] + [str(y) for y in range(2010, 2025)], value="All years", label="Year") qtype = gr.Dropdown(["All types"] + TYPES, value="All types", label="Question type") keyword = gr.Textbox(label="Question contains", placeholder="production, revenue, emissions...") count = gr.Markdown(f"**{len(QUESTIONS)} questions**") question = gr.Dropdown(QUESTIONS, value=QUESTIONS[0], label="Benchmark question") for component in (year, qtype, keyword): component.change(filter_questions, [year, qtype, keyword], [question, count]) gr.Markdown("## Per-question comparison") compare_button = gr.Button("Compare all methods", variant="primary") comparison = gr.HTML() compare_button.click(compare, question, comparison) gr.Markdown("## Inspect one retrieval route") method = gr.Radio(list(METHODS.values()), value=METHODS["both"], label="Method") inspect = gr.Button("Inspect saved Top-10", variant="primary") summary = gr.Markdown() trace = gr.HTML() ranking = gr.HTML() inspect.click(show, [question, method], [summary, trace, ranking]) gr.Markdown("*Research demo. Reference report and year come from the benchmark. Graph paths are fixed typed expansions, not an autonomous agent.*") if __name__ == "__main__": demo.queue(default_concurrency_limit=4).launch(ssr_mode=False)