File size: 14,981 Bytes
cdabceb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
#!/usr/bin/env python3
"""Serve a local live-query prototype for the FinRAG GraphRAG demo."""

from __future__ import annotations

import argparse
import json
import os
import re
import sys
import threading
import time
from collections import defaultdict
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path

import numpy as np
from rank_bm25 import BM25Okapi
from sentence_transformers import CrossEncoder, SentenceTransformer

DEMO_DIR = Path(__file__).resolve().parent
PROJECT_ROOT = DEMO_DIR.parents[1]
sys.path.insert(0, str(PROJECT_ROOT))
CORPUS_FILE = PROJECT_ROOT / "results" / "02_eval_corpus" / "eval_corpus.jsonl"
CACHE_DIR = PROJECT_ROOT / "results" / "05_dense" / "cache"
GRAPH_DIR = PROJECT_ROOT / "results" / "11_graph"
E5_MODEL = "intfloat/e5-large-v2"
RERANKER_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"

METRIC_TERMS = {
    "capital_expenditure": ["capital expenditure", "capex", "investment", "investments"],
    "cash_flow": ["cash flow", "cash flows", "free cash flow"],
    "cost": ["cost", "costs", "expense", "expenses"],
    "emissions": ["emission", "emissions", "co2", "carbon", "greenhouse"],
    "net_debt": ["net debt", "debt ratio", "debt"],
    "production": ["production", "produced", "barrels", "boe", "mboe"],
    "profit": ["profit", "profits", "earnings", "ebit", "income"],
    "renewables": ["renewable", "renewables", "wind", "solar", "ccs"],
    "reserves": ["reserve", "reserves"],
    "revenue": ["revenue", "revenues", "sales"],
    "safety": ["safety", "injury", "injuries", "incident", "incidents"],
    "tax": ["tax", "taxes", "tax rate"],
}

def load_json(path: Path):
    return json.loads(path.read_text(encoding="utf-8"))

def load_jsonl(path: Path) -> list[dict]:
    with path.open(encoding="utf-8") as handle:
        return [json.loads(line) for line in handle if line.strip()]

def tokenize(text: str) -> list[str]:
    return [token for token in re.findall(r"[a-z0-9]+", text.lower()) if len(token) > 2]

def query_terms(text: str) -> set[str]:
    stop = {"the", "and", "for", "with", "from", "what", "which", "how", "why", "according", "report"}
    return {token for token in tokenize(text) if token not in stop}

def lexical_overlap(terms: set[str], text: str) -> float:
    if not terms:
        return 0.0
    return len(terms & set(tokenize(text))) / len(terms)

class LiveGraphRAG:
    def __init__(self) -> None:
        os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
        self.lock = threading.Lock()
        self.corpus = load_jsonl(CORPUS_FILE)
        self.by_id = {row["object_id"]: row for row in self.corpus}
        self.ids = load_json(CACHE_DIR / "intfloat__e5-large-v2__corpus_ids.json")
        if self.ids != [row["object_id"] for row in self.corpus]:
            raise RuntimeError("E5 embedding cache does not match the evaluation corpus.")
        self.embeddings = np.load(CACHE_DIR / "intfloat__e5-large-v2__corpus_embeddings.npy", mmap_mode="r")
        self.e5 = SentenceTransformer(E5_MODEL)
        self.reranker = CrossEncoder(RERANKER_MODEL)
        self.page_to_objects = load_json(GRAPH_DIR / "page_to_objects.json")
        self.entity_to_objects = load_json(GRAPH_DIR / "entity_to_objects.json")
        self.metric_to_objects = load_json(GRAPH_DIR / "metric_to_objects.json")
        self.object_to_entities = load_json(GRAPH_DIR / "object_to_entities.json")
        self.object_to_metrics = load_json(GRAPH_DIR / "object_to_metrics.json")
        from scripts.common.entity_extraction import DomainEntityExtractor
        graph_summary = load_json(GRAPH_DIR / "graph_summary.json")
        self.entity_extractor = DomainEntityExtractor(known_entities=graph_summary.get("known_entities", []))
        self.year_indices: dict[int, np.ndarray] = {}
        self.year_documents: dict[int, str] = {}
        self.bm25: dict[int, BM25Okapi] = {}
        grouped: dict[int, list[int]] = defaultdict(list)
        for index, row in enumerate(self.corpus):
            year = int(row["report_year"])
            grouped[year].append(index)
            self.year_documents[year] = row["document_id"]
        for year, indices in grouped.items():
            array = np.asarray(indices, dtype=np.int64)
            self.year_indices[year] = array
            self.bm25[year] = BM25Okapi([tokenize(self.corpus[i]["text"]) for i in array])

    @property
    def years(self) -> list[int]:
        return sorted(self.year_indices)

    def _rerank(self, question: str, candidates: list[dict], limit: int = 10) -> list[dict]:
        if not candidates:
            return []
        pairs = [(question, item["text"]) for item in candidates]
        with self.lock:
            scores = self.reranker.predict(pairs, batch_size=32, show_progress_bar=False)
        ranked = sorted(zip(candidates, scores), key=lambda pair: float(pair[1]), reverse=True)
        output = []
        for rank, (item, score) in enumerate(ranked[:limit], 1):
            row = dict(item)
            row["rank"] = rank
            row["reranker_score"] = round(float(score), 4)
            output.append(row)
        return output

    def _candidate(self, object_id: str, reasons: set[str]) -> dict:
        obj = self.by_id[object_id]
        return {"object_id": object_id, "document_id": obj.get("document_id"), "report_year": obj.get("report_year"), "page": obj.get("page"), "object_type": obj.get("object_type"), "page_type": obj.get("page_type"), "text": obj.get("text", "")[:1400], "reasons": sorted(reasons)}

    def _hybrid(self, question: str, year: int) -> tuple[list[dict], list[str]]:
        indices = self.year_indices[year]
        tokens = tokenize(question)
        bm25_scores = np.asarray(self.bm25[year].get_scores(tokens), dtype=float)
        bm25_order = np.argsort(bm25_scores)[::-1][:100]
        with self.lock:
            query_embedding = self.e5.encode(["query: " + question], convert_to_numpy=True, normalize_embeddings=True, show_progress_bar=False)[0]
        dense_scores = np.asarray(self.embeddings[indices] @ query_embedding, dtype=float)
        dense_order = np.argsort(dense_scores)[::-1][:100]
        rrf: dict[int, float] = defaultdict(float)
        reasons: dict[int, set[str]] = defaultdict(set)
        for rank, local in enumerate(bm25_order, 1):
            idx = int(indices[int(local)])
            rrf[idx] += 1.0 / (60 + rank)
            reasons[idx].add("BM25")
        for rank, local in enumerate(dense_order, 1):
            idx = int(indices[int(local)])
            rrf[idx] += 1.0 / (60 + rank)
            reasons[idx].add("E5")
        ordered = sorted(rrf, key=rrf.get, reverse=True)[:50]
        candidates = [self._candidate(self.corpus[idx]["object_id"], reasons[idx]) for idx in ordered]
        seed_ids = [self.corpus[idx]["object_id"] for idx in ordered[:10]]
        return candidates, seed_ids

    def _selected_graph(self, seed_ids: list[str], year: int, question: str) -> list[dict]:
        target_doc = self.year_documents[year]
        terms = query_terms(question)
        scores: dict[str, float] = defaultdict(float)
        reasons: dict[str, set[str]] = defaultdict(set)
        def add(oid: str, score: float, reason: str) -> None:
            obj = self.by_id.get(oid)
            if not obj or obj.get("document_id") != target_doc:
                return
            scores[oid] += score + 0.15 * lexical_overlap(terms, obj.get("text", ""))
            reasons[oid].add(reason)
        for seed in seed_ids:
            obj = self.by_id[seed]
            add(seed, 2.0, "Hybrid seed")
            page_key = f"{target_doc}:{int(obj['page'])}"
            for oid in self.page_to_objects.get(page_key, []):
                add(oid, 0.8, f"Same page as {seed}")
            for entity in self.object_to_entities.get(seed, []):
                for oid in self.entity_to_objects.get(entity, []):
                    add(oid, 0.55, f"Same entity: {entity}")
            for metric in self.object_to_metrics.get(seed, []):
                for oid in self.metric_to_objects.get(metric, []):
                    add(oid, 0.45, f"Same metric: {metric}")
        ordered = sorted(scores, key=lambda oid: (scores[oid], lexical_overlap(terms, self.by_id[oid]["text"])), reverse=True)[:50]
        return [self._candidate(oid, reasons[oid]) for oid in ordered]

    def _path_candidates(self, question: str, year: int) -> tuple[list[dict], dict]:
        target_doc = self.year_documents[year]
        lowered = question.lower()
        terms = query_terms(question)
        metrics = [name for name, variants in METRIC_TERMS.items() if any(term in lowered for term in variants)]
        with self.lock:
            entities = self.entity_extractor.graph_entities(question, limit=10)
        entity_keys = [entity.lower().replace(" ", "_") for entity in entities]
        scores: dict[str, float] = defaultdict(float)
        reasons: dict[str, set[str]] = defaultdict(set)
        def add(oid: str, score: float, reason: str) -> None:
            obj = self.by_id.get(oid)
            if not obj or obj.get("document_id") != target_doc:
                return
            scores[oid] += score + lexical_overlap(terms, obj.get("text", ""))
            reasons[oid].add(reason)
        for metric in metrics:
            ranked = sorted(self.metric_to_objects.get(metric, []), key=lambda oid: lexical_overlap(terms, self.by_id.get(oid, {}).get("text", "")), reverse=True)
            for oid in ranked[:120]:
                add(oid, 1.2, f"Query metric: {metric}")
        for entity, entity_key in zip(entities, entity_keys):
            ranked = sorted(self.entity_to_objects.get(entity_key, []), key=lambda oid: lexical_overlap(terms, self.by_id.get(oid, {}).get("text", "")), reverse=True)
            for oid in ranked[:80]:
                add(oid, 0.9, f"Query entity: {entity}")
        if not scores:
            for idx in self.year_indices[year]:
                oid = self.corpus[int(idx)]["object_id"]
                overlap = lexical_overlap(terms, self.by_id[oid].get("text", ""))
                if overlap:
                    add(oid, overlap, "Document lexical path seed")
        seeds = sorted(scores, key=scores.get, reverse=True)[:20]
        for seed in seeds:
            obj = self.by_id[seed]
            page_key = f"{target_doc}:{int(obj['page'])}"
            for oid in self.page_to_objects.get(page_key, []):
                add(oid, 0.35, f"Path via page {obj['page']}")
        ordered = sorted(scores, key=lambda oid: (scores[oid], lexical_overlap(terms, self.by_id[oid]["text"])), reverse=True)[:80]
        return [self._candidate(oid, reasons[oid]) for oid in ordered], {"year": year, "entities": entities, "metrics": metrics, "path_seed_count": len(seeds)}

    def query(self, question: str, year: int, method: str) -> dict:
        started = time.perf_counter()
        hybrid_pool, seed_ids = self._hybrid(question, year)
        hybrid_ranked = self._rerank(question, hybrid_pool)
        after_hybrid = time.perf_counter()
        graph_ranked: list[dict] = []
        path_ranked: list[dict] = []
        trace = {"year": year, "entities": [], "metrics": [], "path_seed_count": 0}
        if method in {"graph", "both"}:
            graph_ranked = self._rerank(question, self._selected_graph(seed_ids, year, question))
        if method in {"paths", "both"}:
            path_pool, trace = self._path_candidates(question, year)
            path_ranked = self._rerank(question, path_pool)
        after_graph = time.perf_counter()
        sources = [(hybrid_ranked, "Retained hybrid")]
        if graph_ranked:
            sources.append((graph_ranked, "Selected graph"))
        if path_ranked:
            sources.append((path_ranked, "Graph path"))
        merged: dict[str, dict] = {}
        for rows, source in sources:
            for row in rows:
                oid = row["object_id"]
                if oid not in merged:
                    merged[oid] = dict(row)
                    merged[oid]["reasons"] = []
                merged[oid]["reasons"] = sorted(set(merged[oid]["reasons"] + row.get("reasons", []) + [source]))
        final = self._rerank(question, list(merged.values())[:80])
        finished = time.perf_counter()
        return {"question": question, "year": year, "method": method, "trace": trace, "seed_object_ids": seed_ids, "timings": {"hybrid_seconds": round(after_hybrid - started, 3), "graph_seconds": round(after_graph - after_hybrid, 3), "final_rerank_seconds": round(finished - after_graph, 3), "total_seconds": round(finished - started, 3)}, "results": final, "note": "Live prototype result; paper metrics are reproduced separately from saved benchmark rankings."}

class Handler(SimpleHTTPRequestHandler):
    engine: LiveGraphRAG
    def __init__(self, *args, **kwargs):
        super().__init__(*args, directory=str(DEMO_DIR), **kwargs)
    def do_GET(self):
        if self.path == "/api/health":
            self._json({"status": "ready", "years": self.engine.years})
            return
        if self.path in {"/live", "/live/"}:
            self.path = "/live.html"
        super().do_GET()
    def do_POST(self):
        if self.path != "/api/live-query":
            self.send_error(404)
            return
        try:
            length = int(self.headers.get("Content-Length", "0"))
            payload = json.loads(self.rfile.read(length).decode("utf-8"))
            question = str(payload.get("question", "")).strip()
            year = int(payload.get("year"))
            method = str(payload.get("method", "both"))
            if len(question) < 5:
                raise ValueError("Enter a complete question.")
            if year not in self.engine.years:
                raise ValueError("Choose a valid report year.")
            if method not in {"hybrid", "graph", "paths", "both"}:
                raise ValueError("Choose a valid retrieval method.")
            self._json(self.engine.query(question, year, method))
        except Exception as exc:
            self._json({"error": str(exc)}, status=400)
    def _json(self, payload: dict, status: int = 200):
        data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
        self.send_response(status)
        self.send_header("Content-Type", "application/json; charset=utf-8")
        self.send_header("Content-Length", str(len(data)))
        self.end_headers()
        self.wfile.write(data)

def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--host", default="127.0.0.1")
    parser.add_argument("--port", type=int, default=8766)
    args = parser.parse_args()
    print("Loading live retrieval models and indexes...")
    engine = LiveGraphRAG()
    Handler.engine = engine
    server = ThreadingHTTPServer((args.host, args.port), Handler)
    print(f"Live GraphRAG demo: http://{args.host}:{args.port}/live")
    server.serve_forever()

if __name__ == "__main__":
    main()