Spaces:
Running
Running
File size: 14,981 Bytes
cdabceb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 | #!/usr/bin/env python3
"""Serve a local live-query prototype for the FinRAG GraphRAG demo."""
from __future__ import annotations
import argparse
import json
import os
import re
import sys
import threading
import time
from collections import defaultdict
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import numpy as np
from rank_bm25 import BM25Okapi
from sentence_transformers import CrossEncoder, SentenceTransformer
DEMO_DIR = Path(__file__).resolve().parent
PROJECT_ROOT = DEMO_DIR.parents[1]
sys.path.insert(0, str(PROJECT_ROOT))
CORPUS_FILE = PROJECT_ROOT / "results" / "02_eval_corpus" / "eval_corpus.jsonl"
CACHE_DIR = PROJECT_ROOT / "results" / "05_dense" / "cache"
GRAPH_DIR = PROJECT_ROOT / "results" / "11_graph"
E5_MODEL = "intfloat/e5-large-v2"
RERANKER_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"
METRIC_TERMS = {
"capital_expenditure": ["capital expenditure", "capex", "investment", "investments"],
"cash_flow": ["cash flow", "cash flows", "free cash flow"],
"cost": ["cost", "costs", "expense", "expenses"],
"emissions": ["emission", "emissions", "co2", "carbon", "greenhouse"],
"net_debt": ["net debt", "debt ratio", "debt"],
"production": ["production", "produced", "barrels", "boe", "mboe"],
"profit": ["profit", "profits", "earnings", "ebit", "income"],
"renewables": ["renewable", "renewables", "wind", "solar", "ccs"],
"reserves": ["reserve", "reserves"],
"revenue": ["revenue", "revenues", "sales"],
"safety": ["safety", "injury", "injuries", "incident", "incidents"],
"tax": ["tax", "taxes", "tax rate"],
}
def load_json(path: Path):
return json.loads(path.read_text(encoding="utf-8"))
def load_jsonl(path: Path) -> list[dict]:
with path.open(encoding="utf-8") as handle:
return [json.loads(line) for line in handle if line.strip()]
def tokenize(text: str) -> list[str]:
return [token for token in re.findall(r"[a-z0-9]+", text.lower()) if len(token) > 2]
def query_terms(text: str) -> set[str]:
stop = {"the", "and", "for", "with", "from", "what", "which", "how", "why", "according", "report"}
return {token for token in tokenize(text) if token not in stop}
def lexical_overlap(terms: set[str], text: str) -> float:
if not terms:
return 0.0
return len(terms & set(tokenize(text))) / len(terms)
class LiveGraphRAG:
def __init__(self) -> None:
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
self.lock = threading.Lock()
self.corpus = load_jsonl(CORPUS_FILE)
self.by_id = {row["object_id"]: row for row in self.corpus}
self.ids = load_json(CACHE_DIR / "intfloat__e5-large-v2__corpus_ids.json")
if self.ids != [row["object_id"] for row in self.corpus]:
raise RuntimeError("E5 embedding cache does not match the evaluation corpus.")
self.embeddings = np.load(CACHE_DIR / "intfloat__e5-large-v2__corpus_embeddings.npy", mmap_mode="r")
self.e5 = SentenceTransformer(E5_MODEL)
self.reranker = CrossEncoder(RERANKER_MODEL)
self.page_to_objects = load_json(GRAPH_DIR / "page_to_objects.json")
self.entity_to_objects = load_json(GRAPH_DIR / "entity_to_objects.json")
self.metric_to_objects = load_json(GRAPH_DIR / "metric_to_objects.json")
self.object_to_entities = load_json(GRAPH_DIR / "object_to_entities.json")
self.object_to_metrics = load_json(GRAPH_DIR / "object_to_metrics.json")
from scripts.common.entity_extraction import DomainEntityExtractor
graph_summary = load_json(GRAPH_DIR / "graph_summary.json")
self.entity_extractor = DomainEntityExtractor(known_entities=graph_summary.get("known_entities", []))
self.year_indices: dict[int, np.ndarray] = {}
self.year_documents: dict[int, str] = {}
self.bm25: dict[int, BM25Okapi] = {}
grouped: dict[int, list[int]] = defaultdict(list)
for index, row in enumerate(self.corpus):
year = int(row["report_year"])
grouped[year].append(index)
self.year_documents[year] = row["document_id"]
for year, indices in grouped.items():
array = np.asarray(indices, dtype=np.int64)
self.year_indices[year] = array
self.bm25[year] = BM25Okapi([tokenize(self.corpus[i]["text"]) for i in array])
@property
def years(self) -> list[int]:
return sorted(self.year_indices)
def _rerank(self, question: str, candidates: list[dict], limit: int = 10) -> list[dict]:
if not candidates:
return []
pairs = [(question, item["text"]) for item in candidates]
with self.lock:
scores = self.reranker.predict(pairs, batch_size=32, show_progress_bar=False)
ranked = sorted(zip(candidates, scores), key=lambda pair: float(pair[1]), reverse=True)
output = []
for rank, (item, score) in enumerate(ranked[:limit], 1):
row = dict(item)
row["rank"] = rank
row["reranker_score"] = round(float(score), 4)
output.append(row)
return output
def _candidate(self, object_id: str, reasons: set[str]) -> dict:
obj = self.by_id[object_id]
return {"object_id": object_id, "document_id": obj.get("document_id"), "report_year": obj.get("report_year"), "page": obj.get("page"), "object_type": obj.get("object_type"), "page_type": obj.get("page_type"), "text": obj.get("text", "")[:1400], "reasons": sorted(reasons)}
def _hybrid(self, question: str, year: int) -> tuple[list[dict], list[str]]:
indices = self.year_indices[year]
tokens = tokenize(question)
bm25_scores = np.asarray(self.bm25[year].get_scores(tokens), dtype=float)
bm25_order = np.argsort(bm25_scores)[::-1][:100]
with self.lock:
query_embedding = self.e5.encode(["query: " + question], convert_to_numpy=True, normalize_embeddings=True, show_progress_bar=False)[0]
dense_scores = np.asarray(self.embeddings[indices] @ query_embedding, dtype=float)
dense_order = np.argsort(dense_scores)[::-1][:100]
rrf: dict[int, float] = defaultdict(float)
reasons: dict[int, set[str]] = defaultdict(set)
for rank, local in enumerate(bm25_order, 1):
idx = int(indices[int(local)])
rrf[idx] += 1.0 / (60 + rank)
reasons[idx].add("BM25")
for rank, local in enumerate(dense_order, 1):
idx = int(indices[int(local)])
rrf[idx] += 1.0 / (60 + rank)
reasons[idx].add("E5")
ordered = sorted(rrf, key=rrf.get, reverse=True)[:50]
candidates = [self._candidate(self.corpus[idx]["object_id"], reasons[idx]) for idx in ordered]
seed_ids = [self.corpus[idx]["object_id"] for idx in ordered[:10]]
return candidates, seed_ids
def _selected_graph(self, seed_ids: list[str], year: int, question: str) -> list[dict]:
target_doc = self.year_documents[year]
terms = query_terms(question)
scores: dict[str, float] = defaultdict(float)
reasons: dict[str, set[str]] = defaultdict(set)
def add(oid: str, score: float, reason: str) -> None:
obj = self.by_id.get(oid)
if not obj or obj.get("document_id") != target_doc:
return
scores[oid] += score + 0.15 * lexical_overlap(terms, obj.get("text", ""))
reasons[oid].add(reason)
for seed in seed_ids:
obj = self.by_id[seed]
add(seed, 2.0, "Hybrid seed")
page_key = f"{target_doc}:{int(obj['page'])}"
for oid in self.page_to_objects.get(page_key, []):
add(oid, 0.8, f"Same page as {seed}")
for entity in self.object_to_entities.get(seed, []):
for oid in self.entity_to_objects.get(entity, []):
add(oid, 0.55, f"Same entity: {entity}")
for metric in self.object_to_metrics.get(seed, []):
for oid in self.metric_to_objects.get(metric, []):
add(oid, 0.45, f"Same metric: {metric}")
ordered = sorted(scores, key=lambda oid: (scores[oid], lexical_overlap(terms, self.by_id[oid]["text"])), reverse=True)[:50]
return [self._candidate(oid, reasons[oid]) for oid in ordered]
def _path_candidates(self, question: str, year: int) -> tuple[list[dict], dict]:
target_doc = self.year_documents[year]
lowered = question.lower()
terms = query_terms(question)
metrics = [name for name, variants in METRIC_TERMS.items() if any(term in lowered for term in variants)]
with self.lock:
entities = self.entity_extractor.graph_entities(question, limit=10)
entity_keys = [entity.lower().replace(" ", "_") for entity in entities]
scores: dict[str, float] = defaultdict(float)
reasons: dict[str, set[str]] = defaultdict(set)
def add(oid: str, score: float, reason: str) -> None:
obj = self.by_id.get(oid)
if not obj or obj.get("document_id") != target_doc:
return
scores[oid] += score + lexical_overlap(terms, obj.get("text", ""))
reasons[oid].add(reason)
for metric in metrics:
ranked = sorted(self.metric_to_objects.get(metric, []), key=lambda oid: lexical_overlap(terms, self.by_id.get(oid, {}).get("text", "")), reverse=True)
for oid in ranked[:120]:
add(oid, 1.2, f"Query metric: {metric}")
for entity, entity_key in zip(entities, entity_keys):
ranked = sorted(self.entity_to_objects.get(entity_key, []), key=lambda oid: lexical_overlap(terms, self.by_id.get(oid, {}).get("text", "")), reverse=True)
for oid in ranked[:80]:
add(oid, 0.9, f"Query entity: {entity}")
if not scores:
for idx in self.year_indices[year]:
oid = self.corpus[int(idx)]["object_id"]
overlap = lexical_overlap(terms, self.by_id[oid].get("text", ""))
if overlap:
add(oid, overlap, "Document lexical path seed")
seeds = sorted(scores, key=scores.get, reverse=True)[:20]
for seed in seeds:
obj = self.by_id[seed]
page_key = f"{target_doc}:{int(obj['page'])}"
for oid in self.page_to_objects.get(page_key, []):
add(oid, 0.35, f"Path via page {obj['page']}")
ordered = sorted(scores, key=lambda oid: (scores[oid], lexical_overlap(terms, self.by_id[oid]["text"])), reverse=True)[:80]
return [self._candidate(oid, reasons[oid]) for oid in ordered], {"year": year, "entities": entities, "metrics": metrics, "path_seed_count": len(seeds)}
def query(self, question: str, year: int, method: str) -> dict:
started = time.perf_counter()
hybrid_pool, seed_ids = self._hybrid(question, year)
hybrid_ranked = self._rerank(question, hybrid_pool)
after_hybrid = time.perf_counter()
graph_ranked: list[dict] = []
path_ranked: list[dict] = []
trace = {"year": year, "entities": [], "metrics": [], "path_seed_count": 0}
if method in {"graph", "both"}:
graph_ranked = self._rerank(question, self._selected_graph(seed_ids, year, question))
if method in {"paths", "both"}:
path_pool, trace = self._path_candidates(question, year)
path_ranked = self._rerank(question, path_pool)
after_graph = time.perf_counter()
sources = [(hybrid_ranked, "Retained hybrid")]
if graph_ranked:
sources.append((graph_ranked, "Selected graph"))
if path_ranked:
sources.append((path_ranked, "Graph path"))
merged: dict[str, dict] = {}
for rows, source in sources:
for row in rows:
oid = row["object_id"]
if oid not in merged:
merged[oid] = dict(row)
merged[oid]["reasons"] = []
merged[oid]["reasons"] = sorted(set(merged[oid]["reasons"] + row.get("reasons", []) + [source]))
final = self._rerank(question, list(merged.values())[:80])
finished = time.perf_counter()
return {"question": question, "year": year, "method": method, "trace": trace, "seed_object_ids": seed_ids, "timings": {"hybrid_seconds": round(after_hybrid - started, 3), "graph_seconds": round(after_graph - after_hybrid, 3), "final_rerank_seconds": round(finished - after_graph, 3), "total_seconds": round(finished - started, 3)}, "results": final, "note": "Live prototype result; paper metrics are reproduced separately from saved benchmark rankings."}
class Handler(SimpleHTTPRequestHandler):
engine: LiveGraphRAG
def __init__(self, *args, **kwargs):
super().__init__(*args, directory=str(DEMO_DIR), **kwargs)
def do_GET(self):
if self.path == "/api/health":
self._json({"status": "ready", "years": self.engine.years})
return
if self.path in {"/live", "/live/"}:
self.path = "/live.html"
super().do_GET()
def do_POST(self):
if self.path != "/api/live-query":
self.send_error(404)
return
try:
length = int(self.headers.get("Content-Length", "0"))
payload = json.loads(self.rfile.read(length).decode("utf-8"))
question = str(payload.get("question", "")).strip()
year = int(payload.get("year"))
method = str(payload.get("method", "both"))
if len(question) < 5:
raise ValueError("Enter a complete question.")
if year not in self.engine.years:
raise ValueError("Choose a valid report year.")
if method not in {"hybrid", "graph", "paths", "both"}:
raise ValueError("Choose a valid retrieval method.")
self._json(self.engine.query(question, year, method))
except Exception as exc:
self._json({"error": str(exc)}, status=400)
def _json(self, payload: dict, status: int = 200):
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=8766)
args = parser.parse_args()
print("Loading live retrieval models and indexes...")
engine = LiveGraphRAG()
Handler.engine = engine
server = ThreadingHTTPServer((args.host, args.port), Handler)
print(f"Live GraphRAG demo: http://{args.host}:{args.port}/live")
server.serve_forever()
if __name__ == "__main__":
main()
|