"""Sourced fact index: bm25s over article lead text + infobox facts, article text in zlib'd SQLite.""" import re, sqlite3, zlib from pathlib import Path def doc_text(r, max_chars=480): """Index/ctx text for an article: lead sentences + infobox facts. Training uses exactly this. The lead is capped so infobox facts (capital, born, ...) always fit.""" t = " ".join(r["sents"]) if r.get("fields"): if len(t) > 260: t = t[:260].rsplit(" ", 1)[0] + " …" t += " Facts: " + "; ".join(f"{k}: {v}" for k, v in r["fields"].items()) return f"{r['title']}: {t}"[:max_chars] _tok = re.compile(r"\w+") def words(s): return _tok.findall(s.lower()) def token_f1(a, b): a, b = words(a), words(b) if not a or not b: return 0.0 from collections import Counter common = sum((Counter(a) & Counter(b)).values()) if not common: return 0.0 p, r = common / len(a), common / len(b) return 2 * p * r / (p + r) class FactIndex: def __init__(self, path): import bm25s path = Path(path) self.bm = bm25s.BM25.load(str(path / "bm25"), mmap=True) self.db = sqlite3.connect(str(path / "docs.sqlite"), check_same_thread=False) self.n = self.db.execute("select count(*) from docs").fetchone()[0] def doc(self, i): title, url, z = self.db.execute("select title, url, z from docs where id=?", (int(i),)).fetchone() return {"title": title, "url": url, "text": zlib.decompress(z).decode()} def search(self, query, k=3, pool=30): """BM25, then rerank: an article whose title appears in the query (or equals it) wins.""" import bm25s toks = bm25s.tokenize([query], stopwords="en", show_progress=False) if not toks.vocab: return [] ids, scores = self.bm.retrieve(toks, k=min(pool, self.n), show_progress=False) q = " " + " ".join(words(query)) + " " cand = {int(i): float(s) for i, s in zip(ids[0], scores[0]) if s > 0} for i in self._title_ids(query): cand.setdefault(i, 0.0) hits = [] for i, s in cand.items(): d = self.doc(i) tw = " ".join(words(re.sub(r"\s*\(.*?\)", "", d["title"]))) if tw and f" {tw} " in q: s += 12 + 2 * len(tw.split()) + (10 if q.strip() == tw else 0) hits.append(dict(d, score=float(s))) return sorted(hits, key=lambda h: -h["score"])[:k] def _title_ids(self, query): """Exact (case-insensitive) title matches for the query and its 1-5 word spans that start with a capital.""" toks = re.findall(r"[\w'’.-]+|\([^)]*\)", query.strip(" ?!.")) spans = {query.strip(" ?!.")} for n in range(1, 6): for j in range(len(toks) - n + 1): if toks[j][:1].isupper() or toks[j][:1].isdigit(): spans.add(" ".join(toks[j:j + n])) out = [] for sp in spans: row = self.db.execute("select id from docs where title=? collate nocase", (sp,)).fetchone() if row: out.append(row[0]) return out def by_title(self, title): row = self.db.execute("select id from docs where title=? collate nocase", (title,)).fetchone() return self.doc(row[0]) if row else None def random(self, rng): return self.doc(rng.randrange(self.n)) def support(self, sentence, k=3, topic=None): """Best token-F1 of a generated sentence against the sentences of its top BM25 hits. With a topic, only that topic's own article counts (otherwise template sentences about other articles, e.g. '... is a municipality in ...', look like support).""" best, src = 0.0, None if topic: d = self.by_title(topic) docs = [d] if d else [h for h in self.search(topic, k) if words(topic) and set(words(topic)) <= set(words(h["title"]))] else: docs = self.search(sentence, k) for d in docs: for s in re.split(r"(?<=[.!?;])\s+", d["text"]): f = token_f1(sentence, s) if f > best: best, src = f, d return best, src