Text Generation
Transformers
Safetensors
English
llama
wikipedia
facts
retrieval
question-answering
smollm2
text-generation-inference
wiki-model / wiki_model /index.py
Lucasgitguy-co's picture
wiki-model 0.1.0
4f2ae40 verified
Raw History Blame Contribute Delete
4.18 kB
"""Sourced fact index: bm25s over article lead text + infobox facts, article text in zlib'd SQLite."""
import re, sqlite3, zlib
from pathlib import Path
def doc_text(r, max_chars=480):
"""Index/ctx text for an article: lead sentences + infobox facts. Training <ctx> uses exactly this.
The lead is capped so infobox facts (capital, born, ...) always fit."""
t = " ".join(r["sents"])
if r.get("fields"):
if len(t) > 260:
t = t[:260].rsplit(" ", 1)[0] + " …"
t += " Facts: " + "; ".join(f"{k}: {v}" for k, v in r["fields"].items())
return f"{r['title']}: {t}"[:max_chars]
_tok = re.compile(r"\w+")
def words(s):
return _tok.findall(s.lower())
def token_f1(a, b):
a, b = words(a), words(b)
if not a or not b: return 0.0
from collections import Counter
common = sum((Counter(a) & Counter(b)).values())
if not common: return 0.0
p, r = common / len(a), common / len(b)
return 2 * p * r / (p + r)
class FactIndex:
def __init__(self, path):
import bm25s
path = Path(path)
self.bm = bm25s.BM25.load(str(path / "bm25"), mmap=True)
self.db = sqlite3.connect(str(path / "docs.sqlite"), check_same_thread=False)
self.n = self.db.execute("select count(*) from docs").fetchone()[0]
def doc(self, i):
title, url, z = self.db.execute("select title, url, z from docs where id=?", (int(i),)).fetchone()
return {"title": title, "url": url, "text": zlib.decompress(z).decode()}
def search(self, query, k=3, pool=30):
"""BM25, then rerank: an article whose title appears in the query (or equals it) wins."""
import bm25s
toks = bm25s.tokenize([query], stopwords="en", show_progress=False)
if not toks.vocab: return []
ids, scores = self.bm.retrieve(toks, k=min(pool, self.n), show_progress=False)
q = " " + " ".join(words(query)) + " "
cand = {int(i): float(s) for i, s in zip(ids[0], scores[0]) if s > 0}
for i in self._title_ids(query):
cand.setdefault(i, 0.0)
hits = []
for i, s in cand.items():
d = self.doc(i)
tw = " ".join(words(re.sub(r"\s*\(.*?\)", "", d["title"])))
if tw and f" {tw} " in q:
s += 12 + 2 * len(tw.split()) + (10 if q.strip() == tw else 0)
hits.append(dict(d, score=float(s)))
return sorted(hits, key=lambda h: -h["score"])[:k]
def _title_ids(self, query):
"""Exact (case-insensitive) title matches for the query and its 1-5 word spans that start with a capital."""
toks = re.findall(r"[\w'’.-]+|\([^)]*\)", query.strip(" ?!."))
spans = {query.strip(" ?!.")}
for n in range(1, 6):
for j in range(len(toks) - n + 1):
if toks[j][:1].isupper() or toks[j][:1].isdigit():
spans.add(" ".join(toks[j:j + n]))
out = []
for sp in spans:
row = self.db.execute("select id from docs where title=? collate nocase", (sp,)).fetchone()
if row: out.append(row[0])
return out
def by_title(self, title):
row = self.db.execute("select id from docs where title=? collate nocase", (title,)).fetchone()
return self.doc(row[0]) if row else None
def random(self, rng):
return self.doc(rng.randrange(self.n))
def support(self, sentence, k=3, topic=None):
"""Best token-F1 of a generated sentence against the sentences of its top BM25 hits.
With a topic, only that topic's own article counts (otherwise template sentences about
other articles, e.g. '... is a municipality in ...', look like support)."""
best, src = 0.0, None
if topic:
d = self.by_title(topic)
docs = [d] if d else [h for h in self.search(topic, k) if words(topic) and set(words(topic)) <= set(words(h["title"]))]
else:
docs = self.search(sentence, k)
for d in docs:
for s in re.split(r"(?<=[.!?;])\s+", d["text"]):
f = token_f1(sentence, s)
if f > best: best, src = f, d
return best, src