docSpaceSearch / app /index.py
Catal's picture
Rename index.py to app/index.py
b02642c verified
Raw History Blame Contribute Delete
8.13 kB
"""In-memory hybrid index over the article chunks exported by the web app.
* BM25 (inverted index) catches exact words: "art. 5", numbers, names, legal terms.
* Embeddings (multilingual-e5-small by default) catch paraphrases: "bani pentru cazare" ↔
"subvenție pentru cămin".
* The two rankings are merged with reciprocal rank fusion (RRF).
Documents carry a content hash: applying a new export re-embeds only documents that changed.
Everything lives in memory and is rebuilt from the app's export on start-up (the free Space disk is
not persistent) — a few thousand documents take seconds to minutes on CPU.
"""
from __future__ import annotations
import math
import threading
from collections import Counter, defaultdict
from dataclasses import dataclass
from typing import Callable, Iterable, Sequence
import numpy as np
from .text import terms
# Texts → L2-normalised vectors (n × dim). `kind` is "query" or "passage" (e5 needs the prefix).
Embedder = Callable[[Sequence[str], str], np.ndarray]
RRF_K = 60
CANDIDATES = 50
K1, B = 1.2, 0.75
@dataclass(frozen=True)
class Chunk:
id: str
document_id: str
anchor: str | None
cite: str | None
heading: str | None
text: str
def passage_text(self, title: str) -> str:
"""What gets embedded and BM25-indexed: the title and heading give the article its subject."""
return "\n".join(part for part in (title, self.heading, self.text) if part)
@dataclass
class Document:
id: str
hash: str
title: str
url: str
type_name: str
issued_date: str | None
repealed_at: str | None
chunks: list[Chunk]
vectors: np.ndarray # one row per chunk
@staticmethod
def parse(raw: dict) -> "Document":
chunks = [
Chunk(
id=c["id"],
document_id=raw["id"],
anchor=c.get("anchor"),
cite=c.get("cite"),
heading=c.get("heading"),
text=c.get("text", ""),
)
for c in raw.get("chunks", [])
if c.get("text", "").strip()
]
return Document(
id=raw["id"],
hash=raw.get("hash", ""),
title=raw.get("title", ""),
url=raw.get("url", ""),
type_name=(raw.get("type") or {}).get("name", ""),
issued_date=raw.get("issuedDate"),
repealed_at=raw.get("repealedAt"),
chunks=chunks,
vectors=np.zeros((0, 0), dtype=np.float32),
)
class SearchIndex:
def __init__(self, embed: Embedder):
self._embed = embed
self._write_lock = threading.Lock() # one update at a time (refresh vs. push from the app)
self._documents: dict[str, Document] = {}
# Searches read an immutable snapshot, replaced in one assignment after every update.
self._view = _View.empty()
# --- updates
def apply(self, upsert: Iterable[dict] = (), delete: Iterable[str] = ()) -> dict:
"""Add or replace documents (only re-embedding changed ones) and remove others."""
incoming = [Document.parse(raw) for raw in upsert]
with self._write_lock:
changed = [d for d in incoming if not d.hash or getattr(self._documents.get(d.id), "hash", None) != d.hash]
# One batch for all changed documents, then split back per document.
texts = [chunk.passage_text(doc.title) for doc in changed for chunk in doc.chunks]
vectors = self._embed(texts, "passage") if texts else np.zeros((0, 0), dtype=np.float32)
offset = 0
for doc in changed:
doc.vectors = vectors[offset : offset + len(doc.chunks)]
offset += len(doc.chunks)
self._documents[doc.id] = doc
removed = [doc_id for doc_id in delete if self._documents.pop(doc_id, None) is not None]
self._view = _View.build(self._documents.values())
return {"updated": len(changed), "unchanged": len(incoming) - len(changed), "deleted": len(removed)}
def replace_all(self, documents: list[dict]) -> dict:
"""Apply a full export: documents missing from it are no longer public."""
keep = {raw["id"] for raw in documents}
stale = [doc_id for doc_id in list(self._documents) if doc_id not in keep]
return self.apply(documents, stale)
# --- queries
@property
def stats(self) -> dict:
view = self._view
return {"documents": len(self._documents), "chunks": len(view.chunks)}
def search(self, query: str, k: int = 6) -> list[dict]:
view = self._view
if not view.chunks or not query.strip():
return []
ranked_lists = [view.bm25(terms(query))]
if view.vectors.size:
q = self._embed([query], "query")[0]
similarity = view.vectors @ q
top = np.argsort(-similarity)[:CANDIDATES]
ranked_lists.append([int(i) for i in top])
fused: dict[int, float] = defaultdict(float)
for ranked in ranked_lists:
for rank, index in enumerate(ranked):
fused[index] += 1.0 / (RRF_K + rank + 1)
best = sorted(fused.items(), key=lambda item: -item[1])[:k]
return [view.passage(index, score) for index, score in best]
class _View:
"""Immutable snapshot of all chunks: BM25 postings and the embedding matrix."""
def __init__(self, chunks, documents, vectors, postings, lengths):
self.chunks: list[Chunk] = chunks
self.documents: list[Document] = documents # document of each chunk
self.vectors: np.ndarray = vectors
self.postings: dict[str, list[tuple[int, int]]] = postings
self.lengths: list[int] = lengths
self.avg_length = (sum(lengths) / len(lengths)) if lengths else 1.0
@staticmethod
def empty() -> "_View":
return _View([], [], np.zeros((0, 0), dtype=np.float32), {}, [])
@staticmethod
def build(documents: Iterable[Document]) -> "_View":
chunks, owners, blocks, lengths = [], [], [], []
postings: dict[str, list[tuple[int, int]]] = defaultdict(list)
for doc in sorted(documents, key=lambda d: d.id):
for row, chunk in enumerate(doc.chunks):
index = len(chunks)
chunks.append(chunk)
owners.append(doc)
tokens = terms(chunk.passage_text(doc.title))
lengths.append(len(tokens))
for term, count in Counter(tokens).items():
postings[term].append((index, count))
if doc.vectors.size:
blocks.append(doc.vectors[row])
vectors = np.vstack(blocks).astype(np.float32) if blocks and len(blocks) == len(chunks) else np.zeros((0, 0), dtype=np.float32)
return _View(chunks, owners, vectors, dict(postings), lengths)
def bm25(self, query_terms: list[str]) -> list[int]:
n = len(self.chunks)
scores: dict[int, float] = defaultdict(float)
for term in set(query_terms):
posting = self.postings.get(term)
if not posting:
continue
idf = math.log(1 + (n - len(posting) + 0.5) / (len(posting) + 0.5))
for index, tf in posting:
norm = tf + K1 * (1 - B + B * self.lengths[index] / self.avg_length)
scores[index] += idf * tf * (K1 + 1) / norm
return [index for index, _ in sorted(scores.items(), key=lambda item: -item[1])[:CANDIDATES]]
def passage(self, index: int, score: float) -> dict:
chunk, doc = self.chunks[index], self.documents[index]
return {
"id": chunk.id,
"documentId": doc.id,
"title": doc.title,
"url": f"{doc.url}#{chunk.anchor}" if chunk.anchor else doc.url,
"cite": chunk.cite,
"heading": chunk.heading,
"text": chunk.text,
"typeName": doc.type_name,
"issuedDate": doc.issued_date,
"repealedAt": doc.repealed_at,
"score": round(score, 6),
}