"""In-memory hybrid index over the article chunks exported by the web app. * BM25 (inverted index) catches exact words: "art. 5", numbers, names, legal terms. * Embeddings (multilingual-e5-small by default) catch paraphrases: "bani pentru cazare" ↔ "subvenție pentru cămin". * The two rankings are merged with reciprocal rank fusion (RRF). Documents carry a content hash: applying a new export re-embeds only documents that changed. Everything lives in memory and is rebuilt from the app's export on start-up (the free Space disk is not persistent) — a few thousand documents take seconds to minutes on CPU. """ from __future__ import annotations import math import threading from collections import Counter, defaultdict from dataclasses import dataclass from typing import Callable, Iterable, Sequence import numpy as np from .text import terms # Texts → L2-normalised vectors (n × dim). `kind` is "query" or "passage" (e5 needs the prefix). Embedder = Callable[[Sequence[str], str], np.ndarray] RRF_K = 60 CANDIDATES = 50 K1, B = 1.2, 0.75 @dataclass(frozen=True) class Chunk: id: str document_id: str anchor: str | None cite: str | None heading: str | None text: str def passage_text(self, title: str) -> str: """What gets embedded and BM25-indexed: the title and heading give the article its subject.""" return "\n".join(part for part in (title, self.heading, self.text) if part) @dataclass class Document: id: str hash: str title: str url: str type_name: str issued_date: str | None repealed_at: str | None chunks: list[Chunk] vectors: np.ndarray # one row per chunk @staticmethod def parse(raw: dict) -> "Document": chunks = [ Chunk( id=c["id"], document_id=raw["id"], anchor=c.get("anchor"), cite=c.get("cite"), heading=c.get("heading"), text=c.get("text", ""), ) for c in raw.get("chunks", []) if c.get("text", "").strip() ] return Document( id=raw["id"], hash=raw.get("hash", ""), title=raw.get("title", ""), url=raw.get("url", ""), type_name=(raw.get("type") or {}).get("name", ""), issued_date=raw.get("issuedDate"), repealed_at=raw.get("repealedAt"), chunks=chunks, vectors=np.zeros((0, 0), dtype=np.float32), ) class SearchIndex: def __init__(self, embed: Embedder): self._embed = embed self._write_lock = threading.Lock() # one update at a time (refresh vs. push from the app) self._documents: dict[str, Document] = {} # Searches read an immutable snapshot, replaced in one assignment after every update. self._view = _View.empty() # --- updates def apply(self, upsert: Iterable[dict] = (), delete: Iterable[str] = ()) -> dict: """Add or replace documents (only re-embedding changed ones) and remove others.""" incoming = [Document.parse(raw) for raw in upsert] with self._write_lock: changed = [d for d in incoming if not d.hash or getattr(self._documents.get(d.id), "hash", None) != d.hash] # One batch for all changed documents, then split back per document. texts = [chunk.passage_text(doc.title) for doc in changed for chunk in doc.chunks] vectors = self._embed(texts, "passage") if texts else np.zeros((0, 0), dtype=np.float32) offset = 0 for doc in changed: doc.vectors = vectors[offset : offset + len(doc.chunks)] offset += len(doc.chunks) self._documents[doc.id] = doc removed = [doc_id for doc_id in delete if self._documents.pop(doc_id, None) is not None] self._view = _View.build(self._documents.values()) return {"updated": len(changed), "unchanged": len(incoming) - len(changed), "deleted": len(removed)} def replace_all(self, documents: list[dict]) -> dict: """Apply a full export: documents missing from it are no longer public.""" keep = {raw["id"] for raw in documents} stale = [doc_id for doc_id in list(self._documents) if doc_id not in keep] return self.apply(documents, stale) # --- queries @property def stats(self) -> dict: view = self._view return {"documents": len(self._documents), "chunks": len(view.chunks)} def search(self, query: str, k: int = 6) -> list[dict]: view = self._view if not view.chunks or not query.strip(): return [] ranked_lists = [view.bm25(terms(query))] if view.vectors.size: q = self._embed([query], "query")[0] similarity = view.vectors @ q top = np.argsort(-similarity)[:CANDIDATES] ranked_lists.append([int(i) for i in top]) fused: dict[int, float] = defaultdict(float) for ranked in ranked_lists: for rank, index in enumerate(ranked): fused[index] += 1.0 / (RRF_K + rank + 1) best = sorted(fused.items(), key=lambda item: -item[1])[:k] return [view.passage(index, score) for index, score in best] class _View: """Immutable snapshot of all chunks: BM25 postings and the embedding matrix.""" def __init__(self, chunks, documents, vectors, postings, lengths): self.chunks: list[Chunk] = chunks self.documents: list[Document] = documents # document of each chunk self.vectors: np.ndarray = vectors self.postings: dict[str, list[tuple[int, int]]] = postings self.lengths: list[int] = lengths self.avg_length = (sum(lengths) / len(lengths)) if lengths else 1.0 @staticmethod def empty() -> "_View": return _View([], [], np.zeros((0, 0), dtype=np.float32), {}, []) @staticmethod def build(documents: Iterable[Document]) -> "_View": chunks, owners, blocks, lengths = [], [], [], [] postings: dict[str, list[tuple[int, int]]] = defaultdict(list) for doc in sorted(documents, key=lambda d: d.id): for row, chunk in enumerate(doc.chunks): index = len(chunks) chunks.append(chunk) owners.append(doc) tokens = terms(chunk.passage_text(doc.title)) lengths.append(len(tokens)) for term, count in Counter(tokens).items(): postings[term].append((index, count)) if doc.vectors.size: blocks.append(doc.vectors[row]) vectors = np.vstack(blocks).astype(np.float32) if blocks and len(blocks) == len(chunks) else np.zeros((0, 0), dtype=np.float32) return _View(chunks, owners, vectors, dict(postings), lengths) def bm25(self, query_terms: list[str]) -> list[int]: n = len(self.chunks) scores: dict[int, float] = defaultdict(float) for term in set(query_terms): posting = self.postings.get(term) if not posting: continue idf = math.log(1 + (n - len(posting) + 0.5) / (len(posting) + 0.5)) for index, tf in posting: norm = tf + K1 * (1 - B + B * self.lengths[index] / self.avg_length) scores[index] += idf * tf * (K1 + 1) / norm return [index for index, _ in sorted(scores.items(), key=lambda item: -item[1])[:CANDIDATES]] def passage(self, index: int, score: float) -> dict: chunk, doc = self.chunks[index], self.documents[index] return { "id": chunk.id, "documentId": doc.id, "title": doc.title, "url": f"{doc.url}#{chunk.anchor}" if chunk.anchor else doc.url, "cite": chunk.cite, "heading": chunk.heading, "text": chunk.text, "typeName": doc.type_name, "issuedDate": doc.issued_date, "repealedAt": doc.repealed_at, "score": round(score, 6), }