Spaces:
Sleeping
Sleeping
Download app/index.py from Catal/docSpaceSearch: direct link, hf CLI and curl.
- Browser
- Download file 8.13 kB
-
https://huggingface.co/spaces/Catal/docSpaceSearch/resolve/main/app/index.py
- Command line
-
hf download hf://spaces/Catal/docSpaceSearch/app/index.py
-
curl -L -o index.py https://huggingface.co/spaces/Catal/docSpaceSearch/resolve/main/app/index.py
8.13 kB
| """In-memory hybrid index over the article chunks exported by the web app. | |
| * BM25 (inverted index) catches exact words: "art. 5", numbers, names, legal terms. | |
| * Embeddings (multilingual-e5-small by default) catch paraphrases: "bani pentru cazare" ↔ | |
| "subvenție pentru cămin". | |
| * The two rankings are merged with reciprocal rank fusion (RRF). | |
| Documents carry a content hash: applying a new export re-embeds only documents that changed. | |
| Everything lives in memory and is rebuilt from the app's export on start-up (the free Space disk is | |
| not persistent) — a few thousand documents take seconds to minutes on CPU. | |
| """ | |
| from __future__ import annotations | |
| import math | |
| import threading | |
| from collections import Counter, defaultdict | |
| from dataclasses import dataclass | |
| from typing import Callable, Iterable, Sequence | |
| import numpy as np | |
| from .text import terms | |
| # Texts → L2-normalised vectors (n × dim). `kind` is "query" or "passage" (e5 needs the prefix). | |
| Embedder = Callable[[Sequence[str], str], np.ndarray] | |
| RRF_K = 60 | |
| CANDIDATES = 50 | |
| K1, B = 1.2, 0.75 | |
| class Chunk: | |
| id: str | |
| document_id: str | |
| anchor: str | None | |
| cite: str | None | |
| heading: str | None | |
| text: str | |
| def passage_text(self, title: str) -> str: | |
| """What gets embedded and BM25-indexed: the title and heading give the article its subject.""" | |
| return "\n".join(part for part in (title, self.heading, self.text) if part) | |
| class Document: | |
| id: str | |
| hash: str | |
| title: str | |
| url: str | |
| type_name: str | |
| issued_date: str | None | |
| repealed_at: str | None | |
| chunks: list[Chunk] | |
| vectors: np.ndarray # one row per chunk | |
| def parse(raw: dict) -> "Document": | |
| chunks = [ | |
| Chunk( | |
| id=c["id"], | |
| document_id=raw["id"], | |
| anchor=c.get("anchor"), | |
| cite=c.get("cite"), | |
| heading=c.get("heading"), | |
| text=c.get("text", ""), | |
| ) | |
| for c in raw.get("chunks", []) | |
| if c.get("text", "").strip() | |
| ] | |
| return Document( | |
| id=raw["id"], | |
| hash=raw.get("hash", ""), | |
| title=raw.get("title", ""), | |
| url=raw.get("url", ""), | |
| type_name=(raw.get("type") or {}).get("name", ""), | |
| issued_date=raw.get("issuedDate"), | |
| repealed_at=raw.get("repealedAt"), | |
| chunks=chunks, | |
| vectors=np.zeros((0, 0), dtype=np.float32), | |
| ) | |
| class SearchIndex: | |
| def __init__(self, embed: Embedder): | |
| self._embed = embed | |
| self._write_lock = threading.Lock() # one update at a time (refresh vs. push from the app) | |
| self._documents: dict[str, Document] = {} | |
| # Searches read an immutable snapshot, replaced in one assignment after every update. | |
| self._view = _View.empty() | |
| # --- updates | |
| def apply(self, upsert: Iterable[dict] = (), delete: Iterable[str] = ()) -> dict: | |
| """Add or replace documents (only re-embedding changed ones) and remove others.""" | |
| incoming = [Document.parse(raw) for raw in upsert] | |
| with self._write_lock: | |
| changed = [d for d in incoming if not d.hash or getattr(self._documents.get(d.id), "hash", None) != d.hash] | |
| # One batch for all changed documents, then split back per document. | |
| texts = [chunk.passage_text(doc.title) for doc in changed for chunk in doc.chunks] | |
| vectors = self._embed(texts, "passage") if texts else np.zeros((0, 0), dtype=np.float32) | |
| offset = 0 | |
| for doc in changed: | |
| doc.vectors = vectors[offset : offset + len(doc.chunks)] | |
| offset += len(doc.chunks) | |
| self._documents[doc.id] = doc | |
| removed = [doc_id for doc_id in delete if self._documents.pop(doc_id, None) is not None] | |
| self._view = _View.build(self._documents.values()) | |
| return {"updated": len(changed), "unchanged": len(incoming) - len(changed), "deleted": len(removed)} | |
| def replace_all(self, documents: list[dict]) -> dict: | |
| """Apply a full export: documents missing from it are no longer public.""" | |
| keep = {raw["id"] for raw in documents} | |
| stale = [doc_id for doc_id in list(self._documents) if doc_id not in keep] | |
| return self.apply(documents, stale) | |
| # --- queries | |
| def stats(self) -> dict: | |
| view = self._view | |
| return {"documents": len(self._documents), "chunks": len(view.chunks)} | |
| def search(self, query: str, k: int = 6) -> list[dict]: | |
| view = self._view | |
| if not view.chunks or not query.strip(): | |
| return [] | |
| ranked_lists = [view.bm25(terms(query))] | |
| if view.vectors.size: | |
| q = self._embed([query], "query")[0] | |
| similarity = view.vectors @ q | |
| top = np.argsort(-similarity)[:CANDIDATES] | |
| ranked_lists.append([int(i) for i in top]) | |
| fused: dict[int, float] = defaultdict(float) | |
| for ranked in ranked_lists: | |
| for rank, index in enumerate(ranked): | |
| fused[index] += 1.0 / (RRF_K + rank + 1) | |
| best = sorted(fused.items(), key=lambda item: -item[1])[:k] | |
| return [view.passage(index, score) for index, score in best] | |
| class _View: | |
| """Immutable snapshot of all chunks: BM25 postings and the embedding matrix.""" | |
| def __init__(self, chunks, documents, vectors, postings, lengths): | |
| self.chunks: list[Chunk] = chunks | |
| self.documents: list[Document] = documents # document of each chunk | |
| self.vectors: np.ndarray = vectors | |
| self.postings: dict[str, list[tuple[int, int]]] = postings | |
| self.lengths: list[int] = lengths | |
| self.avg_length = (sum(lengths) / len(lengths)) if lengths else 1.0 | |
| def empty() -> "_View": | |
| return _View([], [], np.zeros((0, 0), dtype=np.float32), {}, []) | |
| def build(documents: Iterable[Document]) -> "_View": | |
| chunks, owners, blocks, lengths = [], [], [], [] | |
| postings: dict[str, list[tuple[int, int]]] = defaultdict(list) | |
| for doc in sorted(documents, key=lambda d: d.id): | |
| for row, chunk in enumerate(doc.chunks): | |
| index = len(chunks) | |
| chunks.append(chunk) | |
| owners.append(doc) | |
| tokens = terms(chunk.passage_text(doc.title)) | |
| lengths.append(len(tokens)) | |
| for term, count in Counter(tokens).items(): | |
| postings[term].append((index, count)) | |
| if doc.vectors.size: | |
| blocks.append(doc.vectors[row]) | |
| vectors = np.vstack(blocks).astype(np.float32) if blocks and len(blocks) == len(chunks) else np.zeros((0, 0), dtype=np.float32) | |
| return _View(chunks, owners, vectors, dict(postings), lengths) | |
| def bm25(self, query_terms: list[str]) -> list[int]: | |
| n = len(self.chunks) | |
| scores: dict[int, float] = defaultdict(float) | |
| for term in set(query_terms): | |
| posting = self.postings.get(term) | |
| if not posting: | |
| continue | |
| idf = math.log(1 + (n - len(posting) + 0.5) / (len(posting) + 0.5)) | |
| for index, tf in posting: | |
| norm = tf + K1 * (1 - B + B * self.lengths[index] / self.avg_length) | |
| scores[index] += idf * tf * (K1 + 1) / norm | |
| return [index for index, _ in sorted(scores.items(), key=lambda item: -item[1])[:CANDIDATES]] | |
| def passage(self, index: int, score: float) -> dict: | |
| chunk, doc = self.chunks[index], self.documents[index] | |
| return { | |
| "id": chunk.id, | |
| "documentId": doc.id, | |
| "title": doc.title, | |
| "url": f"{doc.url}#{chunk.anchor}" if chunk.anchor else doc.url, | |
| "cite": chunk.cite, | |
| "heading": chunk.heading, | |
| "text": chunk.text, | |
| "typeName": doc.type_name, | |
| "issuedDate": doc.issued_date, | |
| "repealedAt": doc.repealed_at, | |
| "score": round(score, 6), | |
| } | |