docSpaceSearch / app /text.py
Catal's picture
Rename text.py to app/text.py
21752f0 verified
Raw History Blame Contribute Delete
1.65 kB
"""Word handling for article search — kept identical to lib/rag/text.ts in the web app.
Diacritics folded, Romanian function words dropped, and a deliberately light "stem": the first five
letters without trailing vowels, so "bursă", "burse" and "bursele" all become "burs". Words with
digits ("2026", "hs276") are kept whole.
"""
from __future__ import annotations
import re
import unicodedata
STOPWORDS = frozenset(
(
"a ai al ale alt am ar are as asa asta au ca care ce cel cea cei cele cu cum da dar de deci "
"din dupa ei el ea este eu fi fie fost i ii il in inca intr intre isi la le li lor lui ma mai "
"mi mult ne ni nici noi nu o ori pe pentru prin sa se si sau sunt s te toate tot tu un una "
"unde unei unor unui va vor cine cand cat cati cate vreau pot poate trebuie exista despre "
"spune spun prevede prevad avem aveti imi"
).split()
)
_NON_WORD = re.compile(r"[^a-z0-9]+")
def normalize(text: str) -> str:
"""Lower-case without diacritics (ă→a, ș/ş→s, ț/ţ→t), like normalizeForSearch in the app."""
decomposed = unicodedata.normalize("NFD", text)
return "".join(c for c in decomposed if not unicodedata.combining(c)).lower()
def words(text: str) -> list[str]:
return [w for w in _NON_WORD.split(normalize(text)) if w]
def stem(word: str) -> str:
if any(c.isdigit() for c in word) or len(word) <= 3:
return word
return word[:5].rstrip("aeiu") or word[:3]
def terms(text: str) -> list[str]:
"""Stems of the meaningful words of a text (question or passage)."""
return [stem(w) for w in words(text) if w not in STOPWORDS]