"""Word handling for article search — kept identical to lib/rag/text.ts in the web app. Diacritics folded, Romanian function words dropped, and a deliberately light "stem": the first five letters without trailing vowels, so "bursă", "burse" and "bursele" all become "burs". Words with digits ("2026", "hs276") are kept whole. """ from __future__ import annotations import re import unicodedata STOPWORDS = frozenset( ( "a ai al ale alt am ar are as asa asta au ca care ce cel cea cei cele cu cum da dar de deci " "din dupa ei el ea este eu fi fie fost i ii il in inca intr intre isi la le li lor lui ma mai " "mi mult ne ni nici noi nu o ori pe pentru prin sa se si sau sunt s te toate tot tu un una " "unde unei unor unui va vor cine cand cat cati cate vreau pot poate trebuie exista despre " "spune spun prevede prevad avem aveti imi" ).split() ) _NON_WORD = re.compile(r"[^a-z0-9]+") def normalize(text: str) -> str: """Lower-case without diacritics (ă→a, ș/ş→s, ț/ţ→t), like normalizeForSearch in the app.""" decomposed = unicodedata.normalize("NFD", text) return "".join(c for c in decomposed if not unicodedata.combining(c)).lower() def words(text: str) -> list[str]: return [w for w in _NON_WORD.split(normalize(text)) if w] def stem(word: str) -> str: if any(c.isdigit() for c in word) or len(word) <= 3: return word return word[:5].rstrip("aeiu") or word[:3] def terms(text: str) -> list[str]: """Stems of the meaningful words of a text (question or passage).""" return [stem(w) for w in words(text) if w not in STOPWORDS]