Spaces:
Sleeping
Sleeping
File size: 1,647 Bytes
794ae1e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | """Word handling for article search — kept identical to lib/rag/text.ts in the web app.
Diacritics folded, Romanian function words dropped, and a deliberately light "stem": the first five
letters without trailing vowels, so "bursă", "burse" and "bursele" all become "burs". Words with
digits ("2026", "hs276") are kept whole.
"""
from __future__ import annotations
import re
import unicodedata
STOPWORDS = frozenset(
(
"a ai al ale alt am ar are as asa asta au ca care ce cel cea cei cele cu cum da dar de deci "
"din dupa ei el ea este eu fi fie fost i ii il in inca intr intre isi la le li lor lui ma mai "
"mi mult ne ni nici noi nu o ori pe pentru prin sa se si sau sunt s te toate tot tu un una "
"unde unei unor unui va vor cine cand cat cati cate vreau pot poate trebuie exista despre "
"spune spun prevede prevad avem aveti imi"
).split()
)
_NON_WORD = re.compile(r"[^a-z0-9]+")
def normalize(text: str) -> str:
"""Lower-case without diacritics (ă→a, ș/ş→s, ț/ţ→t), like normalizeForSearch in the app."""
decomposed = unicodedata.normalize("NFD", text)
return "".join(c for c in decomposed if not unicodedata.combining(c)).lower()
def words(text: str) -> list[str]:
return [w for w in _NON_WORD.split(normalize(text)) if w]
def stem(word: str) -> str:
if any(c.isdigit() for c in word) or len(word) <= 3:
return word
return word[:5].rstrip("aeiu") or word[:3]
def terms(text: str) -> list[str]:
"""Stems of the meaningful words of a text (question or passage)."""
return [stem(w) for w in words(text) if w not in STOPWORDS]
|