File size: 1,647 Bytes
794ae1e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
"""Word handling for article search — kept identical to lib/rag/text.ts in the web app.

Diacritics folded, Romanian function words dropped, and a deliberately light "stem": the first five
letters without trailing vowels, so "bursă", "burse" and "bursele" all become "burs". Words with
digits ("2026", "hs276") are kept whole.
"""

from __future__ import annotations

import re
import unicodedata

STOPWORDS = frozenset(
    (
        "a ai al ale alt am ar are as asa asta au ca care ce cel cea cei cele cu cum da dar de deci "
        "din dupa ei el ea este eu fi fie fost i ii il in inca intr intre isi la le li lor lui ma mai "
        "mi mult ne ni nici noi nu o ori pe pentru prin sa se si sau sunt s te toate tot tu un una "
        "unde unei unor unui va vor cine cand cat cati cate vreau pot poate trebuie exista despre "
        "spune spun prevede prevad avem aveti imi"
    ).split()
)

_NON_WORD = re.compile(r"[^a-z0-9]+")


def normalize(text: str) -> str:
    """Lower-case without diacritics (ă→a, ș/ş→s, ț/ţ→t), like normalizeForSearch in the app."""
    decomposed = unicodedata.normalize("NFD", text)
    return "".join(c for c in decomposed if not unicodedata.combining(c)).lower()


def words(text: str) -> list[str]:
    return [w for w in _NON_WORD.split(normalize(text)) if w]


def stem(word: str) -> str:
    if any(c.isdigit() for c in word) or len(word) <= 3:
        return word
    return word[:5].rstrip("aeiu") or word[:3]


def terms(text: str) -> list[str]:
    """Stems of the meaningful words of a text (question or passage)."""
    return [stem(w) for w in words(text) if w not in STOPWORDS]