Spaces:
Sleeping
Sleeping
Download app/text.py from Catal/docSpaceSearch: direct link, hf CLI and curl.
- Browser
- Download file 1.65 kB
-
https://huggingface.co/spaces/Catal/docSpaceSearch/resolve/main/app/text.py
- Command line
-
hf download hf://spaces/Catal/docSpaceSearch/app/text.py
-
curl -L -o text.py https://huggingface.co/spaces/Catal/docSpaceSearch/resolve/main/app/text.py
1.65 kB
| """Word handling for article search — kept identical to lib/rag/text.ts in the web app. | |
| Diacritics folded, Romanian function words dropped, and a deliberately light "stem": the first five | |
| letters without trailing vowels, so "bursă", "burse" and "bursele" all become "burs". Words with | |
| digits ("2026", "hs276") are kept whole. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| import unicodedata | |
| STOPWORDS = frozenset( | |
| ( | |
| "a ai al ale alt am ar are as asa asta au ca care ce cel cea cei cele cu cum da dar de deci " | |
| "din dupa ei el ea este eu fi fie fost i ii il in inca intr intre isi la le li lor lui ma mai " | |
| "mi mult ne ni nici noi nu o ori pe pentru prin sa se si sau sunt s te toate tot tu un una " | |
| "unde unei unor unui va vor cine cand cat cati cate vreau pot poate trebuie exista despre " | |
| "spune spun prevede prevad avem aveti imi" | |
| ).split() | |
| ) | |
| _NON_WORD = re.compile(r"[^a-z0-9]+") | |
| def normalize(text: str) -> str: | |
| """Lower-case without diacritics (ă→a, ș/ş→s, ț/ţ→t), like normalizeForSearch in the app.""" | |
| decomposed = unicodedata.normalize("NFD", text) | |
| return "".join(c for c in decomposed if not unicodedata.combining(c)).lower() | |
| def words(text: str) -> list[str]: | |
| return [w for w in _NON_WORD.split(normalize(text)) if w] | |
| def stem(word: str) -> str: | |
| if any(c.isdigit() for c in word) or len(word) <= 3: | |
| return word | |
| return word[:5].rstrip("aeiu") or word[:3] | |
| def terms(text: str) -> list[str]: | |
| """Stems of the meaningful words of a text (question or passage).""" | |
| return [stem(w) for w in words(text) if w not in STOPWORDS] | |