Spaces:
Sleeping
Sleeping
Download scripts/lib/term_extraction.py from defenestrate/asa: direct link, hf CLI and curl.
- Browser
- Download file 2.21 kB
-
https://huggingface.co/spaces/defenestrate/asa/resolve/main/scripts/lib/term_extraction.py
- Command line
-
hf download hf://spaces/defenestrate/asa/scripts/lib/term_extraction.py
-
curl -L -o term_extraction.py https://huggingface.co/spaces/defenestrate/asa/resolve/main/scripts/lib/term_extraction.py
2.21 kB
| """Term detection over text using term_registry.words[] aliases. | |
| Builds one compiled regex from all aliases (case-insensitive, word-bounded). | |
| Maps matches back to canonical slugs. Supports 1-hop expansion via depends_on. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| def build_regex(registry: dict) -> re.Pattern: | |
| """Compile a single regex matching any alias in registry.terms[*].words[]. | |
| Aliases are sorted longest-first so 'elasticity of demand' matches | |
| before 'elasticity'. Word boundaries enforced. Case-insensitive. | |
| """ | |
| aliases: list[tuple[str, str]] = [] # (alias, canonical_slug) | |
| for slug, term in registry.get("terms", {}).items(): | |
| for word in term.get("words", []) or []: | |
| aliases.append((word, slug)) | |
| # Longest first to prevent shorter aliases shadowing longer ones | |
| aliases.sort(key=lambda p: len(p[0]), reverse=True) | |
| if not aliases: | |
| return re.compile(r"$^") # never matches | |
| pattern = r"\b(" + "|".join(re.escape(a[0]) for a in aliases) + r")\b" | |
| return re.compile(pattern, re.IGNORECASE) | |
| def find_terms(text: str, regex: re.Pattern, registry: dict) -> list[str]: | |
| """Return list of canonical slugs whose aliases appear in text. | |
| Deduped, ordered by first appearance. | |
| """ | |
| # Build alias -> slug lookup (case-folded) | |
| alias_to_slug: dict[str, str] = {} | |
| for slug, term in registry.get("terms", {}).items(): | |
| for word in term.get("words", []) or []: | |
| alias_to_slug[word.lower()] = slug | |
| seen: list[str] = [] | |
| for m in regex.finditer(text): | |
| slug = alias_to_slug.get(m.group(0).lower()) | |
| if slug and slug not in seen: | |
| seen.append(slug) | |
| return seen | |
| def expand_one_hop(seed_slugs: list[str], registry: dict) -> list[str]: | |
| """Return 1-hop neighbors (via depends_on) of seed_slugs, excluding seeds. | |
| Output is deterministic: deduped, sorted alphabetically. | |
| """ | |
| seeds = set(seed_slugs) | |
| neighbors: set[str] = set() | |
| terms = registry.get("terms", {}) | |
| for seed in seeds: | |
| for n in terms.get(seed, {}).get("depends_on", []) or []: | |
| if n not in seeds: | |
| neighbors.add(n) | |
| return sorted(neighbors) | |