Spaces:
Runtime error
Runtime error
File size: 2,218 Bytes
fc7db98 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | """
Handcrafted linguistic features for hybrid LCP models.
Literature: word frequency is the strongest single predictor; WordNet senses
capture ambiguity; syntax score captures structural difficulty.
"""
from __future__ import annotations
import math
from functools import lru_cache
import numpy as np
import wordfreq
from syntax_complexity import analyze_syntax
FEATURE_NAMES = [
"log_word_frequency",
"word_length",
"syllable_estimate",
"wordnet_senses",
"syntax_complexity",
]
N_LINGUISTIC_FEATURES = len(FEATURE_NAMES)
@lru_cache(maxsize=1)
def _wordnet_ready() -> bool:
try:
from nltk.corpus import wordnet as wn
wn.synsets("test")
return True
except LookupError:
import nltk
nltk.download("wordnet", quiet=True)
nltk.download("omw-1.4", quiet=True)
return True
def _syllable_estimate(word: str) -> float:
word = word.lower()
if not word:
return 0.0
vowels = "aeiouy"
count = 0
prev_vowel = False
for ch in word:
is_vowel = ch in vowels
if is_vowel and not prev_vowel:
count += 1
prev_vowel = is_vowel
return float(max(1, count))
def _wordnet_sense_count(word: str) -> float:
_wordnet_ready()
from nltk.corpus import wordnet as wn
return float(len(wn.synsets(word.lower())))
def extract_linguistic_features(sentence: str, target_word: str) -> np.ndarray:
"""Return a 5-dim feature vector for one (sentence, target_word) pair."""
word = str(target_word).strip()
freq = wordfreq.word_frequency(word.lower(), "en")
log_freq = math.log10(freq + 1e-12)
word_len = len(word)
syllables = _syllable_estimate(word)
senses = _wordnet_sense_count(word)
syntax = analyze_syntax(str(sentence)).complexity_score
return np.array([log_freq, word_len, syllables, senses, syntax], dtype=np.float32)
def normalize_features(matrix: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Z-score normalize feature matrix; return normalized, mean, std."""
mean = matrix.mean(axis=0)
std = matrix.std(axis=0)
std = np.where(std < 1e-6, 1.0, std)
return (matrix - mean) / std, mean, std
|