update

src/similarity_model/preprocessing.py

@@ -4,14 +4,12 @@
 
 import re
 import logging
+import numpy as np
 from functools import lru_cache
 from pathlib import Path
 import pandas as pd
 from sentence_transformers import SentenceTransformer
 from sklearn.metrics.pairwise import cosine_similarity
-from .llm_feature_extractor import (
-    extract_features_llm
-)
 
 
 
@@ -120,18 +118,68 @@ def semantic_deduplicate(
 
 
 # =====================================================
-# Main Feature Extraction
+# Local Feature Dictionary (cached)
 # =====================================================
-def extract_features(text):
+_PROJECT_ROOT = Path(__file__).resolve().parents[2]
+_METADATA_PATH = _PROJECT_ROOT / "models" / "metadata.parquet"
 
-    logger.info("Using Gemini feature extractor")
 
-    features = extract_features_llm(
-        text
-    )
+@lru_cache(maxsize=1)
+def _load_known_features():
+    """
+    Load all known feature strings from the training metadata.
+    Cached — reads the parquet file only once per process.
+    Sorted longest-first so multi-word features (e.g. 'deep learning')
+    are matched before their sub-words (e.g. 'learning').
+    """
+    if not _METADATA_PATH.exists():
+        logger.warning("metadata.parquet not found; feature extraction will return []")
+        return []
+
+    df = pd.read_parquet(str(_METADATA_PATH))
+
+    features_set = set()
+    for f_list in df.get("features", pd.Series(dtype=object)):
+        if isinstance(f_list, (list, np.ndarray, tuple, set)):
+            for f in f_list:
+                val = str(f).strip().lower()
+                if val and val != "nan" and len(val) >= 3:
+                    features_set.add(val)
+
+    logger.info(f"Loaded {len(features_set)} known features from metadata")
+
+    # longest first → greedy multi-word match wins
+    return sorted(features_set, key=len, reverse=True)
+
+
+# =====================================================
+# Main Feature Extraction  (fully local, no API)
+# =====================================================
+def extract_features(text: str) -> list:
+    """
+    Match technical features from text against the known feature
+    dictionary built from training data.
+    No external API required.
+    """
+    known_features = _load_known_features()
+
+    if not known_features:
+        return []
+
+    text_norm = normalize_text(text)
+    matched = []
+
+    for feat in known_features:
+        pattern = r'\b' + re.escape(feat) + r'\b'
+        if re.search(pattern, text_norm):
+            matched.append(feat)
+            if len(matched) >= 15:   # collect up to 15, dedup will trim
+                break
+
+    logger.info(f"Local feature extraction matched {len(matched)} features")
 
     return semantic_deduplicate(
-        features,
+        matched,
         _get_embed_model(),
         threshold=0.85
     )