Spaces:
Sleeping
Sleeping
Download src/features/text_features.py from koushik2109/mindscope-Model: direct link, hf CLI and curl.
- Browser
- Download file 2.66 kB
-
https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/features/text_features.py
- Command line
-
hf download hf://spaces/koushik2109/mindscope-Model/src/features/text_features.py
-
curl -L -o text_features.py https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/features/text_features.py
2.66 kB
| from pathlib import Path | |
| from typing import Dict, Sequence | |
| import numpy as np | |
| from sentence_transformers import SentenceTransformer | |
| from pathlib import Path | |
| from typing import Optional | |
| from src.features.preprocessing import FeaturePreprocessor, PCATransform | |
| import logging | |
| logger = logging.getLogger(__name__) | |
| from src.features.sanitization import sanitize_array | |
| class TextFeatureExtractor: | |
| """SentenceTransformer-based text embedding for transcripts. | |
| Optionally applies a serialized `FeaturePreprocessor` (normalizer + PCA) | |
| saved during training so the runtime embeddings match the model input | |
| dimensionality (e.g. 768 -> 133). | |
| """ | |
| def __init__(self, model_name: str = "sentence-transformers/all-mpnet-base-v2", preprocessor_path: Optional[Path] = None): | |
| self.model = SentenceTransformer(model_name) | |
| self.preprocessor: Optional[FeaturePreprocessor] = None | |
| if preprocessor_path is not None: | |
| p = Path(preprocessor_path) | |
| if p.exists(): | |
| try: | |
| self.preprocessor = FeaturePreprocessor.load(p) | |
| logger.info("Loaded FeaturePreprocessor from %s", str(p)) | |
| except Exception: | |
| try: | |
| # Try loading a raw PCA reducer if present | |
| pca = PCATransform.load(p) | |
| class _PCAWrap: | |
| def __init__(self, pca): | |
| self._pca = pca | |
| def transform(self, x): | |
| return self._pca.transform(x) | |
| self.preprocessor = _PCAWrap(pca) | |
| logger.info("Loaded PCATransform from %s", str(p)) | |
| except Exception: | |
| logger.warning("Failed to load preprocessor from %s; continuing without it", str(p)) | |
| self.preprocessor = None | |
| def encode(self, texts: Sequence[str]) -> np.ndarray: | |
| embeddings = self.model.encode(list(texts), convert_to_numpy=True, show_progress_bar=False) | |
| arr = np.asarray(embeddings, dtype=np.float32) | |
| if self.preprocessor is not None: | |
| try: | |
| arr = self.preprocessor.transform(arr) | |
| except Exception: | |
| pass | |
| return sanitize_array(arr) | |
| def encode_text(self, transcript: str, chunks: Sequence[str]) -> Dict[str, np.ndarray]: | |
| session_embedding = self.encode([transcript])[0] | |
| chunk_embeddings = self.encode(chunks) | |
| return { | |
| "session": session_embedding, | |
| "chunks": chunk_embeddings, | |
| } | |