Spaces:
Sleeping
Sleeping
Download src/features/video_features.py from koushik2109/mindscope-Model: direct link, hf CLI and curl.
- Browser
- Download file 2.87 kB
-
https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/features/video_features.py
- Command line
-
hf download hf://spaces/koushik2109/mindscope-Model/src/features/video_features.py
-
curl -L -o video_features.py https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/features/video_features.py
2.87 kB
| import csv | |
| from pathlib import Path | |
| from typing import Dict, List, Optional | |
| import numpy as np | |
| import torch | |
| import torchvision.models as models | |
| import torchvision.transforms as T | |
| from src.features.sanitization import sanitize_array | |
| class OpenFaceFeatureLoader: | |
| """Load OpenFace CSV features for consistent train and inference.""" | |
| def __init__(self, csv_path: Path): | |
| self.csv_path = csv_path | |
| def load(self) -> Dict[str, np.ndarray]: | |
| with open(self.csv_path, "r", encoding="utf-8") as handle: | |
| reader = csv.DictReader(handle) | |
| rows = list(reader) | |
| if not rows: | |
| raise ValueError(f"OpenFace CSV is empty: {self.csv_path}") | |
| values = np.array([[float(v or 0.0) for v in row.values()] for row in rows], dtype=np.float32) | |
| columns = list(rows[0].keys()) | |
| return { | |
| "features": sanitize_array(values), | |
| "columns": np.array(columns, dtype=object), | |
| } | |
| class VideoEmbeddingExtractor: | |
| """Visual embedding backend for ResNet50 or placeholder VideoMAE.""" | |
| def __init__(self, backend: str = "resnet50", device: str = "cpu"): | |
| self.backend = backend | |
| self.device = device | |
| if backend == "resnet50": | |
| self.model = models.resnet50(pretrained=True).to(device).eval() | |
| self.model = torch.nn.Sequential(*list(self.model.children())[:-1]) | |
| else: | |
| raise ValueError(f"Unsupported video embedding backend: {backend}") | |
| self.transform = T.Compose([ | |
| T.Resize((224, 224)), | |
| T.ToTensor(), | |
| T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), | |
| ]) | |
| def embed_frames(self, frames: List[np.ndarray]) -> np.ndarray: | |
| tensors = [self.transform(frame).to(self.device) for frame in frames] | |
| batch = torch.stack(tensors, dim=0) | |
| with torch.no_grad(): | |
| embeddings = self.model(batch).squeeze(-1).squeeze(-1) | |
| return sanitize_array(embeddings.cpu().numpy()) | |
| class VideoFeatureExtractor: | |
| """Combine behavioral OpenFace features with visual embedding features.""" | |
| def __init__(self, config: dict, device: str = "cpu"): | |
| self.loader_backend = config.get("openface_csv_key", "openface") | |
| self.embedding_backend = config.get("embedding_backend", "resnet50") | |
| self.embedding_extractor = VideoEmbeddingExtractor(self.embedding_backend, device=device) | |
| def extract_openface(self, csv_path: Path) -> Dict[str, np.ndarray]: | |
| loader = OpenFaceFeatureLoader(csv_path) | |
| loaded = loader.load() | |
| features = loaded["features"] | |
| return { | |
| "behavior": features, | |
| "columns": loaded["columns"], | |
| } | |
| def embed_frames(self, frames: List[np.ndarray]) -> np.ndarray: | |
| return self.embedding_extractor.embed_frames(frames) | |