mindscope-Model / src /features /video_features.py
koushik2109's picture
Model Changes
8d7b4b1
Raw History Blame Contribute Delete
2.87 kB
import csv
from pathlib import Path
from typing import Dict, List, Optional
import numpy as np
import torch
import torchvision.models as models
import torchvision.transforms as T
from src.features.sanitization import sanitize_array
class OpenFaceFeatureLoader:
"""Load OpenFace CSV features for consistent train and inference."""
def __init__(self, csv_path: Path):
self.csv_path = csv_path
def load(self) -> Dict[str, np.ndarray]:
with open(self.csv_path, "r", encoding="utf-8") as handle:
reader = csv.DictReader(handle)
rows = list(reader)
if not rows:
raise ValueError(f"OpenFace CSV is empty: {self.csv_path}")
values = np.array([[float(v or 0.0) for v in row.values()] for row in rows], dtype=np.float32)
columns = list(rows[0].keys())
return {
"features": sanitize_array(values),
"columns": np.array(columns, dtype=object),
}
class VideoEmbeddingExtractor:
"""Visual embedding backend for ResNet50 or placeholder VideoMAE."""
def __init__(self, backend: str = "resnet50", device: str = "cpu"):
self.backend = backend
self.device = device
if backend == "resnet50":
self.model = models.resnet50(pretrained=True).to(device).eval()
self.model = torch.nn.Sequential(*list(self.model.children())[:-1])
else:
raise ValueError(f"Unsupported video embedding backend: {backend}")
self.transform = T.Compose([
T.Resize((224, 224)),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
def embed_frames(self, frames: List[np.ndarray]) -> np.ndarray:
tensors = [self.transform(frame).to(self.device) for frame in frames]
batch = torch.stack(tensors, dim=0)
with torch.no_grad():
embeddings = self.model(batch).squeeze(-1).squeeze(-1)
return sanitize_array(embeddings.cpu().numpy())
class VideoFeatureExtractor:
"""Combine behavioral OpenFace features with visual embedding features."""
def __init__(self, config: dict, device: str = "cpu"):
self.loader_backend = config.get("openface_csv_key", "openface")
self.embedding_backend = config.get("embedding_backend", "resnet50")
self.embedding_extractor = VideoEmbeddingExtractor(self.embedding_backend, device=device)
def extract_openface(self, csv_path: Path) -> Dict[str, np.ndarray]:
loader = OpenFaceFeatureLoader(csv_path)
loaded = loader.load()
features = loaded["features"]
return {
"behavior": features,
"columns": loaded["columns"],
}
def embed_frames(self, frames: List[np.ndarray]) -> np.ndarray:
return self.embedding_extractor.embed_frames(frames)