Spaces:
Sleeping
Sleeping
File size: 2,867 Bytes
8d7b4b1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | import csv
from pathlib import Path
from typing import Dict, List, Optional
import numpy as np
import torch
import torchvision.models as models
import torchvision.transforms as T
from src.features.sanitization import sanitize_array
class OpenFaceFeatureLoader:
"""Load OpenFace CSV features for consistent train and inference."""
def __init__(self, csv_path: Path):
self.csv_path = csv_path
def load(self) -> Dict[str, np.ndarray]:
with open(self.csv_path, "r", encoding="utf-8") as handle:
reader = csv.DictReader(handle)
rows = list(reader)
if not rows:
raise ValueError(f"OpenFace CSV is empty: {self.csv_path}")
values = np.array([[float(v or 0.0) for v in row.values()] for row in rows], dtype=np.float32)
columns = list(rows[0].keys())
return {
"features": sanitize_array(values),
"columns": np.array(columns, dtype=object),
}
class VideoEmbeddingExtractor:
"""Visual embedding backend for ResNet50 or placeholder VideoMAE."""
def __init__(self, backend: str = "resnet50", device: str = "cpu"):
self.backend = backend
self.device = device
if backend == "resnet50":
self.model = models.resnet50(pretrained=True).to(device).eval()
self.model = torch.nn.Sequential(*list(self.model.children())[:-1])
else:
raise ValueError(f"Unsupported video embedding backend: {backend}")
self.transform = T.Compose([
T.Resize((224, 224)),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
def embed_frames(self, frames: List[np.ndarray]) -> np.ndarray:
tensors = [self.transform(frame).to(self.device) for frame in frames]
batch = torch.stack(tensors, dim=0)
with torch.no_grad():
embeddings = self.model(batch).squeeze(-1).squeeze(-1)
return sanitize_array(embeddings.cpu().numpy())
class VideoFeatureExtractor:
"""Combine behavioral OpenFace features with visual embedding features."""
def __init__(self, config: dict, device: str = "cpu"):
self.loader_backend = config.get("openface_csv_key", "openface")
self.embedding_backend = config.get("embedding_backend", "resnet50")
self.embedding_extractor = VideoEmbeddingExtractor(self.embedding_backend, device=device)
def extract_openface(self, csv_path: Path) -> Dict[str, np.ndarray]:
loader = OpenFaceFeatureLoader(csv_path)
loaded = loader.load()
features = loaded["features"]
return {
"behavior": features,
"columns": loaded["columns"],
}
def embed_frames(self, frames: List[np.ndarray]) -> np.ndarray:
return self.embedding_extractor.embed_frames(frames)
|