File size: 2,867 Bytes
8d7b4b1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
import csv
from pathlib import Path
from typing import Dict, List, Optional

import numpy as np
import torch
import torchvision.models as models
import torchvision.transforms as T

from src.features.sanitization import sanitize_array


class OpenFaceFeatureLoader:
    """Load OpenFace CSV features for consistent train and inference."""

    def __init__(self, csv_path: Path):
        self.csv_path = csv_path

    def load(self) -> Dict[str, np.ndarray]:
        with open(self.csv_path, "r", encoding="utf-8") as handle:
            reader = csv.DictReader(handle)
            rows = list(reader)

        if not rows:
            raise ValueError(f"OpenFace CSV is empty: {self.csv_path}")

        values = np.array([[float(v or 0.0) for v in row.values()] for row in rows], dtype=np.float32)
        columns = list(rows[0].keys())
        return {
            "features": sanitize_array(values),
            "columns": np.array(columns, dtype=object),
        }


class VideoEmbeddingExtractor:
    """Visual embedding backend for ResNet50 or placeholder VideoMAE."""

    def __init__(self, backend: str = "resnet50", device: str = "cpu"):
        self.backend = backend
        self.device = device
        if backend == "resnet50":
            self.model = models.resnet50(pretrained=True).to(device).eval()
            self.model = torch.nn.Sequential(*list(self.model.children())[:-1])
        else:
            raise ValueError(f"Unsupported video embedding backend: {backend}")
        self.transform = T.Compose([
            T.Resize((224, 224)),
            T.ToTensor(),
            T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
        ])

    def embed_frames(self, frames: List[np.ndarray]) -> np.ndarray:
        tensors = [self.transform(frame).to(self.device) for frame in frames]
        batch = torch.stack(tensors, dim=0)
        with torch.no_grad():
            embeddings = self.model(batch).squeeze(-1).squeeze(-1)
        return sanitize_array(embeddings.cpu().numpy())


class VideoFeatureExtractor:
    """Combine behavioral OpenFace features with visual embedding features."""

    def __init__(self, config: dict, device: str = "cpu"):
        self.loader_backend = config.get("openface_csv_key", "openface")
        self.embedding_backend = config.get("embedding_backend", "resnet50")
        self.embedding_extractor = VideoEmbeddingExtractor(self.embedding_backend, device=device)

    def extract_openface(self, csv_path: Path) -> Dict[str, np.ndarray]:
        loader = OpenFaceFeatureLoader(csv_path)
        loaded = loader.load()
        features = loaded["features"]
        return {
            "behavior": features,
            "columns": loaded["columns"],
        }

    def embed_frames(self, frames: List[np.ndarray]) -> np.ndarray:
        return self.embedding_extractor.embed_frames(frames)