"""T2 sequence classifiers (fine-grained action recognition). Every model takes ``x: {modality: (B, T, F_m)}`` and ``mask: (B, T)`` (True = valid frame) and returns ``{"verb_fine", "verb_composite", "noun", "hand"}`` logits. Inputs are 20 Hz windows (4 s sliding windows, or annotated segments of at most ``max_T`` frames). Modalities are always consumed in the order of ``modality_dims`` (the config order), never in dict-iteration order of ``x``. Models (sensor adaptations of the published methods; each keeps the method's key idea and uses the shared :class:`~pulse.models.t2.heads.TripletHead`): * ``deepconvlstm`` DeepConvLSTM (Ordonez & Roggen 2016): 4 conv + 2 LSTM layers on the concatenated modalities. * ``rulstm`` RU-LSTM (Furnari & Farinella 2020): per-modality rolling LSTM + unrolling LSTMCell, late (mean) fusion. * ``futr`` FUTR (Gong et al. 2022): Transformer encoder + one learnable decoder query. * ``afft`` AFFT (Zhong et al. 2023): one token per frame and modality, causal attention, read-out at the last valid frame. * ``handformer`` HandFormer (Shamil et al. 2024): multi-scale temporal conv + Transformer, mean pool. * ``actionllm`` ActionLLM *surrogate*: conv stem + 2-layer Transformer trained from scratch (no language model). * ``dailyactformer_v2`` DailyActFormer (DAF, ours): per-modality multi-scale conv stems, whole-modality dropout, concat + 1x1 fusion, Transformer, learnable-query pooling, joint verb-noun head. Differences from the pre-camera-ready code (``experiments/models_seqpred.py``), all of which only matter for variable-length (segment-aligned) batches: the LSTMs of DeepConvLSTM and RU-LSTM read packed sequences, so the final state is that of the last valid frame (previously the last padded frame); AFFT reads out the tokens of the last valid frame (previously the last padded frame). Head sizes come from the taxonomy instead of module constants. """ from __future__ import annotations from typing import Dict, List, Mapping, Optional import torch import torch.nn as nn from torch.nn.utils.rnn import pack_padded_sequence from .heads import JointVNTripletHead, TripletHead def masked_mean(h: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: m = mask.to(h.dtype).unsqueeze(-1) return (h * m).sum(dim=1) / m.sum(dim=1).clamp(min=1.0) def lengths_of(mask: torch.Tensor) -> torch.Tensor: return mask.sum(dim=1).clamp(min=1).to(torch.int64) def _concat(x: Mapping[str, torch.Tensor], order: List[str]) -> torch.Tensor: missing = [m for m in order if m not in x] if missing: raise KeyError(f"model expects modalities {order}, input lacks {missing}") return torch.cat([x[m] for m in order], dim=-1) def _encoder(d_model: int, n_heads: int, n_layers: int, dropout: float) -> nn.TransformerEncoder: layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=n_heads, dim_feedforward=4 * d_model, dropout=dropout, batch_first=True, activation="gelu") return nn.TransformerEncoder(layer, num_layers=n_layers, enable_nested_tensor=False) def _check_T(T: int, max_T: int, name: str) -> None: if T > max_T: raise ValueError(f"{name}: input has T={T} frames > max_T={max_T}") class _Base(nn.Module): def __init__(self, modality_dims: Mapping[str, int]): super().__init__() if not modality_dims: raise ValueError("a T2 model needs at least one modality") self.modality_dims = {m: int(d) for m, d in modality_dims.items()} self.modalities = list(self.modality_dims) self.in_dim = sum(self.modality_dims.values()) # --------------------------------------------------------------------------- # DeepConvLSTM # --------------------------------------------------------------------------- class DeepConvLSTMTriplet(_Base): def __init__(self, modality_dims, sizes, conv_filters: int = 64, conv_kernel: int = 5, num_conv_layers: int = 4, lstm_hidden: int = 128, num_lstm_layers: int = 2, dropout: float = 0.2, head_hidden: int = 256): super().__init__(modality_dims) convs: List[nn.Module] = [] c = self.in_dim for i in range(num_conv_layers): convs.append(nn.Sequential( nn.Conv1d(c, conv_filters, conv_kernel, padding=conv_kernel // 2), nn.BatchNorm1d(conv_filters), nn.ReLU(), nn.Dropout(dropout if i < num_conv_layers - 1 else dropout + 0.1))) c = conv_filters self.convs = nn.Sequential(*convs) self.lstm = nn.LSTM(conv_filters, lstm_hidden, num_layers=num_lstm_layers, batch_first=True, dropout=dropout if num_lstm_layers > 1 else 0.0) self.head = TripletHead(lstm_hidden, sizes, hidden=head_hidden, dropout=dropout) def forward(self, x, mask): f = self.convs(_concat(x, self.modalities).transpose(1, 2)).transpose(1, 2) packed = pack_padded_sequence(f, lengths_of(mask).cpu(), batch_first=True, enforce_sorted=False) _, (h_n, _) = self.lstm(packed) return self.head(h_n[-1]) # --------------------------------------------------------------------------- # RU-LSTM # --------------------------------------------------------------------------- class _RULSTMBranch(nn.Module): def __init__(self, in_dim: int, hidden: int, future_steps: int, dropout: float): super().__init__() self.future_steps = future_steps self.rolling = nn.LSTM(in_dim, hidden, batch_first=True) self.unrolling = nn.LSTMCell(hidden, hidden) self.drop = nn.Dropout(dropout) def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: packed = pack_padded_sequence(x, lengths_of(mask).cpu(), batch_first=True, enforce_sorted=False) _, (h_n, c_n) = self.rolling(packed) h, c = h_n.squeeze(0), c_n.squeeze(0) inp = h for _ in range(self.future_steps): h, c = self.unrolling(inp, (h, c)) inp = h return self.drop(h) class RULSTMTriplet(_Base): def __init__(self, modality_dims, sizes, hidden: int = 128, future_steps: int = 8, dropout: float = 0.2, head_hidden: int = 256): super().__init__(modality_dims) self.branches = nn.ModuleDict({m: _RULSTMBranch(d, hidden, future_steps, dropout) for m, d in self.modality_dims.items()}) self.head = TripletHead(hidden, sizes, hidden=head_hidden, dropout=dropout) def forward(self, x, mask): feats = [self.branches[m](x[m], mask) for m in self.modalities] return self.head(torch.stack(feats, dim=0).mean(dim=0)) # --------------------------------------------------------------------------- # FUTR # --------------------------------------------------------------------------- class FUTRTriplet(_Base): def __init__(self, modality_dims, sizes, d_model: int = 128, n_heads: int = 4, n_layers: int = 3, dropout: float = 0.1, head_hidden: int = 256, max_T: int = 256): super().__init__(modality_dims) self.in_proj = nn.Linear(self.in_dim, d_model) self.pos = nn.Parameter(torch.zeros(1, max_T, d_model)) nn.init.trunc_normal_(self.pos, std=0.02) self.max_T = max_T self.encoder = _encoder(d_model, n_heads, n_layers, dropout) self.future_q = nn.Parameter(torch.zeros(1, 1, d_model)) nn.init.trunc_normal_(self.future_q, std=0.02) self.cross_attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout, batch_first=True) self.head = TripletHead(d_model, sizes, hidden=head_hidden, dropout=dropout) def forward(self, x, mask): f = _concat(x, self.modalities) B, T, _ = f.shape _check_T(T, self.max_T, "FUTR") h = self.encoder(self.in_proj(f) + self.pos[:, :T], src_key_padding_mask=~mask) out, _ = self.cross_attn(self.future_q.expand(B, -1, -1), h, h, key_padding_mask=~mask, need_weights=False) return self.head(out.squeeze(1)) # --------------------------------------------------------------------------- # AFFT # --------------------------------------------------------------------------- class _CausalBlock(nn.Module): """Pre-norm Transformer block with an additive attention mask.""" def __init__(self, d_model: int, n_heads: int, mlp_ratio: float = 4.0, dropout: float = 0.1): super().__init__() self.attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout, batch_first=True) self.norm1 = nn.LayerNorm(d_model) self.norm2 = nn.LayerNorm(d_model) hid = int(d_model * mlp_ratio) self.mlp = nn.Sequential(nn.Linear(d_model, hid), nn.GELU(), nn.Dropout(dropout), nn.Linear(hid, d_model), nn.Dropout(dropout)) def forward(self, x, attn_mask, key_padding_mask): h = self.norm1(x) h, _ = self.attn(h, h, h, attn_mask=attn_mask, key_padding_mask=key_padding_mask, need_weights=False) x = x + h return x + self.mlp(self.norm2(x)) class AFFTTriplet(_Base): def __init__(self, modality_dims, sizes, d_model: int = 96, n_heads: int = 4, n_layers: int = 3, dropout: float = 0.1, head_hidden: int = 256, max_T: int = 256): super().__init__(modality_dims) self.stems = nn.ModuleDict({m: nn.Linear(d, d_model) for m, d in self.modality_dims.items()}) self.mod_embed = nn.Parameter(torch.zeros(len(self.modalities), d_model)) nn.init.trunc_normal_(self.mod_embed, std=0.02) self.pos = nn.Parameter(torch.zeros(1, max_T, d_model)) nn.init.trunc_normal_(self.pos, std=0.02) self.max_T = max_T self.d_model = d_model self.blocks = nn.ModuleList([_CausalBlock(d_model, n_heads, dropout=dropout) for _ in range(n_layers)]) self.head = TripletHead(d_model, sizes, hidden=head_hidden, dropout=dropout) def forward(self, x, mask): B, T, _ = x[self.modalities[0]].shape _check_T(T, self.max_T, "AFFT") M = len(self.modalities) tok = torch.stack([self.stems[m](x[m]) + self.mod_embed[i] for i, m in enumerate(self.modalities)], dim=2) # (B, T, M, D) tok = (tok + self.pos[:, :T].unsqueeze(2)).reshape(B, T * M, self.d_model) ts = torch.arange(T, device=tok.device).repeat_interleave(M) attn_mask = ts[:, None] < ts[None, :] # True = future frame, masked kp = (~mask).unsqueeze(2).expand(-1, -1, M).reshape(B, T * M) for blk in self.blocks: tok = blk(tok, attn_mask, kp) last = (lengths_of(mask) - 1).clamp(min=0) # (B,) tok = tok.reshape(B, T, M, self.d_model) pooled = tok[torch.arange(B, device=tok.device), last].mean(dim=1) # (B, D) return self.head(pooled) # --------------------------------------------------------------------------- # HandFormer # --------------------------------------------------------------------------- class HandFormerTriplet(_Base): def __init__(self, modality_dims, sizes, d_model: int = 128, n_heads: int = 4, n_layers: int = 3, kernels=(3, 5, 9), dropout: float = 0.1, head_hidden: int = 256, max_T: int = 256): super().__init__(modality_dims) self.multi_conv = nn.ModuleList([nn.Conv1d(self.in_dim, d_model, k, padding=k // 2) for k in kernels]) self.conv_merge = nn.Conv1d(d_model * len(kernels), d_model, 1) self.pos = nn.Parameter(torch.zeros(1, max_T, d_model)) nn.init.trunc_normal_(self.pos, std=0.02) self.max_T = max_T self.encoder = _encoder(d_model, n_heads, n_layers, dropout) self.head = TripletHead(d_model, sizes, hidden=head_hidden, dropout=dropout) def forward(self, x, mask): f = _concat(x, self.modalities).transpose(1, 2) h = self.conv_merge(torch.cat([c(f) for c in self.multi_conv], dim=1)).transpose(1, 2) T = h.size(1) _check_T(T, self.max_T, "HandFormer") h = self.encoder(h + self.pos[:, :T], src_key_padding_mask=~mask) return self.head(masked_mean(h, mask)) # --------------------------------------------------------------------------- # ActionLLM surrogate # --------------------------------------------------------------------------- class ActionLLMSurrogate(_Base): def __init__(self, modality_dims, sizes, d_model: int = 192, n_heads: int = 6, n_layers: int = 2, dropout: float = 0.1, head_hidden: int = 256, max_T: int = 256): super().__init__(modality_dims) self.stem = nn.Sequential(nn.Conv1d(self.in_dim, d_model, 5, padding=2), nn.GELU(), nn.Conv1d(d_model, d_model, 5, padding=2)) self.pos = nn.Parameter(torch.zeros(1, max_T, d_model)) nn.init.trunc_normal_(self.pos, std=0.02) self.max_T = max_T self.encoder = _encoder(d_model, n_heads, n_layers, dropout) self.head = TripletHead(d_model, sizes, hidden=head_hidden, dropout=dropout) def forward(self, x, mask): h = self.stem(_concat(x, self.modalities).transpose(1, 2)).transpose(1, 2) T = h.size(1) _check_T(T, self.max_T, "ActionLLM") h = self.encoder(h + self.pos[:, :T], src_key_padding_mask=~mask) return self.head(masked_mean(h, mask)) # --------------------------------------------------------------------------- # DailyActFormer v2 (DAF, ours) # --------------------------------------------------------------------------- class ModalityStem(nn.Module): """Multi-scale 1-D conv stem (kernels 3/5/9 at 20 Hz) -> 1x1 merge -> LayerNorm.""" def __init__(self, in_dim: int, d_model: int, kernels=(3, 5, 9), dropout: float = 0.1): super().__init__() self.branches = nn.ModuleList([nn.Conv1d(in_dim, d_model, k, padding=k // 2) for k in kernels]) self.merge = nn.Sequential(nn.GELU(), nn.Conv1d(d_model * len(kernels), d_model, 1)) self.norm = nn.LayerNorm(d_model) self.drop = nn.Dropout(dropout) def forward(self, x: torch.Tensor) -> torch.Tensor: z = x.transpose(1, 2) h = self.merge(torch.cat([c(z) for c in self.branches], dim=1)).transpose(1, 2) return self.drop(self.norm(h)) class QueryPool(nn.Module): """One learnable query cross-attends to the valid frames.""" def __init__(self, d_model: int, n_heads: int = 4, dropout: float = 0.1): super().__init__() self.q = nn.Parameter(torch.zeros(1, 1, d_model)) nn.init.trunc_normal_(self.q, std=0.02) self.attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout, batch_first=True) self.norm = nn.LayerNorm(d_model) def forward(self, h: torch.Tensor, key_padding_mask: Optional[torch.Tensor]): out, _ = self.attn(self.q.expand(h.size(0), -1, -1), h, h, key_padding_mask=key_padding_mask, need_weights=False) return self.norm(out.squeeze(1)) class DailyActFormerV2(_Base): """DAF. Parameter names (``stems..``, ``fuse.``, ``fuse_norm.``, ``pos_embed``, ``encoder.``, ``pool.``, ``head.``) are those of the original implementation so encoder checkpoints (F5 S9 pre-training) transfer by name. Modality dropout: during training, with probability ``mod_dropout_p`` each modality's stem output is zeroed for the *whole batch* (one Bernoulli draw per modality per batch; at least one modality is kept), i.e. per-batch, not per-sample. """ def __init__(self, modality_dims, sizes, d_model: int = 128, n_layers: int = 3, n_heads: int = 4, dropout: float = 0.1, head_hidden: int = 256, max_T: int = 256, mod_dropout_p: float = 0.3, fusion_type: str = "concat", pool_type: str = "query", joint_head: bool = True): super().__init__(modality_dims) if fusion_type != "concat": raise ValueError("DAF supports fusion_type='concat' only (the configuration " "described in the paper)") if pool_type not in ("query", "mean"): raise ValueError(f"unknown pool_type {pool_type!r}") self.mod_dropout_p = float(mod_dropout_p) self.pool_type = pool_type self.stems = nn.ModuleDict({m: ModalityStem(d, d_model, dropout=dropout) for m, d in self.modality_dims.items()}) self.fuse = nn.Conv1d(d_model * len(self.modalities), d_model, 1) self.fuse_norm = nn.LayerNorm(d_model) self.pos_embed = nn.Parameter(torch.zeros(1, max_T, d_model)) nn.init.trunc_normal_(self.pos_embed, std=0.02) self.max_T = max_T self.encoder = _encoder(d_model, n_heads, n_layers, dropout) self.pool = QueryPool(d_model, n_heads=n_heads, dropout=dropout) head_cls = JointVNTripletHead if joint_head else TripletHead self.head = head_cls(d_model, sizes, hidden=head_hidden, dropout=dropout) def forward(self, x, mask): feats = [self.stems[m](x[m]) for m in self.modalities] M = len(feats) if self.training and self.mod_dropout_p > 0 and M > 1: keep = torch.rand(M, device=feats[0].device) >= self.mod_dropout_p if not bool(keep.any()): keep = torch.zeros_like(keep) keep[torch.randint(0, M, (1,), device=feats[0].device)] = True feats = [f if bool(k) else torch.zeros_like(f) for f, k in zip(feats, keep)] h = self.fuse(torch.cat(feats, dim=-1).transpose(1, 2)).transpose(1, 2) h = self.fuse_norm(h) T = h.size(1) _check_T(T, self.max_T, "DailyActFormerV2") h = self.encoder(h + self.pos_embed[:, :T], src_key_padding_mask=~mask) pooled = masked_mean(h, mask) if self.pool_type == "mean" else self.pool(h, ~mask) return self.head(pooled)