Download pulse/models/t2/seqpred.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 18.6 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/pulse/models/t2/seqpred.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/pulse/models/t2/seqpred.py
-
curl -L -o seqpred.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/pulse/models/t2/seqpred.py
18.6 kB
| """T2 sequence classifiers (fine-grained action recognition). | |
| Every model takes ``x: {modality: (B, T, F_m)}`` and ``mask: (B, T)`` (True = | |
| valid frame) and returns ``{"verb_fine", "verb_composite", "noun", "hand"}`` | |
| logits. Inputs are 20 Hz windows (4 s sliding windows, or annotated segments | |
| of at most ``max_T`` frames). Modalities are always consumed in the order of | |
| ``modality_dims`` (the config order), never in dict-iteration order of ``x``. | |
| Models (sensor adaptations of the published methods; each keeps the method's | |
| key idea and uses the shared :class:`~pulse.models.t2.heads.TripletHead`): | |
| * ``deepconvlstm`` DeepConvLSTM (Ordonez & Roggen 2016): 4 conv + 2 LSTM layers on | |
| the concatenated modalities. | |
| * ``rulstm`` RU-LSTM (Furnari & Farinella 2020): per-modality rolling LSTM + | |
| unrolling LSTMCell, late (mean) fusion. | |
| * ``futr`` FUTR (Gong et al. 2022): Transformer encoder + one learnable | |
| decoder query. | |
| * ``afft`` AFFT (Zhong et al. 2023): one token per frame and modality, | |
| causal attention, read-out at the last valid frame. | |
| * ``handformer`` HandFormer (Shamil et al. 2024): multi-scale temporal conv + | |
| Transformer, mean pool. | |
| * ``actionllm`` ActionLLM *surrogate*: conv stem + 2-layer Transformer trained from | |
| scratch (no language model). | |
| * ``dailyactformer_v2`` DailyActFormer (DAF, ours): per-modality multi-scale conv stems, | |
| whole-modality dropout, concat + 1x1 fusion, Transformer, | |
| learnable-query pooling, joint verb-noun head. | |
| Differences from the pre-camera-ready code (``experiments/models_seqpred.py``), | |
| all of which only matter for variable-length (segment-aligned) batches: the | |
| LSTMs of DeepConvLSTM and RU-LSTM read packed sequences, so the final state is | |
| that of the last valid frame (previously the last padded frame); AFFT reads | |
| out the tokens of the last valid frame (previously the last padded frame). | |
| Head sizes come from the taxonomy instead of module constants. | |
| """ | |
| from __future__ import annotations | |
| from typing import Dict, List, Mapping, Optional | |
| import torch | |
| import torch.nn as nn | |
| from torch.nn.utils.rnn import pack_padded_sequence | |
| from .heads import JointVNTripletHead, TripletHead | |
| def masked_mean(h: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: | |
| m = mask.to(h.dtype).unsqueeze(-1) | |
| return (h * m).sum(dim=1) / m.sum(dim=1).clamp(min=1.0) | |
| def lengths_of(mask: torch.Tensor) -> torch.Tensor: | |
| return mask.sum(dim=1).clamp(min=1).to(torch.int64) | |
| def _concat(x: Mapping[str, torch.Tensor], order: List[str]) -> torch.Tensor: | |
| missing = [m for m in order if m not in x] | |
| if missing: | |
| raise KeyError(f"model expects modalities {order}, input lacks {missing}") | |
| return torch.cat([x[m] for m in order], dim=-1) | |
| def _encoder(d_model: int, n_heads: int, n_layers: int, dropout: float) -> nn.TransformerEncoder: | |
| layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=n_heads, dim_feedforward=4 * d_model, | |
| dropout=dropout, batch_first=True, activation="gelu") | |
| return nn.TransformerEncoder(layer, num_layers=n_layers, enable_nested_tensor=False) | |
| def _check_T(T: int, max_T: int, name: str) -> None: | |
| if T > max_T: | |
| raise ValueError(f"{name}: input has T={T} frames > max_T={max_T}") | |
| class _Base(nn.Module): | |
| def __init__(self, modality_dims: Mapping[str, int]): | |
| super().__init__() | |
| if not modality_dims: | |
| raise ValueError("a T2 model needs at least one modality") | |
| self.modality_dims = {m: int(d) for m, d in modality_dims.items()} | |
| self.modalities = list(self.modality_dims) | |
| self.in_dim = sum(self.modality_dims.values()) | |
| # --------------------------------------------------------------------------- | |
| # DeepConvLSTM | |
| # --------------------------------------------------------------------------- | |
| class DeepConvLSTMTriplet(_Base): | |
| def __init__(self, modality_dims, sizes, conv_filters: int = 64, conv_kernel: int = 5, | |
| num_conv_layers: int = 4, lstm_hidden: int = 128, num_lstm_layers: int = 2, | |
| dropout: float = 0.2, head_hidden: int = 256): | |
| super().__init__(modality_dims) | |
| convs: List[nn.Module] = [] | |
| c = self.in_dim | |
| for i in range(num_conv_layers): | |
| convs.append(nn.Sequential( | |
| nn.Conv1d(c, conv_filters, conv_kernel, padding=conv_kernel // 2), | |
| nn.BatchNorm1d(conv_filters), nn.ReLU(), | |
| nn.Dropout(dropout if i < num_conv_layers - 1 else dropout + 0.1))) | |
| c = conv_filters | |
| self.convs = nn.Sequential(*convs) | |
| self.lstm = nn.LSTM(conv_filters, lstm_hidden, num_layers=num_lstm_layers, batch_first=True, | |
| dropout=dropout if num_lstm_layers > 1 else 0.0) | |
| self.head = TripletHead(lstm_hidden, sizes, hidden=head_hidden, dropout=dropout) | |
| def forward(self, x, mask): | |
| f = self.convs(_concat(x, self.modalities).transpose(1, 2)).transpose(1, 2) | |
| packed = pack_padded_sequence(f, lengths_of(mask).cpu(), batch_first=True, | |
| enforce_sorted=False) | |
| _, (h_n, _) = self.lstm(packed) | |
| return self.head(h_n[-1]) | |
| # --------------------------------------------------------------------------- | |
| # RU-LSTM | |
| # --------------------------------------------------------------------------- | |
| class _RULSTMBranch(nn.Module): | |
| def __init__(self, in_dim: int, hidden: int, future_steps: int, dropout: float): | |
| super().__init__() | |
| self.future_steps = future_steps | |
| self.rolling = nn.LSTM(in_dim, hidden, batch_first=True) | |
| self.unrolling = nn.LSTMCell(hidden, hidden) | |
| self.drop = nn.Dropout(dropout) | |
| def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: | |
| packed = pack_padded_sequence(x, lengths_of(mask).cpu(), batch_first=True, | |
| enforce_sorted=False) | |
| _, (h_n, c_n) = self.rolling(packed) | |
| h, c = h_n.squeeze(0), c_n.squeeze(0) | |
| inp = h | |
| for _ in range(self.future_steps): | |
| h, c = self.unrolling(inp, (h, c)) | |
| inp = h | |
| return self.drop(h) | |
| class RULSTMTriplet(_Base): | |
| def __init__(self, modality_dims, sizes, hidden: int = 128, future_steps: int = 8, | |
| dropout: float = 0.2, head_hidden: int = 256): | |
| super().__init__(modality_dims) | |
| self.branches = nn.ModuleDict({m: _RULSTMBranch(d, hidden, future_steps, dropout) | |
| for m, d in self.modality_dims.items()}) | |
| self.head = TripletHead(hidden, sizes, hidden=head_hidden, dropout=dropout) | |
| def forward(self, x, mask): | |
| feats = [self.branches[m](x[m], mask) for m in self.modalities] | |
| return self.head(torch.stack(feats, dim=0).mean(dim=0)) | |
| # --------------------------------------------------------------------------- | |
| # FUTR | |
| # --------------------------------------------------------------------------- | |
| class FUTRTriplet(_Base): | |
| def __init__(self, modality_dims, sizes, d_model: int = 128, n_heads: int = 4, | |
| n_layers: int = 3, dropout: float = 0.1, head_hidden: int = 256, | |
| max_T: int = 256): | |
| super().__init__(modality_dims) | |
| self.in_proj = nn.Linear(self.in_dim, d_model) | |
| self.pos = nn.Parameter(torch.zeros(1, max_T, d_model)) | |
| nn.init.trunc_normal_(self.pos, std=0.02) | |
| self.max_T = max_T | |
| self.encoder = _encoder(d_model, n_heads, n_layers, dropout) | |
| self.future_q = nn.Parameter(torch.zeros(1, 1, d_model)) | |
| nn.init.trunc_normal_(self.future_q, std=0.02) | |
| self.cross_attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout, batch_first=True) | |
| self.head = TripletHead(d_model, sizes, hidden=head_hidden, dropout=dropout) | |
| def forward(self, x, mask): | |
| f = _concat(x, self.modalities) | |
| B, T, _ = f.shape | |
| _check_T(T, self.max_T, "FUTR") | |
| h = self.encoder(self.in_proj(f) + self.pos[:, :T], src_key_padding_mask=~mask) | |
| out, _ = self.cross_attn(self.future_q.expand(B, -1, -1), h, h, key_padding_mask=~mask, | |
| need_weights=False) | |
| return self.head(out.squeeze(1)) | |
| # --------------------------------------------------------------------------- | |
| # AFFT | |
| # --------------------------------------------------------------------------- | |
| class _CausalBlock(nn.Module): | |
| """Pre-norm Transformer block with an additive attention mask.""" | |
| def __init__(self, d_model: int, n_heads: int, mlp_ratio: float = 4.0, dropout: float = 0.1): | |
| super().__init__() | |
| self.attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout, batch_first=True) | |
| self.norm1 = nn.LayerNorm(d_model) | |
| self.norm2 = nn.LayerNorm(d_model) | |
| hid = int(d_model * mlp_ratio) | |
| self.mlp = nn.Sequential(nn.Linear(d_model, hid), nn.GELU(), nn.Dropout(dropout), | |
| nn.Linear(hid, d_model), nn.Dropout(dropout)) | |
| def forward(self, x, attn_mask, key_padding_mask): | |
| h = self.norm1(x) | |
| h, _ = self.attn(h, h, h, attn_mask=attn_mask, key_padding_mask=key_padding_mask, | |
| need_weights=False) | |
| x = x + h | |
| return x + self.mlp(self.norm2(x)) | |
| class AFFTTriplet(_Base): | |
| def __init__(self, modality_dims, sizes, d_model: int = 96, n_heads: int = 4, | |
| n_layers: int = 3, dropout: float = 0.1, head_hidden: int = 256, | |
| max_T: int = 256): | |
| super().__init__(modality_dims) | |
| self.stems = nn.ModuleDict({m: nn.Linear(d, d_model) for m, d in self.modality_dims.items()}) | |
| self.mod_embed = nn.Parameter(torch.zeros(len(self.modalities), d_model)) | |
| nn.init.trunc_normal_(self.mod_embed, std=0.02) | |
| self.pos = nn.Parameter(torch.zeros(1, max_T, d_model)) | |
| nn.init.trunc_normal_(self.pos, std=0.02) | |
| self.max_T = max_T | |
| self.d_model = d_model | |
| self.blocks = nn.ModuleList([_CausalBlock(d_model, n_heads, dropout=dropout) | |
| for _ in range(n_layers)]) | |
| self.head = TripletHead(d_model, sizes, hidden=head_hidden, dropout=dropout) | |
| def forward(self, x, mask): | |
| B, T, _ = x[self.modalities[0]].shape | |
| _check_T(T, self.max_T, "AFFT") | |
| M = len(self.modalities) | |
| tok = torch.stack([self.stems[m](x[m]) + self.mod_embed[i] | |
| for i, m in enumerate(self.modalities)], dim=2) # (B, T, M, D) | |
| tok = (tok + self.pos[:, :T].unsqueeze(2)).reshape(B, T * M, self.d_model) | |
| ts = torch.arange(T, device=tok.device).repeat_interleave(M) | |
| attn_mask = ts[:, None] < ts[None, :] # True = future frame, masked | |
| kp = (~mask).unsqueeze(2).expand(-1, -1, M).reshape(B, T * M) | |
| for blk in self.blocks: | |
| tok = blk(tok, attn_mask, kp) | |
| last = (lengths_of(mask) - 1).clamp(min=0) # (B,) | |
| tok = tok.reshape(B, T, M, self.d_model) | |
| pooled = tok[torch.arange(B, device=tok.device), last].mean(dim=1) # (B, D) | |
| return self.head(pooled) | |
| # --------------------------------------------------------------------------- | |
| # HandFormer | |
| # --------------------------------------------------------------------------- | |
| class HandFormerTriplet(_Base): | |
| def __init__(self, modality_dims, sizes, d_model: int = 128, n_heads: int = 4, | |
| n_layers: int = 3, kernels=(3, 5, 9), dropout: float = 0.1, | |
| head_hidden: int = 256, max_T: int = 256): | |
| super().__init__(modality_dims) | |
| self.multi_conv = nn.ModuleList([nn.Conv1d(self.in_dim, d_model, k, padding=k // 2) | |
| for k in kernels]) | |
| self.conv_merge = nn.Conv1d(d_model * len(kernels), d_model, 1) | |
| self.pos = nn.Parameter(torch.zeros(1, max_T, d_model)) | |
| nn.init.trunc_normal_(self.pos, std=0.02) | |
| self.max_T = max_T | |
| self.encoder = _encoder(d_model, n_heads, n_layers, dropout) | |
| self.head = TripletHead(d_model, sizes, hidden=head_hidden, dropout=dropout) | |
| def forward(self, x, mask): | |
| f = _concat(x, self.modalities).transpose(1, 2) | |
| h = self.conv_merge(torch.cat([c(f) for c in self.multi_conv], dim=1)).transpose(1, 2) | |
| T = h.size(1) | |
| _check_T(T, self.max_T, "HandFormer") | |
| h = self.encoder(h + self.pos[:, :T], src_key_padding_mask=~mask) | |
| return self.head(masked_mean(h, mask)) | |
| # --------------------------------------------------------------------------- | |
| # ActionLLM surrogate | |
| # --------------------------------------------------------------------------- | |
| class ActionLLMSurrogate(_Base): | |
| def __init__(self, modality_dims, sizes, d_model: int = 192, n_heads: int = 6, | |
| n_layers: int = 2, dropout: float = 0.1, head_hidden: int = 256, | |
| max_T: int = 256): | |
| super().__init__(modality_dims) | |
| self.stem = nn.Sequential(nn.Conv1d(self.in_dim, d_model, 5, padding=2), nn.GELU(), | |
| nn.Conv1d(d_model, d_model, 5, padding=2)) | |
| self.pos = nn.Parameter(torch.zeros(1, max_T, d_model)) | |
| nn.init.trunc_normal_(self.pos, std=0.02) | |
| self.max_T = max_T | |
| self.encoder = _encoder(d_model, n_heads, n_layers, dropout) | |
| self.head = TripletHead(d_model, sizes, hidden=head_hidden, dropout=dropout) | |
| def forward(self, x, mask): | |
| h = self.stem(_concat(x, self.modalities).transpose(1, 2)).transpose(1, 2) | |
| T = h.size(1) | |
| _check_T(T, self.max_T, "ActionLLM") | |
| h = self.encoder(h + self.pos[:, :T], src_key_padding_mask=~mask) | |
| return self.head(masked_mean(h, mask)) | |
| # --------------------------------------------------------------------------- | |
| # DailyActFormer v2 (DAF, ours) | |
| # --------------------------------------------------------------------------- | |
| class ModalityStem(nn.Module): | |
| """Multi-scale 1-D conv stem (kernels 3/5/9 at 20 Hz) -> 1x1 merge -> LayerNorm.""" | |
| def __init__(self, in_dim: int, d_model: int, kernels=(3, 5, 9), dropout: float = 0.1): | |
| super().__init__() | |
| self.branches = nn.ModuleList([nn.Conv1d(in_dim, d_model, k, padding=k // 2) | |
| for k in kernels]) | |
| self.merge = nn.Sequential(nn.GELU(), nn.Conv1d(d_model * len(kernels), d_model, 1)) | |
| self.norm = nn.LayerNorm(d_model) | |
| self.drop = nn.Dropout(dropout) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| z = x.transpose(1, 2) | |
| h = self.merge(torch.cat([c(z) for c in self.branches], dim=1)).transpose(1, 2) | |
| return self.drop(self.norm(h)) | |
| class QueryPool(nn.Module): | |
| """One learnable query cross-attends to the valid frames.""" | |
| def __init__(self, d_model: int, n_heads: int = 4, dropout: float = 0.1): | |
| super().__init__() | |
| self.q = nn.Parameter(torch.zeros(1, 1, d_model)) | |
| nn.init.trunc_normal_(self.q, std=0.02) | |
| self.attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout, batch_first=True) | |
| self.norm = nn.LayerNorm(d_model) | |
| def forward(self, h: torch.Tensor, key_padding_mask: Optional[torch.Tensor]): | |
| out, _ = self.attn(self.q.expand(h.size(0), -1, -1), h, h, | |
| key_padding_mask=key_padding_mask, need_weights=False) | |
| return self.norm(out.squeeze(1)) | |
| class DailyActFormerV2(_Base): | |
| """DAF. Parameter names (``stems.<m>.``, ``fuse.``, ``fuse_norm.``, ``pos_embed``, | |
| ``encoder.``, ``pool.``, ``head.``) are those of the original implementation so | |
| encoder checkpoints (F5 S9 pre-training) transfer by name. | |
| Modality dropout: during training, with probability ``mod_dropout_p`` each | |
| modality's stem output is zeroed for the *whole batch* (one Bernoulli draw | |
| per modality per batch; at least one modality is kept), i.e. per-batch, | |
| not per-sample. | |
| """ | |
| def __init__(self, modality_dims, sizes, d_model: int = 128, n_layers: int = 3, | |
| n_heads: int = 4, dropout: float = 0.1, head_hidden: int = 256, | |
| max_T: int = 256, mod_dropout_p: float = 0.3, fusion_type: str = "concat", | |
| pool_type: str = "query", joint_head: bool = True): | |
| super().__init__(modality_dims) | |
| if fusion_type != "concat": | |
| raise ValueError("DAF supports fusion_type='concat' only (the configuration " | |
| "described in the paper)") | |
| if pool_type not in ("query", "mean"): | |
| raise ValueError(f"unknown pool_type {pool_type!r}") | |
| self.mod_dropout_p = float(mod_dropout_p) | |
| self.pool_type = pool_type | |
| self.stems = nn.ModuleDict({m: ModalityStem(d, d_model, dropout=dropout) | |
| for m, d in self.modality_dims.items()}) | |
| self.fuse = nn.Conv1d(d_model * len(self.modalities), d_model, 1) | |
| self.fuse_norm = nn.LayerNorm(d_model) | |
| self.pos_embed = nn.Parameter(torch.zeros(1, max_T, d_model)) | |
| nn.init.trunc_normal_(self.pos_embed, std=0.02) | |
| self.max_T = max_T | |
| self.encoder = _encoder(d_model, n_heads, n_layers, dropout) | |
| self.pool = QueryPool(d_model, n_heads=n_heads, dropout=dropout) | |
| head_cls = JointVNTripletHead if joint_head else TripletHead | |
| self.head = head_cls(d_model, sizes, hidden=head_hidden, dropout=dropout) | |
| def forward(self, x, mask): | |
| feats = [self.stems[m](x[m]) for m in self.modalities] | |
| M = len(feats) | |
| if self.training and self.mod_dropout_p > 0 and M > 1: | |
| keep = torch.rand(M, device=feats[0].device) >= self.mod_dropout_p | |
| if not bool(keep.any()): | |
| keep = torch.zeros_like(keep) | |
| keep[torch.randint(0, M, (1,), device=feats[0].device)] = True | |
| feats = [f if bool(k) else torch.zeros_like(f) for f, k in zip(feats, keep)] | |
| h = self.fuse(torch.cat(feats, dim=-1).transpose(1, 2)).transpose(1, 2) | |
| h = self.fuse_norm(h) | |
| T = h.size(1) | |
| _check_T(T, self.max_T, "DailyActFormerV2") | |
| h = self.encoder(h + self.pos_embed[:, :T], src_key_padding_mask=~mask) | |
| pooled = masked_mean(h, mask) if self.pool_type == "mean" else self.pool(h, ~mask) | |
| return self.head(pooled) | |