PULSE-code / pulse /models /t2 /seqpred.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
18.6 kB
"""T2 sequence classifiers (fine-grained action recognition).
Every model takes ``x: {modality: (B, T, F_m)}`` and ``mask: (B, T)`` (True =
valid frame) and returns ``{"verb_fine", "verb_composite", "noun", "hand"}``
logits. Inputs are 20 Hz windows (4 s sliding windows, or annotated segments
of at most ``max_T`` frames). Modalities are always consumed in the order of
``modality_dims`` (the config order), never in dict-iteration order of ``x``.
Models (sensor adaptations of the published methods; each keeps the method's
key idea and uses the shared :class:`~pulse.models.t2.heads.TripletHead`):
* ``deepconvlstm`` DeepConvLSTM (Ordonez & Roggen 2016): 4 conv + 2 LSTM layers on
the concatenated modalities.
* ``rulstm`` RU-LSTM (Furnari & Farinella 2020): per-modality rolling LSTM +
unrolling LSTMCell, late (mean) fusion.
* ``futr`` FUTR (Gong et al. 2022): Transformer encoder + one learnable
decoder query.
* ``afft`` AFFT (Zhong et al. 2023): one token per frame and modality,
causal attention, read-out at the last valid frame.
* ``handformer`` HandFormer (Shamil et al. 2024): multi-scale temporal conv +
Transformer, mean pool.
* ``actionllm`` ActionLLM *surrogate*: conv stem + 2-layer Transformer trained from
scratch (no language model).
* ``dailyactformer_v2`` DailyActFormer (DAF, ours): per-modality multi-scale conv stems,
whole-modality dropout, concat + 1x1 fusion, Transformer,
learnable-query pooling, joint verb-noun head.
Differences from the pre-camera-ready code (``experiments/models_seqpred.py``),
all of which only matter for variable-length (segment-aligned) batches: the
LSTMs of DeepConvLSTM and RU-LSTM read packed sequences, so the final state is
that of the last valid frame (previously the last padded frame); AFFT reads
out the tokens of the last valid frame (previously the last padded frame).
Head sizes come from the taxonomy instead of module constants.
"""
from __future__ import annotations
from typing import Dict, List, Mapping, Optional
import torch
import torch.nn as nn
from torch.nn.utils.rnn import pack_padded_sequence
from .heads import JointVNTripletHead, TripletHead
def masked_mean(h: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
m = mask.to(h.dtype).unsqueeze(-1)
return (h * m).sum(dim=1) / m.sum(dim=1).clamp(min=1.0)
def lengths_of(mask: torch.Tensor) -> torch.Tensor:
return mask.sum(dim=1).clamp(min=1).to(torch.int64)
def _concat(x: Mapping[str, torch.Tensor], order: List[str]) -> torch.Tensor:
missing = [m for m in order if m not in x]
if missing:
raise KeyError(f"model expects modalities {order}, input lacks {missing}")
return torch.cat([x[m] for m in order], dim=-1)
def _encoder(d_model: int, n_heads: int, n_layers: int, dropout: float) -> nn.TransformerEncoder:
layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=n_heads, dim_feedforward=4 * d_model,
dropout=dropout, batch_first=True, activation="gelu")
return nn.TransformerEncoder(layer, num_layers=n_layers, enable_nested_tensor=False)
def _check_T(T: int, max_T: int, name: str) -> None:
if T > max_T:
raise ValueError(f"{name}: input has T={T} frames > max_T={max_T}")
class _Base(nn.Module):
def __init__(self, modality_dims: Mapping[str, int]):
super().__init__()
if not modality_dims:
raise ValueError("a T2 model needs at least one modality")
self.modality_dims = {m: int(d) for m, d in modality_dims.items()}
self.modalities = list(self.modality_dims)
self.in_dim = sum(self.modality_dims.values())
# ---------------------------------------------------------------------------
# DeepConvLSTM
# ---------------------------------------------------------------------------
class DeepConvLSTMTriplet(_Base):
def __init__(self, modality_dims, sizes, conv_filters: int = 64, conv_kernel: int = 5,
num_conv_layers: int = 4, lstm_hidden: int = 128, num_lstm_layers: int = 2,
dropout: float = 0.2, head_hidden: int = 256):
super().__init__(modality_dims)
convs: List[nn.Module] = []
c = self.in_dim
for i in range(num_conv_layers):
convs.append(nn.Sequential(
nn.Conv1d(c, conv_filters, conv_kernel, padding=conv_kernel // 2),
nn.BatchNorm1d(conv_filters), nn.ReLU(),
nn.Dropout(dropout if i < num_conv_layers - 1 else dropout + 0.1)))
c = conv_filters
self.convs = nn.Sequential(*convs)
self.lstm = nn.LSTM(conv_filters, lstm_hidden, num_layers=num_lstm_layers, batch_first=True,
dropout=dropout if num_lstm_layers > 1 else 0.0)
self.head = TripletHead(lstm_hidden, sizes, hidden=head_hidden, dropout=dropout)
def forward(self, x, mask):
f = self.convs(_concat(x, self.modalities).transpose(1, 2)).transpose(1, 2)
packed = pack_padded_sequence(f, lengths_of(mask).cpu(), batch_first=True,
enforce_sorted=False)
_, (h_n, _) = self.lstm(packed)
return self.head(h_n[-1])
# ---------------------------------------------------------------------------
# RU-LSTM
# ---------------------------------------------------------------------------
class _RULSTMBranch(nn.Module):
def __init__(self, in_dim: int, hidden: int, future_steps: int, dropout: float):
super().__init__()
self.future_steps = future_steps
self.rolling = nn.LSTM(in_dim, hidden, batch_first=True)
self.unrolling = nn.LSTMCell(hidden, hidden)
self.drop = nn.Dropout(dropout)
def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
packed = pack_padded_sequence(x, lengths_of(mask).cpu(), batch_first=True,
enforce_sorted=False)
_, (h_n, c_n) = self.rolling(packed)
h, c = h_n.squeeze(0), c_n.squeeze(0)
inp = h
for _ in range(self.future_steps):
h, c = self.unrolling(inp, (h, c))
inp = h
return self.drop(h)
class RULSTMTriplet(_Base):
def __init__(self, modality_dims, sizes, hidden: int = 128, future_steps: int = 8,
dropout: float = 0.2, head_hidden: int = 256):
super().__init__(modality_dims)
self.branches = nn.ModuleDict({m: _RULSTMBranch(d, hidden, future_steps, dropout)
for m, d in self.modality_dims.items()})
self.head = TripletHead(hidden, sizes, hidden=head_hidden, dropout=dropout)
def forward(self, x, mask):
feats = [self.branches[m](x[m], mask) for m in self.modalities]
return self.head(torch.stack(feats, dim=0).mean(dim=0))
# ---------------------------------------------------------------------------
# FUTR
# ---------------------------------------------------------------------------
class FUTRTriplet(_Base):
def __init__(self, modality_dims, sizes, d_model: int = 128, n_heads: int = 4,
n_layers: int = 3, dropout: float = 0.1, head_hidden: int = 256,
max_T: int = 256):
super().__init__(modality_dims)
self.in_proj = nn.Linear(self.in_dim, d_model)
self.pos = nn.Parameter(torch.zeros(1, max_T, d_model))
nn.init.trunc_normal_(self.pos, std=0.02)
self.max_T = max_T
self.encoder = _encoder(d_model, n_heads, n_layers, dropout)
self.future_q = nn.Parameter(torch.zeros(1, 1, d_model))
nn.init.trunc_normal_(self.future_q, std=0.02)
self.cross_attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout, batch_first=True)
self.head = TripletHead(d_model, sizes, hidden=head_hidden, dropout=dropout)
def forward(self, x, mask):
f = _concat(x, self.modalities)
B, T, _ = f.shape
_check_T(T, self.max_T, "FUTR")
h = self.encoder(self.in_proj(f) + self.pos[:, :T], src_key_padding_mask=~mask)
out, _ = self.cross_attn(self.future_q.expand(B, -1, -1), h, h, key_padding_mask=~mask,
need_weights=False)
return self.head(out.squeeze(1))
# ---------------------------------------------------------------------------
# AFFT
# ---------------------------------------------------------------------------
class _CausalBlock(nn.Module):
"""Pre-norm Transformer block with an additive attention mask."""
def __init__(self, d_model: int, n_heads: int, mlp_ratio: float = 4.0, dropout: float = 0.1):
super().__init__()
self.attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout, batch_first=True)
self.norm1 = nn.LayerNorm(d_model)
self.norm2 = nn.LayerNorm(d_model)
hid = int(d_model * mlp_ratio)
self.mlp = nn.Sequential(nn.Linear(d_model, hid), nn.GELU(), nn.Dropout(dropout),
nn.Linear(hid, d_model), nn.Dropout(dropout))
def forward(self, x, attn_mask, key_padding_mask):
h = self.norm1(x)
h, _ = self.attn(h, h, h, attn_mask=attn_mask, key_padding_mask=key_padding_mask,
need_weights=False)
x = x + h
return x + self.mlp(self.norm2(x))
class AFFTTriplet(_Base):
def __init__(self, modality_dims, sizes, d_model: int = 96, n_heads: int = 4,
n_layers: int = 3, dropout: float = 0.1, head_hidden: int = 256,
max_T: int = 256):
super().__init__(modality_dims)
self.stems = nn.ModuleDict({m: nn.Linear(d, d_model) for m, d in self.modality_dims.items()})
self.mod_embed = nn.Parameter(torch.zeros(len(self.modalities), d_model))
nn.init.trunc_normal_(self.mod_embed, std=0.02)
self.pos = nn.Parameter(torch.zeros(1, max_T, d_model))
nn.init.trunc_normal_(self.pos, std=0.02)
self.max_T = max_T
self.d_model = d_model
self.blocks = nn.ModuleList([_CausalBlock(d_model, n_heads, dropout=dropout)
for _ in range(n_layers)])
self.head = TripletHead(d_model, sizes, hidden=head_hidden, dropout=dropout)
def forward(self, x, mask):
B, T, _ = x[self.modalities[0]].shape
_check_T(T, self.max_T, "AFFT")
M = len(self.modalities)
tok = torch.stack([self.stems[m](x[m]) + self.mod_embed[i]
for i, m in enumerate(self.modalities)], dim=2) # (B, T, M, D)
tok = (tok + self.pos[:, :T].unsqueeze(2)).reshape(B, T * M, self.d_model)
ts = torch.arange(T, device=tok.device).repeat_interleave(M)
attn_mask = ts[:, None] < ts[None, :] # True = future frame, masked
kp = (~mask).unsqueeze(2).expand(-1, -1, M).reshape(B, T * M)
for blk in self.blocks:
tok = blk(tok, attn_mask, kp)
last = (lengths_of(mask) - 1).clamp(min=0) # (B,)
tok = tok.reshape(B, T, M, self.d_model)
pooled = tok[torch.arange(B, device=tok.device), last].mean(dim=1) # (B, D)
return self.head(pooled)
# ---------------------------------------------------------------------------
# HandFormer
# ---------------------------------------------------------------------------
class HandFormerTriplet(_Base):
def __init__(self, modality_dims, sizes, d_model: int = 128, n_heads: int = 4,
n_layers: int = 3, kernels=(3, 5, 9), dropout: float = 0.1,
head_hidden: int = 256, max_T: int = 256):
super().__init__(modality_dims)
self.multi_conv = nn.ModuleList([nn.Conv1d(self.in_dim, d_model, k, padding=k // 2)
for k in kernels])
self.conv_merge = nn.Conv1d(d_model * len(kernels), d_model, 1)
self.pos = nn.Parameter(torch.zeros(1, max_T, d_model))
nn.init.trunc_normal_(self.pos, std=0.02)
self.max_T = max_T
self.encoder = _encoder(d_model, n_heads, n_layers, dropout)
self.head = TripletHead(d_model, sizes, hidden=head_hidden, dropout=dropout)
def forward(self, x, mask):
f = _concat(x, self.modalities).transpose(1, 2)
h = self.conv_merge(torch.cat([c(f) for c in self.multi_conv], dim=1)).transpose(1, 2)
T = h.size(1)
_check_T(T, self.max_T, "HandFormer")
h = self.encoder(h + self.pos[:, :T], src_key_padding_mask=~mask)
return self.head(masked_mean(h, mask))
# ---------------------------------------------------------------------------
# ActionLLM surrogate
# ---------------------------------------------------------------------------
class ActionLLMSurrogate(_Base):
def __init__(self, modality_dims, sizes, d_model: int = 192, n_heads: int = 6,
n_layers: int = 2, dropout: float = 0.1, head_hidden: int = 256,
max_T: int = 256):
super().__init__(modality_dims)
self.stem = nn.Sequential(nn.Conv1d(self.in_dim, d_model, 5, padding=2), nn.GELU(),
nn.Conv1d(d_model, d_model, 5, padding=2))
self.pos = nn.Parameter(torch.zeros(1, max_T, d_model))
nn.init.trunc_normal_(self.pos, std=0.02)
self.max_T = max_T
self.encoder = _encoder(d_model, n_heads, n_layers, dropout)
self.head = TripletHead(d_model, sizes, hidden=head_hidden, dropout=dropout)
def forward(self, x, mask):
h = self.stem(_concat(x, self.modalities).transpose(1, 2)).transpose(1, 2)
T = h.size(1)
_check_T(T, self.max_T, "ActionLLM")
h = self.encoder(h + self.pos[:, :T], src_key_padding_mask=~mask)
return self.head(masked_mean(h, mask))
# ---------------------------------------------------------------------------
# DailyActFormer v2 (DAF, ours)
# ---------------------------------------------------------------------------
class ModalityStem(nn.Module):
"""Multi-scale 1-D conv stem (kernels 3/5/9 at 20 Hz) -> 1x1 merge -> LayerNorm."""
def __init__(self, in_dim: int, d_model: int, kernels=(3, 5, 9), dropout: float = 0.1):
super().__init__()
self.branches = nn.ModuleList([nn.Conv1d(in_dim, d_model, k, padding=k // 2)
for k in kernels])
self.merge = nn.Sequential(nn.GELU(), nn.Conv1d(d_model * len(kernels), d_model, 1))
self.norm = nn.LayerNorm(d_model)
self.drop = nn.Dropout(dropout)
def forward(self, x: torch.Tensor) -> torch.Tensor:
z = x.transpose(1, 2)
h = self.merge(torch.cat([c(z) for c in self.branches], dim=1)).transpose(1, 2)
return self.drop(self.norm(h))
class QueryPool(nn.Module):
"""One learnable query cross-attends to the valid frames."""
def __init__(self, d_model: int, n_heads: int = 4, dropout: float = 0.1):
super().__init__()
self.q = nn.Parameter(torch.zeros(1, 1, d_model))
nn.init.trunc_normal_(self.q, std=0.02)
self.attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout, batch_first=True)
self.norm = nn.LayerNorm(d_model)
def forward(self, h: torch.Tensor, key_padding_mask: Optional[torch.Tensor]):
out, _ = self.attn(self.q.expand(h.size(0), -1, -1), h, h,
key_padding_mask=key_padding_mask, need_weights=False)
return self.norm(out.squeeze(1))
class DailyActFormerV2(_Base):
"""DAF. Parameter names (``stems.<m>.``, ``fuse.``, ``fuse_norm.``, ``pos_embed``,
``encoder.``, ``pool.``, ``head.``) are those of the original implementation so
encoder checkpoints (F5 S9 pre-training) transfer by name.
Modality dropout: during training, with probability ``mod_dropout_p`` each
modality's stem output is zeroed for the *whole batch* (one Bernoulli draw
per modality per batch; at least one modality is kept), i.e. per-batch,
not per-sample.
"""
def __init__(self, modality_dims, sizes, d_model: int = 128, n_layers: int = 3,
n_heads: int = 4, dropout: float = 0.1, head_hidden: int = 256,
max_T: int = 256, mod_dropout_p: float = 0.3, fusion_type: str = "concat",
pool_type: str = "query", joint_head: bool = True):
super().__init__(modality_dims)
if fusion_type != "concat":
raise ValueError("DAF supports fusion_type='concat' only (the configuration "
"described in the paper)")
if pool_type not in ("query", "mean"):
raise ValueError(f"unknown pool_type {pool_type!r}")
self.mod_dropout_p = float(mod_dropout_p)
self.pool_type = pool_type
self.stems = nn.ModuleDict({m: ModalityStem(d, d_model, dropout=dropout)
for m, d in self.modality_dims.items()})
self.fuse = nn.Conv1d(d_model * len(self.modalities), d_model, 1)
self.fuse_norm = nn.LayerNorm(d_model)
self.pos_embed = nn.Parameter(torch.zeros(1, max_T, d_model))
nn.init.trunc_normal_(self.pos_embed, std=0.02)
self.max_T = max_T
self.encoder = _encoder(d_model, n_heads, n_layers, dropout)
self.pool = QueryPool(d_model, n_heads=n_heads, dropout=dropout)
head_cls = JointVNTripletHead if joint_head else TripletHead
self.head = head_cls(d_model, sizes, hidden=head_hidden, dropout=dropout)
def forward(self, x, mask):
feats = [self.stems[m](x[m]) for m in self.modalities]
M = len(feats)
if self.training and self.mod_dropout_p > 0 and M > 1:
keep = torch.rand(M, device=feats[0].device) >= self.mod_dropout_p
if not bool(keep.any()):
keep = torch.zeros_like(keep)
keep[torch.randint(0, M, (1,), device=feats[0].device)] = True
feats = [f if bool(k) else torch.zeros_like(f) for f, k in zip(feats, keep)]
h = self.fuse(torch.cat(feats, dim=-1).transpose(1, 2)).transpose(1, 2)
h = self.fuse_norm(h)
T = h.size(1)
_check_T(T, self.max_T, "DailyActFormerV2")
h = self.encoder(h + self.pos_embed[:, :T], src_key_padding_mask=~mask)
pooled = masked_mean(h, mask) if self.pool_type == "mean" else self.pool(h, ~mask)
return self.head(pooled)