PULSE-code / pulse /models /t1 /syncfuse.py
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
3.75 kB
"""SyncFuse, the paper's T1 model (``ActionSensePlus`` in the submission code).
ActionSense's per-modality MLP stem and shared BiLSTM, plus
* a per-modality BiGRU temporal stem with LayerNorm before fusion,
* modality dropout during training (``mod_dropout_p``, default 0.3),
* attention pooling over time (one learned query) instead of mean pooling.
Modality dropout is **per sample**: for every training sample each modality
is dropped independently with probability ``p`` (its stem output is zeroed),
and a sample that would lose every modality keeps one chosen uniformly at
random. (The submission code drew one keep-vector per batch; the paper text
describes per-sample dropout, which is what this implementation does.)
The parameter count is whatever the model has; with 5 modalities and
``hidden=128`` it is about 3.1M (the submitted text's "1.6M" was wrong).
"""
from __future__ import annotations
from typing import Mapping, Optional
import torch
import torch.nn as nn
from .common import offsets
def per_sample_keep_mask(batch: int, n_mod: int, p: float, device=None,
generator: Optional[torch.Generator] = None) -> torch.Tensor:
"""``(B, M)`` bool keep-mask: Bernoulli(1-p) per sample and modality, at
least one modality kept per sample (rescued uniformly at random)."""
keep = torch.rand(batch, n_mod, generator=generator, device=device) >= p
dead = ~keep.any(dim=1)
if dead.any():
idx = torch.randint(0, n_mod, (int(dead.sum()),), generator=generator, device=device)
rows = dead.nonzero(as_tuple=True)[0]
keep[rows, idx] = True
return keep
class SyncFuse(nn.Module):
def __init__(self, modality_dims: Mapping[str, int], num_classes, hidden=128,
mod_dropout_p=0.3, gru_layers=1):
super().__init__()
self.mod_dims = dict(modality_dims)
self.mod_names = list(self.mod_dims)
self.mod_dropout_p = float(mod_dropout_p)
self.per_mod = nn.ModuleDict({
m: nn.Sequential(nn.Linear(d, hidden), nn.ReLU(), nn.Dropout(0.2),
nn.Linear(hidden, hidden), nn.ReLU())
for m, d in self.mod_dims.items()})
self.per_mod_gru = nn.ModuleDict({
m: nn.GRU(hidden, hidden, num_layers=gru_layers, batch_first=True,
bidirectional=True, dropout=0.0)
for m in self.mod_names})
self.pre_fuse_norm = nn.ModuleDict({m: nn.LayerNorm(hidden * 2) for m in self.mod_names})
self.lstm = nn.LSTM(hidden * 2 * len(self.mod_names), hidden, num_layers=2,
batch_first=True, bidirectional=True, dropout=0.2)
self.attn_query = nn.Parameter(torch.zeros(1, 1, hidden * 2))
nn.init.trunc_normal_(self.attn_query, std=0.02)
self.attn_proj = nn.Linear(hidden * 2, hidden * 2)
self.head = nn.Linear(hidden * 2, num_classes)
def forward(self, x, mask):
B = x.size(0)
feats = []
for m, (s, e) in offsets(self.mod_dims).items():
h, _ = self.per_mod_gru[m](self.per_mod[m](x[..., s:e]))
feats.append(self.pre_fuse_norm[m](h))
if self.training and self.mod_dropout_p > 0:
keep = per_sample_keep_mask(B, len(feats), self.mod_dropout_p, device=x.device)
feats = [f * keep[:, i].to(f.dtype).view(B, 1, 1) for i, f in enumerate(feats)]
h, _ = self.lstm(torch.cat(feats, dim=-1)) # (B, T, 2H)
q = self.attn_query.expand(B, -1, -1)
scores = torch.bmm(q, self.attn_proj(h).transpose(1, 2)).squeeze(1)
weights = torch.softmax(scores.masked_fill(~mask, float("-inf")), dim=-1)
return self.head((h * weights.unsqueeze(-1)).sum(dim=1))