"""SyncFuse, the paper's T1 model (``ActionSensePlus`` in the submission code). ActionSense's per-modality MLP stem and shared BiLSTM, plus * a per-modality BiGRU temporal stem with LayerNorm before fusion, * modality dropout during training (``mod_dropout_p``, default 0.3), * attention pooling over time (one learned query) instead of mean pooling. Modality dropout is **per sample**: for every training sample each modality is dropped independently with probability ``p`` (its stem output is zeroed), and a sample that would lose every modality keeps one chosen uniformly at random. (The submission code drew one keep-vector per batch; the paper text describes per-sample dropout, which is what this implementation does.) The parameter count is whatever the model has; with 5 modalities and ``hidden=128`` it is about 3.1M (the submitted text's "1.6M" was wrong). """ from __future__ import annotations from typing import Mapping, Optional import torch import torch.nn as nn from .common import offsets def per_sample_keep_mask(batch: int, n_mod: int, p: float, device=None, generator: Optional[torch.Generator] = None) -> torch.Tensor: """``(B, M)`` bool keep-mask: Bernoulli(1-p) per sample and modality, at least one modality kept per sample (rescued uniformly at random).""" keep = torch.rand(batch, n_mod, generator=generator, device=device) >= p dead = ~keep.any(dim=1) if dead.any(): idx = torch.randint(0, n_mod, (int(dead.sum()),), generator=generator, device=device) rows = dead.nonzero(as_tuple=True)[0] keep[rows, idx] = True return keep class SyncFuse(nn.Module): def __init__(self, modality_dims: Mapping[str, int], num_classes, hidden=128, mod_dropout_p=0.3, gru_layers=1): super().__init__() self.mod_dims = dict(modality_dims) self.mod_names = list(self.mod_dims) self.mod_dropout_p = float(mod_dropout_p) self.per_mod = nn.ModuleDict({ m: nn.Sequential(nn.Linear(d, hidden), nn.ReLU(), nn.Dropout(0.2), nn.Linear(hidden, hidden), nn.ReLU()) for m, d in self.mod_dims.items()}) self.per_mod_gru = nn.ModuleDict({ m: nn.GRU(hidden, hidden, num_layers=gru_layers, batch_first=True, bidirectional=True, dropout=0.0) for m in self.mod_names}) self.pre_fuse_norm = nn.ModuleDict({m: nn.LayerNorm(hidden * 2) for m in self.mod_names}) self.lstm = nn.LSTM(hidden * 2 * len(self.mod_names), hidden, num_layers=2, batch_first=True, bidirectional=True, dropout=0.2) self.attn_query = nn.Parameter(torch.zeros(1, 1, hidden * 2)) nn.init.trunc_normal_(self.attn_query, std=0.02) self.attn_proj = nn.Linear(hidden * 2, hidden * 2) self.head = nn.Linear(hidden * 2, num_classes) def forward(self, x, mask): B = x.size(0) feats = [] for m, (s, e) in offsets(self.mod_dims).items(): h, _ = self.per_mod_gru[m](self.per_mod[m](x[..., s:e])) feats.append(self.pre_fuse_norm[m](h)) if self.training and self.mod_dropout_p > 0: keep = per_sample_keep_mask(B, len(feats), self.mod_dropout_p, device=x.device) feats = [f * keep[:, i].to(f.dtype).view(B, 1, 1) for i, f in enumerate(feats)] h, _ = self.lstm(torch.cat(feats, dim=-1)) # (B, T, 2H) q = self.attn_query.expand(B, -1, -1) scores = torch.bmm(q, self.attn_proj(h).transpose(1, 2)).squeeze(1) weights = torch.softmax(scores.masked_fill(~mask, float("-inf")), dim=-1) return self.head((h * weights.unsqueeze(-1)).sum(dim=1))