Download pulse/models/t1/syncfuse.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 3.75 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/pulse/models/t1/syncfuse.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/pulse/models/t1/syncfuse.py
-
curl -L -o syncfuse.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/pulse/models/t1/syncfuse.py
3.75 kB
| """SyncFuse, the paper's T1 model (``ActionSensePlus`` in the submission code). | |
| ActionSense's per-modality MLP stem and shared BiLSTM, plus | |
| * a per-modality BiGRU temporal stem with LayerNorm before fusion, | |
| * modality dropout during training (``mod_dropout_p``, default 0.3), | |
| * attention pooling over time (one learned query) instead of mean pooling. | |
| Modality dropout is **per sample**: for every training sample each modality | |
| is dropped independently with probability ``p`` (its stem output is zeroed), | |
| and a sample that would lose every modality keeps one chosen uniformly at | |
| random. (The submission code drew one keep-vector per batch; the paper text | |
| describes per-sample dropout, which is what this implementation does.) | |
| The parameter count is whatever the model has; with 5 modalities and | |
| ``hidden=128`` it is about 3.1M (the submitted text's "1.6M" was wrong). | |
| """ | |
| from __future__ import annotations | |
| from typing import Mapping, Optional | |
| import torch | |
| import torch.nn as nn | |
| from .common import offsets | |
| def per_sample_keep_mask(batch: int, n_mod: int, p: float, device=None, | |
| generator: Optional[torch.Generator] = None) -> torch.Tensor: | |
| """``(B, M)`` bool keep-mask: Bernoulli(1-p) per sample and modality, at | |
| least one modality kept per sample (rescued uniformly at random).""" | |
| keep = torch.rand(batch, n_mod, generator=generator, device=device) >= p | |
| dead = ~keep.any(dim=1) | |
| if dead.any(): | |
| idx = torch.randint(0, n_mod, (int(dead.sum()),), generator=generator, device=device) | |
| rows = dead.nonzero(as_tuple=True)[0] | |
| keep[rows, idx] = True | |
| return keep | |
| class SyncFuse(nn.Module): | |
| def __init__(self, modality_dims: Mapping[str, int], num_classes, hidden=128, | |
| mod_dropout_p=0.3, gru_layers=1): | |
| super().__init__() | |
| self.mod_dims = dict(modality_dims) | |
| self.mod_names = list(self.mod_dims) | |
| self.mod_dropout_p = float(mod_dropout_p) | |
| self.per_mod = nn.ModuleDict({ | |
| m: nn.Sequential(nn.Linear(d, hidden), nn.ReLU(), nn.Dropout(0.2), | |
| nn.Linear(hidden, hidden), nn.ReLU()) | |
| for m, d in self.mod_dims.items()}) | |
| self.per_mod_gru = nn.ModuleDict({ | |
| m: nn.GRU(hidden, hidden, num_layers=gru_layers, batch_first=True, | |
| bidirectional=True, dropout=0.0) | |
| for m in self.mod_names}) | |
| self.pre_fuse_norm = nn.ModuleDict({m: nn.LayerNorm(hidden * 2) for m in self.mod_names}) | |
| self.lstm = nn.LSTM(hidden * 2 * len(self.mod_names), hidden, num_layers=2, | |
| batch_first=True, bidirectional=True, dropout=0.2) | |
| self.attn_query = nn.Parameter(torch.zeros(1, 1, hidden * 2)) | |
| nn.init.trunc_normal_(self.attn_query, std=0.02) | |
| self.attn_proj = nn.Linear(hidden * 2, hidden * 2) | |
| self.head = nn.Linear(hidden * 2, num_classes) | |
| def forward(self, x, mask): | |
| B = x.size(0) | |
| feats = [] | |
| for m, (s, e) in offsets(self.mod_dims).items(): | |
| h, _ = self.per_mod_gru[m](self.per_mod[m](x[..., s:e])) | |
| feats.append(self.pre_fuse_norm[m](h)) | |
| if self.training and self.mod_dropout_p > 0: | |
| keep = per_sample_keep_mask(B, len(feats), self.mod_dropout_p, device=x.device) | |
| feats = [f * keep[:, i].to(f.dtype).view(B, 1, 1) for i, f in enumerate(feats)] | |
| h, _ = self.lstm(torch.cat(feats, dim=-1)) # (B, T, 2H) | |
| q = self.attn_query.expand(B, -1, -1) | |
| scores = torch.bmm(q, self.attn_proj(h).transpose(1, 2)).squeeze(1) | |
| weights = torch.softmax(scores.masked_fill(~mask, float("-inf")), dim=-1) | |
| return self.head((h * weights.unsqueeze(-1)).sum(dim=1)) | |