Buckets:
| import torch | |
| import torch.nn as nn | |
| class StageTransformer(nn.Module): | |
| def __init__(self, | |
| d_model: int = 512, | |
| vis_emb_dim: int = 512, | |
| text_emb_dim: int = 512, | |
| state_dim: int = 7, | |
| n_layers: int = 6, | |
| n_heads: int = 8, | |
| dropout: float = 0.1, | |
| num_cameras: int = 1, | |
| num_classes_sparse: int = 4, | |
| num_classes_dense: int = 8, | |
| ): | |
| super().__init__() | |
| self.d_model = d_model | |
| self.num_cameras = num_cameras | |
| # Projections | |
| self.lang_proj = nn.Linear(text_emb_dim, d_model) | |
| self.visual_proj = nn.Linear(vis_emb_dim, d_model) | |
| self.state_proj = nn.Linear(state_dim, d_model) | |
| # Encoder | |
| enc_layer = nn.TransformerEncoderLayer( | |
| d_model, n_heads, 4 * d_model, dropout, batch_first=True | |
| ) | |
| self.transformer = nn.TransformerEncoder(enc_layer, n_layers) | |
| # Positional bias (only tag the first visual frame of each camera) | |
| self.first_pos = nn.Parameter(torch.zeros(1, d_model)) | |
| # Shared fusion MLP (produces a d_model feature per timestep) | |
| fused_in = d_model * (num_cameras + 2) | |
| self.fusion_backbone = nn.Sequential( | |
| nn.LayerNorm(fused_in), | |
| nn.Linear(fused_in, d_model), | |
| nn.ReLU(), | |
| ) | |
| # Scheme-specific heads | |
| self.heads = nn.ModuleDict({ | |
| "sparse": nn.Linear(d_model, num_classes_sparse), | |
| "dense": nn.Linear(d_model, num_classes_dense), | |
| }) | |
| def _prep_lang(self, lang_emb: torch.Tensor, B: int, T: int, D: int) -> torch.Tensor: | |
| """ | |
| Accepts lang_emb of shape: | |
| - (B, text_emb_dim) -> broadcast across time | |
| - (B, T, text_emb_dim) -> per-timestep (dense) | |
| Returns (B, 1, T, D) | |
| """ | |
| if lang_emb.dim() == 3: | |
| # (B, T, E) -> (B, T, D) -> (B, 1, T, D) | |
| lang_proj = self.lang_proj(lang_emb).unsqueeze(1) | |
| else: | |
| # (B, E) -> (B, 1, 1, D) -> expand to (B, 1, T, D) | |
| lang_proj = self.lang_proj(lang_emb).unsqueeze(1).unsqueeze(2).expand(B, 1, T, D) | |
| return lang_proj | |
| def forward(self, | |
| img_seq: torch.Tensor, # (B, N, T, vis_emb_dim) | |
| lang_emb: torch.Tensor, # (B, E) or (B, T, E) | |
| state: torch.Tensor, # (B, T, state_dim) | |
| lengths: torch.Tensor, # (B,) | |
| scheme: str = "sparse", # "sparse" or "dense" | |
| ) -> torch.Tensor: | |
| assert scheme in self.heads, f"Unknown scheme '{scheme}'. Use one of {list(self.heads.keys())}." | |
| B, N, T, _ = img_seq.shape | |
| D = self.d_model | |
| device = img_seq.device | |
| # Project inputs | |
| vis_proj = self.visual_proj(img_seq) # (B, N, T, D) | |
| state_proj = self.state_proj(state).unsqueeze(1) # (B, 1, T, D) | |
| lang_proj = self._prep_lang(lang_emb, B, T, D) # (B, 1, T, D) | |
| # Concatenate streams (cameras + lang + state) | |
| x = torch.cat([vis_proj, lang_proj, state_proj], dim=1) # (B, N+2, T, D) | |
| x[:, :N, 0, :] += self.first_pos | |
| # Flatten to tokens for Transformer | |
| x_tokens = x.view(B, (N + 2) * T, D) | |
| L = x_tokens.size(1) | |
| base_mask = torch.arange(T, device=device).expand(B, T) >= lengths.unsqueeze(1) # (B, T) | |
| mask = base_mask.unsqueeze(1).expand(B, N + 2, T).reshape(B, (N + 2) * T) | |
| causal_mask = torch.triu( | |
| torch.ones(L, L, device=device, dtype=torch.bool), | |
| diagonal=1 | |
| ) # (L, L) | |
| # Encode | |
| h = self.transformer(x_tokens, | |
| mask=causal_mask, | |
| src_key_padding_mask=mask, | |
| is_causal=True) # (B, (N+2)*T, D) | |
| h = h.view(B, N + 2, T, D).permute(0, 2, 1, 3).reshape(B, T, (N + 2) * D) | |
| fused = self.fusion_backbone(h) # (B, T, D) | |
| # Scheme-specific logits | |
| logits = self.heads[scheme](fused) # (B, T, C_scheme) | |
| return logits | |
Xet Storage Details
- Size:
- 4.27 kB
- Xet hash:
- 8d946b75927a62e936ffd1a083453ed2c3897004bb8f86a027e0ef89a2dbe531
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.