Nadeem-Shoukath115113/opensarm / models /action_estimator.py
Nadeem-Shoukath115113's picture
download
raw
3.84 kB
import torch
import torch.nn as nn
from typing import Optional, Tuple
class ActionTransformer(nn.Module):
def __init__(self,
d_model: int = 512,
vis_emb_dim: int = 512,
state_dim: int = 7,
n_layers: int = 6,
n_heads: int = 8,
dropout: float = 0.1,
num_cameras: int = 1,
num_tasks: int = 27,
num_classes: Optional[int] = None,
max_t: int = 200, # Maximum supported sequence length
):
super().__init__()
self.d_model = d_model
self.num_cameras = num_cameras
self.max_t = max_t
# Projections
self.visual_proj = nn.Linear(vis_emb_dim, d_model)
self.state_proj = nn.Linear(state_dim, d_model)
# --- Positional Encoding ---
# Learned positional embeddings for each frame
self.time_pos_emb = nn.Parameter(torch.zeros(1, 1, max_t, d_model))
# Encoder Backbone
enc_layer = nn.TransformerEncoderLayer(
d_model, n_heads, 4 * d_model, dropout, batch_first=True
)
self.transformer = nn.TransformerEncoder(enc_layer, n_layers)
# Shared fusion MLP
fused_in = d_model * (num_cameras + 1)
self.fusion_backbone = nn.Sequential(
nn.LayerNorm(fused_in),
nn.Linear(fused_in, d_model),
nn.ReLU(),
)
self.task_head = nn.Linear(d_model, num_tasks)
self.class_head = nn.Linear(d_model, num_classes) if num_classes is not None else None
def forward(self,
img_seq: torch.Tensor, # (B, N, T, vis_emb_dim)
state: torch.Tensor, # (B, T, state_dim)
lengths: torch.Tensor, # (B,)
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
B, N, T, _ = img_seq.shape
D = self.d_model
device = img_seq.device
# 1. Input Projections
vis_proj = self.visual_proj(img_seq) # (B, N, T, D)
state_proj = self.state_proj(state).unsqueeze(1) # (B, 1, T, D)
# 2. Token Concatenation
# x shape: (B, N + 1, T, D)
x = torch.cat([vis_proj, state_proj], dim=1)
# 3. Add Temporal Position Encoding
# We slice the embedding to current T and broadcast across B and (N+1)
# self.time_pos_emb[:, :, :T, :] has shape (1, 1, T, D)
x = x + self.time_pos_emb[:, :, :T, :]
# 4. Flatten for Transformer
# (B, (N + 1) * T, D)
x = x.view(B, (N + 1) * T, D)
L = x.size(1)
# 5. Masking Logic
base_mask = torch.arange(T, device=device).expand(B, T) >= lengths.unsqueeze(1)
pad_mask = base_mask.unsqueeze(1).expand(B, N + 1, T).reshape(B, (N + 1) * T)
causal_mask = torch.triu(torch.ones(L, L, device=device, dtype=torch.bool), diagonal=1)
# 6. Encoding
h = self.transformer(
x,
mask=causal_mask,
src_key_padding_mask=pad_mask,
is_causal=True
)
# 7. Spatio-Temporal Fusion
# Reshape back to (B, T, (N+1)*D) to fuse camera and state tokens per frame
h = h.view(B, N + 1, T, D).permute(0, 2, 1, 3).reshape(B, T, (N + 1) * D)
fused = self.fusion_backbone(h)
# 8. Final State Extraction (Last valid frame)
last_idx = (lengths - 1).long()
fused_final = fused[torch.arange(B), last_idx]
# 9. Multi-task Output Logic
task_logits = self.task_head(fused_final)
if self.class_head is None:
return task_logits
else:
class_logits = self.class_head(fused_final)
return task_logits, class_logits

Xet Storage Details

Size:
3.84 kB
·
Xet hash:
3264a11856447388128f4e15ed84b3f624aa3016bc1755fb34d3eb956fc637aa

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.