"""Parvulus Neural Architecture: Dual-POV Flow Matching Model for Robot Manipulation.""" import json import math from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Tuple, Union import torch import torch.nn as nn import torch.nn.functional as F from safetensors.torch import load_file def sinusoidal_embedding(t: torch.Tensor, dim: int) -> torch.Tensor: half = dim // 2 freqs = torch.exp( -math.log(10000) * torch.arange(half, device=t.device, dtype=torch.float32) / half ) args = t.float()[:, None] * freqs[None, :] * 1000.0 emb = torch.cat([torch.sin(args), torch.cos(args)], dim=-1) if dim % 2: emb = F.pad(emb, (0, 1)) return emb class TimestepMLP(nn.Module): def __init__(self, cond_dim: int): super().__init__() self.cond_dim = cond_dim self.mlp = nn.Sequential( nn.Linear(cond_dim, cond_dim), nn.SiLU(), nn.Linear(cond_dim, cond_dim), ) def forward(self, t: torch.Tensor) -> torch.Tensor: return self.mlp(sinusoidal_embedding(t, self.cond_dim)) class FiLMResBlock2d(nn.Module): def __init__(self, in_ch: int, out_ch: int, cond_dim: int, groups: int = 8): super().__init__() groups_in = min(groups, in_ch) groups_out = min(groups, out_ch) self.norm1 = nn.GroupNorm(groups_in, in_ch) self.conv1 = nn.Conv2d(in_ch, out_ch, 3, padding=1) self.norm2 = nn.GroupNorm(groups_out, out_ch) self.conv2 = nn.Conv2d(out_ch, out_ch, 3, padding=1) self.film = nn.Linear(cond_dim, 2 * out_ch) self.skip = nn.Conv2d(in_ch, out_ch, 1) if in_ch != out_ch else nn.Identity() def forward(self, x: torch.Tensor, cond: torch.Tensor) -> torch.Tensor: h = self.conv1(F.silu(self.norm1(x))) scale, shift = self.film(cond).chunk(2, dim=-1) h = self.norm2(h) * (1.0 + scale[:, :, None, None]) + shift[:, :, None, None] h = self.conv2(F.silu(h)) return h + self.skip(x) class CVMixer(nn.Module): """Bidirectional cross-view mixer with gated contextual residuals.""" def __init__(self, ch: int): super().__init__() self.ch = ch self.context_conv = nn.Sequential( nn.Conv2d(2 * ch, ch, 1), nn.GroupNorm(min(8, ch), ch), nn.SiLU(), ) self.agent_proj = nn.Conv2d(ch, ch, 3, padding=1) self.agent_gate = nn.Conv2d(ch, ch, 1) self.wrist_proj = nn.Conv2d(ch, ch, 3, padding=1) self.wrist_gate = nn.Conv2d(ch, ch, 1) def forward(self, f_agent: torch.Tensor, f_wrist: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: joint = torch.cat([f_agent, f_wrist], dim=1) ctx = self.context_conv(joint) f_agent_out = f_agent + self.agent_proj(ctx) * torch.sigmoid(self.agent_gate(ctx)) f_wrist_out = f_wrist + self.wrist_proj(ctx) * torch.sigmoid(self.wrist_gate(ctx)) f_mixed = torch.cat([f_agent_out, f_wrist_out], dim=1) return f_agent_out, f_wrist_out, f_mixed class AttnPool2d(nn.Module): """Multi-head attention pooling extracting K visual query tokens.""" def __init__(self, in_ch: int, embed_dim: int = 256, num_queries: int = 4, num_heads: int = 4): super().__init__() self.in_proj = nn.Conv2d(in_ch, embed_dim, 1) self.queries = nn.Parameter(torch.randn(1, num_queries, embed_dim) * 0.02) self.cross_attn = nn.MultiheadAttention(embed_dim, num_heads=num_heads, batch_first=True) self.norm = nn.LayerNorm(embed_dim) def forward(self, x: torch.Tensor) -> torch.Tensor: b = x.shape[0] feats = self.in_proj(x).flatten(2).transpose(1, 2) q = self.queries.expand(b, -1, -1) out, _ = self.cross_attn(query=q, key=feats, value=feats) return self.norm(out) class DualViewEncoder(nn.Module): """Hierarchical dual-view vision encoder with cross-view mixing at all stages.""" def __init__( self, base_ch: int = 48, channel_mults: Sequence[int] = (1, 2, 2, 4), num_res_blocks: int = 1, cond_dim: int = 256, tokens_per_layer: int = 4, ): super().__init__() self.base_ch = base_ch self.channel_mults = channel_mults self.cond_dim = cond_dim self.agent_stem = nn.Conv2d(3, base_ch, 3, padding=1) self.wrist_stem = nn.Conv2d(3, base_ch, 3, padding=1) self.agent_stages = nn.ModuleList() self.wrist_stages = nn.ModuleList() self.cv_mixers = nn.ModuleList() self.downsamples = nn.ModuleList() self.attn_poolers = nn.ModuleList() self.level_channels: List[int] = [] in_ch = base_ch for i, mult in enumerate(channel_mults): out_ch = base_ch * mult a_blocks = nn.ModuleList([ FiLMResBlock2d(in_ch if j == 0 else out_ch, out_ch, cond_dim) for j in range(num_res_blocks) ]) self.agent_stages.append(a_blocks) w_blocks = nn.ModuleList([ FiLMResBlock2d(in_ch if j == 0 else out_ch, out_ch, cond_dim) for j in range(num_res_blocks) ]) self.wrist_stages.append(w_blocks) self.cv_mixers.append(CVMixer(out_ch)) self.level_channels.append(2 * out_ch) self.attn_poolers.append(AttnPool2d(2 * out_ch, embed_dim=cond_dim, num_queries=tokens_per_layer)) self.downsamples.append( nn.Conv2d(out_ch, out_ch, 3, stride=2, padding=1) if i < len(channel_mults) - 1 else nn.Identity() ) in_ch = out_ch def forward(self, x: torch.Tensor, cond: torch.Tensor) -> Tuple[List[torch.Tensor], torch.Tensor]: h_agent = self.agent_stem(x[:, 0:3]) h_wrist = self.wrist_stem(x[:, 3:6]) skips = [] visual_token_list = [] for i, (a_stage, w_stage, mixer, pooler, down) in enumerate( zip(self.agent_stages, self.wrist_stages, self.cv_mixers, self.attn_poolers, self.downsamples) ): if i == 0: h_agent = F.interpolate(h_agent, scale_factor=0.5, mode='bilinear', align_corners=False) h_wrist = F.interpolate(h_wrist, scale_factor=0.5, mode='bilinear', align_corners=False) for a_blk, w_blk in zip(a_stage, w_stage): h_agent = a_blk(h_agent, cond) h_wrist = w_blk(h_wrist, cond) h_agent, h_wrist, f_mixed = mixer(h_agent, h_wrist) skips.append(f_mixed) layer_tokens = pooler(f_mixed) visual_token_list.append(layer_tokens) h_agent = down(h_agent) h_wrist = down(h_wrist) z_vis = torch.cat(visual_token_list, dim=1) return skips, z_vis class TransformerDecoderLayer(nn.Module): def __init__(self, embed_dim: int = 256, num_heads: int = 8, mlp_ratio: int = 4): super().__init__() self.norm1 = nn.LayerNorm(embed_dim) self.self_attn = nn.MultiheadAttention(embed_dim, num_heads=num_heads, batch_first=True) self.norm2 = nn.LayerNorm(embed_dim) self.cross_attn = nn.MultiheadAttention(embed_dim, num_heads=num_heads, batch_first=True) self.norm3 = nn.LayerNorm(embed_dim) self.mlp = nn.Sequential( nn.Linear(embed_dim, embed_dim * mlp_ratio), nn.GELU(), nn.Linear(embed_dim * mlp_ratio, embed_dim), ) self.film = nn.Linear(embed_dim, 2 * embed_dim) def forward(self, x: torch.Tensor, z_vis: torch.Tensor, cond: torch.Tensor) -> torch.Tensor: h = x + self.self_attn(query=self.norm1(x), key=self.norm1(x), value=self.norm1(x))[0] h = h + self.cross_attn(query=self.norm2(h), key=z_vis, value=z_vis)[0] scale, shift = self.film(cond).unsqueeze(1).chunk(2, dim=-1) norm_h = self.norm3(h) * (1.0 + scale) + shift h = h + self.mlp(norm_h) return h class OptimizedKVCacheDecoderLayer(nn.Module): def __init__(self, orig_layer: TransformerDecoderLayer): super().__init__() self.norm1 = orig_layer.norm1 self.self_attn = orig_layer.self_attn self.norm2 = orig_layer.norm2 self.cross_attn = orig_layer.cross_attn self.norm3 = orig_layer.norm3 self.mlp = orig_layer.mlp self.film = orig_layer.film def precompute_kv(self, z_vis: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: w = self.cross_attn.in_proj_weight b = self.cross_attn.in_proj_bias w_k = w[256:512, :] b_k = b[256:512] if b is not None else None w_v = w[512:768, :] b_v = b[512:768] if b is not None else None k = F.linear(z_vis, w_k, b_k) v = F.linear(z_vis, w_v, b_v) return k, v def forward_with_kv(self, x: torch.Tensor, k_vis: torch.Tensor, v_vis: torch.Tensor, cond: torch.Tensor) -> torch.Tensor: q1 = self.norm1(x) h = x + self.self_attn(q1, q1, q1, need_weights=False)[0] q2 = self.norm2(h) w_q = self.cross_attn.in_proj_weight[:256, :] b_q = self.cross_attn.in_proj_bias[:256] if self.cross_attn.in_proj_bias is not None else None q_proj = F.linear(q2, w_q, b_q) B, S, D = q_proj.shape H = 8 d_k = 32 q_h = q_proj.view(B, S, H, d_k).transpose(1, 2) k_h = k_vis.view(B, 16, H, d_k).transpose(1, 2) v_h = v_vis.view(B, 16, H, d_k).transpose(1, 2) attn_out = F.scaled_dot_product_attention(q_h, k_h, v_h) attn_out = attn_out.transpose(1, 2).contiguous().view(B, S, D) out = F.linear(attn_out, self.cross_attn.out_proj.weight, self.cross_attn.out_proj.bias) h = h + out scale, shift = self.film(cond).unsqueeze(1).chunk(2, dim=-1) norm_h = self.norm3(h) * (1.0 + scale) + shift h = h + self.mlp(norm_h) return h class ActionChunkTransformerDecoder(nn.Module): def __init__( self, action_dim: int = 7, chunk_size: int = 8, embed_dim: int = 256, num_layers: int = 4, num_heads: int = 8, ): super().__init__() self.action_dim = action_dim self.chunk_size = chunk_size self.embed_dim = embed_dim self.action_in_proj = nn.Linear(action_dim, embed_dim) self.pos_emb = nn.Parameter(torch.randn(1, chunk_size, embed_dim) * 0.02) self.layers = nn.ModuleList([ TransformerDecoderLayer(embed_dim, num_heads=num_heads) for _ in range(num_layers) ]) self.final_norm = nn.LayerNorm(embed_dim) self.out_head = nn.Linear(embed_dim, action_dim) def forward(self, z_sigma: torch.Tensor, z_vis: torch.Tensor, cond: torch.Tensor) -> torch.Tensor: b, s, d = z_sigma.shape h = self.action_in_proj(z_sigma) + self.pos_emb[:, :s, :] for layer in self.layers: h = layer(h, z_vis, cond) return self.out_head(self.final_norm(h)) class MultiScaleUNetDecoder(nn.Module): def __init__( self, level_channels: Sequence[int], cond_dim: int = 256, out_channels: int = 6, num_res_blocks: int = 1, ): super().__init__() rev_levels = list(reversed(level_channels)) self.stages = nn.ModuleList() self.upsamples = nn.ModuleList() in_ch = rev_levels[0] for i, lvl_ch in enumerate(rev_levels): blocks = nn.ModuleList() for j in range(num_res_blocks): b_in = in_ch + (lvl_ch if j == 0 else 0) blocks.append(FiLMResBlock2d(b_in, lvl_ch, cond_dim)) in_ch = lvl_ch self.stages.append(blocks) self.upsamples.append( nn.Sequential( nn.Upsample(scale_factor=2, mode='nearest'), nn.Conv2d(lvl_ch, lvl_ch, 3, padding=1) ) if i < len(rev_levels) - 1 else nn.Identity() ) self.final_upsample = nn.Sequential( nn.Upsample(scale_factor=2, mode='nearest'), nn.Conv2d(rev_levels[-1], rev_levels[-1], 3, padding=1), nn.GroupNorm(min(8, rev_levels[-1]), rev_levels[-1]), nn.SiLU(), nn.Conv2d(rev_levels[-1], out_channels, 3, padding=1), ) def forward(self, skips: Sequence[torch.Tensor], cond: torch.Tensor) -> torch.Tensor: rev_skips = list(reversed(skips)) x = rev_skips[0] for stage, up, skip in zip(self.stages, self.upsamples, rev_skips): for j, block in enumerate(stage): if j == 0: x = torch.cat([x, skip], dim=1) x = block(x, cond) x = up(x) return self.final_upsample(x) class ParvulusOutput(NamedTuple): action_flow: Optional[torch.Tensor] = None video_flow: Optional[torch.Tensor] = None self_recon: Optional[torch.Tensor] = None change_pred: Optional[torch.Tensor] = None class ParvulusModel(nn.Module): """Parvulus: Dual-POV Flow Matching Robot Manipulation Model.""" def __init__( self, base_channels: int = 48, channel_mults: Sequence[int] = (1, 2, 2, 4), num_res_blocks: int = 1, cond_dim: int = 256, action_dim: int = 7, chunk_size: int = 8, text_encoder: Optional[nn.Module] = None, use_video_decoder: bool = False, use_self_decoder: bool = False, use_change_decoder: bool = False, ): super().__init__() self.cond_dim = cond_dim self.action_dim = action_dim self.chunk_size = chunk_size self.text_encoder = text_encoder self.encoder = DualViewEncoder( base_ch=base_channels, channel_mults=channel_mults, num_res_blocks=num_res_blocks, cond_dim=cond_dim, tokens_per_layer=4, ) self.action_t_mlp = TimestepMLP(cond_dim) self.video_t_mlp = TimestepMLP(cond_dim) self.action_decoder = ActionChunkTransformerDecoder( action_dim=action_dim, chunk_size=chunk_size, embed_dim=cond_dim, num_layers=4, num_heads=8, ) self.video_decoder = MultiScaleUNetDecoder( level_channels=self.encoder.level_channels, cond_dim=cond_dim, out_channels=6, num_res_blocks=num_res_blocks, ) if use_video_decoder else None self.self_decoder = MultiScaleUNetDecoder( level_channels=self.encoder.level_channels, cond_dim=cond_dim, out_channels=6, num_res_blocks=num_res_blocks, ) if use_self_decoder else None self.change_decoder = MultiScaleUNetDecoder( level_channels=self.encoder.level_channels, cond_dim=cond_dim, out_channels=2, num_res_blocks=num_res_blocks, ) if use_change_decoder else None self._kv_layers: Optional[nn.ModuleList] = None def _get_kv_layers(self) -> nn.ModuleList: if self._kv_layers is None: self._kv_layers = nn.ModuleList([ OptimizedKVCacheDecoderLayer(l) for l in self.action_decoder.layers ]) return self._kv_layers def _text_cond( self, texts: Optional[Sequence[str]], text_emb: Optional[torch.Tensor], batch: int, device: torch.device, ) -> torch.Tensor: if text_emb is not None: return text_emb if texts is not None and self.text_encoder is not None: return self.text_encoder(texts) return torch.zeros(batch, self.cond_dim, device=device) def forward( self, obs_input: torch.Tensor, z_sigma: Optional[torch.Tensor] = None, action_sigma: Optional[torch.Tensor] = None, next_target: Optional[torch.Tensor] = None, video_tau: Optional[torch.Tensor] = None, texts: Optional[Sequence[str]] = None, text_emb: Optional[torch.Tensor] = None, compute_action: bool = True, compute_video: bool = True, compute_self_recon: bool = True, compute_change: bool = True, ) -> ParvulusOutput: b = obs_input.shape[0] device = obs_input.device t_emb = self._text_cond(texts, text_emb, b, device) skips_obs, z_vis = self.encoder(obs_input, t_emb) action_flow = None if compute_action and z_sigma is not None and action_sigma is not None: act_cond = t_emb + self.action_t_mlp(action_sigma) action_flow = self.action_decoder(z_sigma, z_vis, act_cond) video_flow = None if compute_video and self.video_decoder is not None and video_tau is not None: vid_cond = t_emb + self.video_t_mlp(video_tau) video_flow = self.video_decoder(skips_obs, vid_cond) self_recon = None if compute_self_recon and self.self_decoder is not None: self_recon = self.self_decoder(skips_obs, t_emb) change_pred = None if compute_change and self.change_decoder is not None: change_pred = self.change_decoder(skips_obs, t_emb) return ParvulusOutput( action_flow=action_flow, video_flow=video_flow, self_recon=self_recon, change_pred=change_pred, ) @torch.no_grad() def predict_action_chunk( self, obs: torch.Tensor, text_emb: Optional[torch.Tensor] = None, texts: Optional[Sequence[str]] = None, num_ode_steps: int = 10, ode_solver: str = "euler", use_kv_cache: bool = False, z_init: Optional[torch.Tensor] = None, ) -> torch.Tensor: b = obs.shape[0] device = obs.device t_emb = self._text_cond(texts, text_emb, b, device) _, z_vis = self.encoder(obs, t_emb) if z_init is not None: z = z_init.clone() else: z = torch.randn(b, self.chunk_size, self.action_dim, device=device) if use_kv_cache and ode_solver == "euler": kv_layers = self._get_kv_layers() kv_caches = [layer.precompute_kv(z_vis) for layer in kv_layers] dt = 1.0 / num_ode_steps for step in range(num_ode_steps): sigma = torch.full((b,), step * dt, device=device) cond = t_emb + self.action_t_mlp(sigma) h = self.action_decoder.action_in_proj(z) + self.action_decoder.pos_emb[:, :self.chunk_size, :] for layer, (k_v, v_v) in zip(kv_layers, kv_caches): h = layer.forward_with_kv(h, k_v, v_v, cond) v_pred = self.action_decoder.out_head(self.action_decoder.final_norm(h)) z = z + v_pred * dt return z.clamp(-1.0, 1.0) if ode_solver == "euler": dt = 1.0 / num_ode_steps for step in range(num_ode_steps): sigma = torch.full((b,), step * dt, device=device) act_cond = t_emb + self.action_t_mlp(sigma) v_pred = self.action_decoder(z, z_vis, act_cond) z = z + v_pred * dt elif ode_solver == "heun": dt = 1.0 / num_ode_steps for step in range(num_ode_steps): t_curr = step * dt t_next = (step + 1) * dt sigma_curr = torch.full((b,), t_curr, device=device) sigma_next = torch.full((b,), t_next, device=device) act_cond_curr = t_emb + self.action_t_mlp(sigma_curr) v1 = self.action_decoder(z, z_vis, act_cond_curr) z_euler = z + v1 * dt act_cond_next = t_emb + self.action_t_mlp(sigma_next) v2 = self.action_decoder(z_euler, z_vis, act_cond_next) z = z + 0.5 * dt * (v1 + v2) else: raise ValueError(f"Unsupported ode_solver: {ode_solver}") return z.clamp(-1.0, 1.0) @torch.no_grad() def predict_next_frame( self, obs: torch.Tensor, text_emb: Optional[torch.Tensor] = None, texts: Optional[Sequence[str]] = None, num_ode_steps: int = 10, x_init: Optional[torch.Tensor] = None, ) -> torch.Tensor: if self.video_decoder is None: raise RuntimeError("video_decoder is not initialized. Pass use_video_decoder=True when loading model.") b = obs.shape[0] device = obs.device t_emb = self._text_cond(texts, text_emb, b, device) skips, _ = self.encoder(obs, t_emb) if x_init is not None: x = x_init.clone() else: x = torch.randn(b, 6, 256, 256, device=device) dt = 1.0 / num_ode_steps for step in range(num_ode_steps): tau = torch.full((b,), step * dt, device=device) cond = t_emb + self.video_t_mlp(tau) v_pred = self.video_decoder(skips, cond) x = x + v_pred * dt return x.clamp(-1.0, 1.0) @torch.no_grad() def predict_change( self, obs: torch.Tensor, text_emb: Optional[torch.Tensor] = None, texts: Optional[Sequence[str]] = None, ) -> torch.Tensor: if self.change_decoder is None: raise RuntimeError("change_decoder is not initialized. Pass use_change_decoder=True when loading model.") b = obs.shape[0] device = obs.device t_emb = self._text_cond(texts, text_emb, b, device) skips, _ = self.encoder(obs, t_emb) return self.change_decoder(skips, t_emb) @torch.no_grad() def reconstruct_observation( self, obs: torch.Tensor, text_emb: Optional[torch.Tensor] = None, texts: Optional[Sequence[str]] = None, ) -> torch.Tensor: if self.self_decoder is None: raise RuntimeError("self_decoder is not initialized. Pass use_self_decoder=True when loading model.") b = obs.shape[0] device = obs.device t_emb = self._text_cond(texts, text_emb, b, device) skips, _ = self.encoder(obs, t_emb) return self.self_decoder(skips, t_emb) @classmethod def from_pretrained( cls, pretrained_path: Union[str, Path] = ".", load_auxiliary: bool = False, load_text_encoder: bool = True, device: Union[str, torch.device] = "cpu", ) -> "ParvulusModel": root = Path(pretrained_path) cfg_file = root / "config.json" if cfg_file.exists(): with open(cfg_file, "r") as f: cfg = json.load(f) else: cfg = {} text_enc = None if load_text_encoder: from text_encoder import ParvulusTextEncoder t5_cfg = cfg.get("text_encoder", {}) text_enc = ParvulusTextEncoder( cond_dim=t5_cfg.get("cond_dim", 256), model_name=t5_cfg.get("model_name", "google-t5/t5-small"), revision=t5_cfg.get("revision", "df1b051c49625cf57a3d0d8d3863ed4d13564fe4"), ) model = cls( base_channels=cfg.get("base_channels", 48), channel_mults=cfg.get("channel_mults", (1, 2, 2, 4)), num_res_blocks=cfg.get("num_res_blocks", 1), cond_dim=cfg.get("cond_dim", 256), action_dim=cfg.get("action_dim", 7), chunk_size=cfg.get("chunk_size", 8), text_encoder=text_enc, use_video_decoder=load_auxiliary, use_self_decoder=load_auxiliary, use_change_decoder=load_auxiliary, ) if load_auxiliary and (root / "parvulus_full.safetensors").exists(): state = load_file(str(root / "parvulus_full.safetensors")) model.load_state_dict(state, strict=False) else: core_weights_file = root / "parvulus.safetensors" if not core_weights_file.exists(): core_weights_file = root / "model.safetensors" if not core_weights_file.exists(): raise FileNotFoundError(f"Neither parvulus.safetensors nor model.safetensors found in {root}") state = load_file(str(core_weights_file)) model.load_state_dict(state, strict=False) if load_auxiliary: aux_weights_file = root / "parvulus_auxiliary.safetensors" if aux_weights_file.exists(): aux_state = load_file(str(aux_weights_file)) model.load_state_dict(aux_state, strict=False) else: print(f"[Warning] Auxiliary weights requested but {aux_weights_file} was not found.") model = model.to(device) model.eval() return model