Download parvulus.py from aasivas/Parvulus: direct link, hf CLI and curl.
- Browser
- Download file 25 kB
-
https://huggingface.co/aasivas/Parvulus/resolve/main/parvulus.py
- Command line
-
hf download hf://aasivas/Parvulus/parvulus.py
-
curl -L -o parvulus.py https://huggingface.co/aasivas/Parvulus/resolve/main/parvulus.py
25 kB
| """Parvulus Neural Architecture: Dual-POV Flow Matching Model for Robot Manipulation.""" | |
| import json | |
| import math | |
| from pathlib import Path | |
| from typing import Dict, List, NamedTuple, Optional, Sequence, Tuple, Union | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from safetensors.torch import load_file | |
| def sinusoidal_embedding(t: torch.Tensor, dim: int) -> torch.Tensor: | |
| half = dim // 2 | |
| freqs = torch.exp( | |
| -math.log(10000) * torch.arange(half, device=t.device, dtype=torch.float32) / half | |
| ) | |
| args = t.float()[:, None] * freqs[None, :] * 1000.0 | |
| emb = torch.cat([torch.sin(args), torch.cos(args)], dim=-1) | |
| if dim % 2: | |
| emb = F.pad(emb, (0, 1)) | |
| return emb | |
| class TimestepMLP(nn.Module): | |
| def __init__(self, cond_dim: int): | |
| super().__init__() | |
| self.cond_dim = cond_dim | |
| self.mlp = nn.Sequential( | |
| nn.Linear(cond_dim, cond_dim), | |
| nn.SiLU(), | |
| nn.Linear(cond_dim, cond_dim), | |
| ) | |
| def forward(self, t: torch.Tensor) -> torch.Tensor: | |
| return self.mlp(sinusoidal_embedding(t, self.cond_dim)) | |
| class FiLMResBlock2d(nn.Module): | |
| def __init__(self, in_ch: int, out_ch: int, cond_dim: int, groups: int = 8): | |
| super().__init__() | |
| groups_in = min(groups, in_ch) | |
| groups_out = min(groups, out_ch) | |
| self.norm1 = nn.GroupNorm(groups_in, in_ch) | |
| self.conv1 = nn.Conv2d(in_ch, out_ch, 3, padding=1) | |
| self.norm2 = nn.GroupNorm(groups_out, out_ch) | |
| self.conv2 = nn.Conv2d(out_ch, out_ch, 3, padding=1) | |
| self.film = nn.Linear(cond_dim, 2 * out_ch) | |
| self.skip = nn.Conv2d(in_ch, out_ch, 1) if in_ch != out_ch else nn.Identity() | |
| def forward(self, x: torch.Tensor, cond: torch.Tensor) -> torch.Tensor: | |
| h = self.conv1(F.silu(self.norm1(x))) | |
| scale, shift = self.film(cond).chunk(2, dim=-1) | |
| h = self.norm2(h) * (1.0 + scale[:, :, None, None]) + shift[:, :, None, None] | |
| h = self.conv2(F.silu(h)) | |
| return h + self.skip(x) | |
| class CVMixer(nn.Module): | |
| """Bidirectional cross-view mixer with gated contextual residuals.""" | |
| def __init__(self, ch: int): | |
| super().__init__() | |
| self.ch = ch | |
| self.context_conv = nn.Sequential( | |
| nn.Conv2d(2 * ch, ch, 1), | |
| nn.GroupNorm(min(8, ch), ch), | |
| nn.SiLU(), | |
| ) | |
| self.agent_proj = nn.Conv2d(ch, ch, 3, padding=1) | |
| self.agent_gate = nn.Conv2d(ch, ch, 1) | |
| self.wrist_proj = nn.Conv2d(ch, ch, 3, padding=1) | |
| self.wrist_gate = nn.Conv2d(ch, ch, 1) | |
| def forward(self, f_agent: torch.Tensor, f_wrist: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: | |
| joint = torch.cat([f_agent, f_wrist], dim=1) | |
| ctx = self.context_conv(joint) | |
| f_agent_out = f_agent + self.agent_proj(ctx) * torch.sigmoid(self.agent_gate(ctx)) | |
| f_wrist_out = f_wrist + self.wrist_proj(ctx) * torch.sigmoid(self.wrist_gate(ctx)) | |
| f_mixed = torch.cat([f_agent_out, f_wrist_out], dim=1) | |
| return f_agent_out, f_wrist_out, f_mixed | |
| class AttnPool2d(nn.Module): | |
| """Multi-head attention pooling extracting K visual query tokens.""" | |
| def __init__(self, in_ch: int, embed_dim: int = 256, num_queries: int = 4, num_heads: int = 4): | |
| super().__init__() | |
| self.in_proj = nn.Conv2d(in_ch, embed_dim, 1) | |
| self.queries = nn.Parameter(torch.randn(1, num_queries, embed_dim) * 0.02) | |
| self.cross_attn = nn.MultiheadAttention(embed_dim, num_heads=num_heads, batch_first=True) | |
| self.norm = nn.LayerNorm(embed_dim) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| b = x.shape[0] | |
| feats = self.in_proj(x).flatten(2).transpose(1, 2) | |
| q = self.queries.expand(b, -1, -1) | |
| out, _ = self.cross_attn(query=q, key=feats, value=feats) | |
| return self.norm(out) | |
| class DualViewEncoder(nn.Module): | |
| """Hierarchical dual-view vision encoder with cross-view mixing at all stages.""" | |
| def __init__( | |
| self, | |
| base_ch: int = 48, | |
| channel_mults: Sequence[int] = (1, 2, 2, 4), | |
| num_res_blocks: int = 1, | |
| cond_dim: int = 256, | |
| tokens_per_layer: int = 4, | |
| ): | |
| super().__init__() | |
| self.base_ch = base_ch | |
| self.channel_mults = channel_mults | |
| self.cond_dim = cond_dim | |
| self.agent_stem = nn.Conv2d(3, base_ch, 3, padding=1) | |
| self.wrist_stem = nn.Conv2d(3, base_ch, 3, padding=1) | |
| self.agent_stages = nn.ModuleList() | |
| self.wrist_stages = nn.ModuleList() | |
| self.cv_mixers = nn.ModuleList() | |
| self.downsamples = nn.ModuleList() | |
| self.attn_poolers = nn.ModuleList() | |
| self.level_channels: List[int] = [] | |
| in_ch = base_ch | |
| for i, mult in enumerate(channel_mults): | |
| out_ch = base_ch * mult | |
| a_blocks = nn.ModuleList([ | |
| FiLMResBlock2d(in_ch if j == 0 else out_ch, out_ch, cond_dim) | |
| for j in range(num_res_blocks) | |
| ]) | |
| self.agent_stages.append(a_blocks) | |
| w_blocks = nn.ModuleList([ | |
| FiLMResBlock2d(in_ch if j == 0 else out_ch, out_ch, cond_dim) | |
| for j in range(num_res_blocks) | |
| ]) | |
| self.wrist_stages.append(w_blocks) | |
| self.cv_mixers.append(CVMixer(out_ch)) | |
| self.level_channels.append(2 * out_ch) | |
| self.attn_poolers.append(AttnPool2d(2 * out_ch, embed_dim=cond_dim, num_queries=tokens_per_layer)) | |
| self.downsamples.append( | |
| nn.Conv2d(out_ch, out_ch, 3, stride=2, padding=1) if i < len(channel_mults) - 1 else nn.Identity() | |
| ) | |
| in_ch = out_ch | |
| def forward(self, x: torch.Tensor, cond: torch.Tensor) -> Tuple[List[torch.Tensor], torch.Tensor]: | |
| h_agent = self.agent_stem(x[:, 0:3]) | |
| h_wrist = self.wrist_stem(x[:, 3:6]) | |
| skips = [] | |
| visual_token_list = [] | |
| for i, (a_stage, w_stage, mixer, pooler, down) in enumerate( | |
| zip(self.agent_stages, self.wrist_stages, self.cv_mixers, self.attn_poolers, self.downsamples) | |
| ): | |
| if i == 0: | |
| h_agent = F.interpolate(h_agent, scale_factor=0.5, mode='bilinear', align_corners=False) | |
| h_wrist = F.interpolate(h_wrist, scale_factor=0.5, mode='bilinear', align_corners=False) | |
| for a_blk, w_blk in zip(a_stage, w_stage): | |
| h_agent = a_blk(h_agent, cond) | |
| h_wrist = w_blk(h_wrist, cond) | |
| h_agent, h_wrist, f_mixed = mixer(h_agent, h_wrist) | |
| skips.append(f_mixed) | |
| layer_tokens = pooler(f_mixed) | |
| visual_token_list.append(layer_tokens) | |
| h_agent = down(h_agent) | |
| h_wrist = down(h_wrist) | |
| z_vis = torch.cat(visual_token_list, dim=1) | |
| return skips, z_vis | |
| class TransformerDecoderLayer(nn.Module): | |
| def __init__(self, embed_dim: int = 256, num_heads: int = 8, mlp_ratio: int = 4): | |
| super().__init__() | |
| self.norm1 = nn.LayerNorm(embed_dim) | |
| self.self_attn = nn.MultiheadAttention(embed_dim, num_heads=num_heads, batch_first=True) | |
| self.norm2 = nn.LayerNorm(embed_dim) | |
| self.cross_attn = nn.MultiheadAttention(embed_dim, num_heads=num_heads, batch_first=True) | |
| self.norm3 = nn.LayerNorm(embed_dim) | |
| self.mlp = nn.Sequential( | |
| nn.Linear(embed_dim, embed_dim * mlp_ratio), | |
| nn.GELU(), | |
| nn.Linear(embed_dim * mlp_ratio, embed_dim), | |
| ) | |
| self.film = nn.Linear(embed_dim, 2 * embed_dim) | |
| def forward(self, x: torch.Tensor, z_vis: torch.Tensor, cond: torch.Tensor) -> torch.Tensor: | |
| h = x + self.self_attn(query=self.norm1(x), key=self.norm1(x), value=self.norm1(x))[0] | |
| h = h + self.cross_attn(query=self.norm2(h), key=z_vis, value=z_vis)[0] | |
| scale, shift = self.film(cond).unsqueeze(1).chunk(2, dim=-1) | |
| norm_h = self.norm3(h) * (1.0 + scale) + shift | |
| h = h + self.mlp(norm_h) | |
| return h | |
| class OptimizedKVCacheDecoderLayer(nn.Module): | |
| def __init__(self, orig_layer: TransformerDecoderLayer): | |
| super().__init__() | |
| self.norm1 = orig_layer.norm1 | |
| self.self_attn = orig_layer.self_attn | |
| self.norm2 = orig_layer.norm2 | |
| self.cross_attn = orig_layer.cross_attn | |
| self.norm3 = orig_layer.norm3 | |
| self.mlp = orig_layer.mlp | |
| self.film = orig_layer.film | |
| def precompute_kv(self, z_vis: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: | |
| w = self.cross_attn.in_proj_weight | |
| b = self.cross_attn.in_proj_bias | |
| w_k = w[256:512, :] | |
| b_k = b[256:512] if b is not None else None | |
| w_v = w[512:768, :] | |
| b_v = b[512:768] if b is not None else None | |
| k = F.linear(z_vis, w_k, b_k) | |
| v = F.linear(z_vis, w_v, b_v) | |
| return k, v | |
| def forward_with_kv(self, x: torch.Tensor, k_vis: torch.Tensor, v_vis: torch.Tensor, cond: torch.Tensor) -> torch.Tensor: | |
| q1 = self.norm1(x) | |
| h = x + self.self_attn(q1, q1, q1, need_weights=False)[0] | |
| q2 = self.norm2(h) | |
| w_q = self.cross_attn.in_proj_weight[:256, :] | |
| b_q = self.cross_attn.in_proj_bias[:256] if self.cross_attn.in_proj_bias is not None else None | |
| q_proj = F.linear(q2, w_q, b_q) | |
| B, S, D = q_proj.shape | |
| H = 8 | |
| d_k = 32 | |
| q_h = q_proj.view(B, S, H, d_k).transpose(1, 2) | |
| k_h = k_vis.view(B, 16, H, d_k).transpose(1, 2) | |
| v_h = v_vis.view(B, 16, H, d_k).transpose(1, 2) | |
| attn_out = F.scaled_dot_product_attention(q_h, k_h, v_h) | |
| attn_out = attn_out.transpose(1, 2).contiguous().view(B, S, D) | |
| out = F.linear(attn_out, self.cross_attn.out_proj.weight, self.cross_attn.out_proj.bias) | |
| h = h + out | |
| scale, shift = self.film(cond).unsqueeze(1).chunk(2, dim=-1) | |
| norm_h = self.norm3(h) * (1.0 + scale) + shift | |
| h = h + self.mlp(norm_h) | |
| return h | |
| class ActionChunkTransformerDecoder(nn.Module): | |
| def __init__( | |
| self, | |
| action_dim: int = 7, | |
| chunk_size: int = 8, | |
| embed_dim: int = 256, | |
| num_layers: int = 4, | |
| num_heads: int = 8, | |
| ): | |
| super().__init__() | |
| self.action_dim = action_dim | |
| self.chunk_size = chunk_size | |
| self.embed_dim = embed_dim | |
| self.action_in_proj = nn.Linear(action_dim, embed_dim) | |
| self.pos_emb = nn.Parameter(torch.randn(1, chunk_size, embed_dim) * 0.02) | |
| self.layers = nn.ModuleList([ | |
| TransformerDecoderLayer(embed_dim, num_heads=num_heads) | |
| for _ in range(num_layers) | |
| ]) | |
| self.final_norm = nn.LayerNorm(embed_dim) | |
| self.out_head = nn.Linear(embed_dim, action_dim) | |
| def forward(self, z_sigma: torch.Tensor, z_vis: torch.Tensor, cond: torch.Tensor) -> torch.Tensor: | |
| b, s, d = z_sigma.shape | |
| h = self.action_in_proj(z_sigma) + self.pos_emb[:, :s, :] | |
| for layer in self.layers: | |
| h = layer(h, z_vis, cond) | |
| return self.out_head(self.final_norm(h)) | |
| class MultiScaleUNetDecoder(nn.Module): | |
| def __init__( | |
| self, | |
| level_channels: Sequence[int], | |
| cond_dim: int = 256, | |
| out_channels: int = 6, | |
| num_res_blocks: int = 1, | |
| ): | |
| super().__init__() | |
| rev_levels = list(reversed(level_channels)) | |
| self.stages = nn.ModuleList() | |
| self.upsamples = nn.ModuleList() | |
| in_ch = rev_levels[0] | |
| for i, lvl_ch in enumerate(rev_levels): | |
| blocks = nn.ModuleList() | |
| for j in range(num_res_blocks): | |
| b_in = in_ch + (lvl_ch if j == 0 else 0) | |
| blocks.append(FiLMResBlock2d(b_in, lvl_ch, cond_dim)) | |
| in_ch = lvl_ch | |
| self.stages.append(blocks) | |
| self.upsamples.append( | |
| nn.Sequential( | |
| nn.Upsample(scale_factor=2, mode='nearest'), | |
| nn.Conv2d(lvl_ch, lvl_ch, 3, padding=1) | |
| ) if i < len(rev_levels) - 1 else nn.Identity() | |
| ) | |
| self.final_upsample = nn.Sequential( | |
| nn.Upsample(scale_factor=2, mode='nearest'), | |
| nn.Conv2d(rev_levels[-1], rev_levels[-1], 3, padding=1), | |
| nn.GroupNorm(min(8, rev_levels[-1]), rev_levels[-1]), | |
| nn.SiLU(), | |
| nn.Conv2d(rev_levels[-1], out_channels, 3, padding=1), | |
| ) | |
| def forward(self, skips: Sequence[torch.Tensor], cond: torch.Tensor) -> torch.Tensor: | |
| rev_skips = list(reversed(skips)) | |
| x = rev_skips[0] | |
| for stage, up, skip in zip(self.stages, self.upsamples, rev_skips): | |
| for j, block in enumerate(stage): | |
| if j == 0: | |
| x = torch.cat([x, skip], dim=1) | |
| x = block(x, cond) | |
| x = up(x) | |
| return self.final_upsample(x) | |
| class ParvulusOutput(NamedTuple): | |
| action_flow: Optional[torch.Tensor] = None | |
| video_flow: Optional[torch.Tensor] = None | |
| self_recon: Optional[torch.Tensor] = None | |
| change_pred: Optional[torch.Tensor] = None | |
| class ParvulusModel(nn.Module): | |
| """Parvulus: Dual-POV Flow Matching Robot Manipulation Model.""" | |
| def __init__( | |
| self, | |
| base_channels: int = 48, | |
| channel_mults: Sequence[int] = (1, 2, 2, 4), | |
| num_res_blocks: int = 1, | |
| cond_dim: int = 256, | |
| action_dim: int = 7, | |
| chunk_size: int = 8, | |
| text_encoder: Optional[nn.Module] = None, | |
| use_video_decoder: bool = False, | |
| use_self_decoder: bool = False, | |
| use_change_decoder: bool = False, | |
| ): | |
| super().__init__() | |
| self.cond_dim = cond_dim | |
| self.action_dim = action_dim | |
| self.chunk_size = chunk_size | |
| self.text_encoder = text_encoder | |
| self.encoder = DualViewEncoder( | |
| base_ch=base_channels, | |
| channel_mults=channel_mults, | |
| num_res_blocks=num_res_blocks, | |
| cond_dim=cond_dim, | |
| tokens_per_layer=4, | |
| ) | |
| self.action_t_mlp = TimestepMLP(cond_dim) | |
| self.video_t_mlp = TimestepMLP(cond_dim) | |
| self.action_decoder = ActionChunkTransformerDecoder( | |
| action_dim=action_dim, | |
| chunk_size=chunk_size, | |
| embed_dim=cond_dim, | |
| num_layers=4, | |
| num_heads=8, | |
| ) | |
| self.video_decoder = MultiScaleUNetDecoder( | |
| level_channels=self.encoder.level_channels, | |
| cond_dim=cond_dim, | |
| out_channels=6, | |
| num_res_blocks=num_res_blocks, | |
| ) if use_video_decoder else None | |
| self.self_decoder = MultiScaleUNetDecoder( | |
| level_channels=self.encoder.level_channels, | |
| cond_dim=cond_dim, | |
| out_channels=6, | |
| num_res_blocks=num_res_blocks, | |
| ) if use_self_decoder else None | |
| self.change_decoder = MultiScaleUNetDecoder( | |
| level_channels=self.encoder.level_channels, | |
| cond_dim=cond_dim, | |
| out_channels=2, | |
| num_res_blocks=num_res_blocks, | |
| ) if use_change_decoder else None | |
| self._kv_layers: Optional[nn.ModuleList] = None | |
| def _get_kv_layers(self) -> nn.ModuleList: | |
| if self._kv_layers is None: | |
| self._kv_layers = nn.ModuleList([ | |
| OptimizedKVCacheDecoderLayer(l) for l in self.action_decoder.layers | |
| ]) | |
| return self._kv_layers | |
| def _text_cond( | |
| self, | |
| texts: Optional[Sequence[str]], | |
| text_emb: Optional[torch.Tensor], | |
| batch: int, | |
| device: torch.device, | |
| ) -> torch.Tensor: | |
| if text_emb is not None: | |
| return text_emb | |
| if texts is not None and self.text_encoder is not None: | |
| return self.text_encoder(texts) | |
| return torch.zeros(batch, self.cond_dim, device=device) | |
| def forward( | |
| self, | |
| obs_input: torch.Tensor, | |
| z_sigma: Optional[torch.Tensor] = None, | |
| action_sigma: Optional[torch.Tensor] = None, | |
| next_target: Optional[torch.Tensor] = None, | |
| video_tau: Optional[torch.Tensor] = None, | |
| texts: Optional[Sequence[str]] = None, | |
| text_emb: Optional[torch.Tensor] = None, | |
| compute_action: bool = True, | |
| compute_video: bool = True, | |
| compute_self_recon: bool = True, | |
| compute_change: bool = True, | |
| ) -> ParvulusOutput: | |
| b = obs_input.shape[0] | |
| device = obs_input.device | |
| t_emb = self._text_cond(texts, text_emb, b, device) | |
| skips_obs, z_vis = self.encoder(obs_input, t_emb) | |
| action_flow = None | |
| if compute_action and z_sigma is not None and action_sigma is not None: | |
| act_cond = t_emb + self.action_t_mlp(action_sigma) | |
| action_flow = self.action_decoder(z_sigma, z_vis, act_cond) | |
| video_flow = None | |
| if compute_video and self.video_decoder is not None and video_tau is not None: | |
| vid_cond = t_emb + self.video_t_mlp(video_tau) | |
| video_flow = self.video_decoder(skips_obs, vid_cond) | |
| self_recon = None | |
| if compute_self_recon and self.self_decoder is not None: | |
| self_recon = self.self_decoder(skips_obs, t_emb) | |
| change_pred = None | |
| if compute_change and self.change_decoder is not None: | |
| change_pred = self.change_decoder(skips_obs, t_emb) | |
| return ParvulusOutput( | |
| action_flow=action_flow, | |
| video_flow=video_flow, | |
| self_recon=self_recon, | |
| change_pred=change_pred, | |
| ) | |
| def predict_action_chunk( | |
| self, | |
| obs: torch.Tensor, | |
| text_emb: Optional[torch.Tensor] = None, | |
| texts: Optional[Sequence[str]] = None, | |
| num_ode_steps: int = 10, | |
| ode_solver: str = "euler", | |
| use_kv_cache: bool = False, | |
| z_init: Optional[torch.Tensor] = None, | |
| ) -> torch.Tensor: | |
| b = obs.shape[0] | |
| device = obs.device | |
| t_emb = self._text_cond(texts, text_emb, b, device) | |
| _, z_vis = self.encoder(obs, t_emb) | |
| if z_init is not None: | |
| z = z_init.clone() | |
| else: | |
| z = torch.randn(b, self.chunk_size, self.action_dim, device=device) | |
| if use_kv_cache and ode_solver == "euler": | |
| kv_layers = self._get_kv_layers() | |
| kv_caches = [layer.precompute_kv(z_vis) for layer in kv_layers] | |
| dt = 1.0 / num_ode_steps | |
| for step in range(num_ode_steps): | |
| sigma = torch.full((b,), step * dt, device=device) | |
| cond = t_emb + self.action_t_mlp(sigma) | |
| h = self.action_decoder.action_in_proj(z) + self.action_decoder.pos_emb[:, :self.chunk_size, :] | |
| for layer, (k_v, v_v) in zip(kv_layers, kv_caches): | |
| h = layer.forward_with_kv(h, k_v, v_v, cond) | |
| v_pred = self.action_decoder.out_head(self.action_decoder.final_norm(h)) | |
| z = z + v_pred * dt | |
| return z.clamp(-1.0, 1.0) | |
| if ode_solver == "euler": | |
| dt = 1.0 / num_ode_steps | |
| for step in range(num_ode_steps): | |
| sigma = torch.full((b,), step * dt, device=device) | |
| act_cond = t_emb + self.action_t_mlp(sigma) | |
| v_pred = self.action_decoder(z, z_vis, act_cond) | |
| z = z + v_pred * dt | |
| elif ode_solver == "heun": | |
| dt = 1.0 / num_ode_steps | |
| for step in range(num_ode_steps): | |
| t_curr = step * dt | |
| t_next = (step + 1) * dt | |
| sigma_curr = torch.full((b,), t_curr, device=device) | |
| sigma_next = torch.full((b,), t_next, device=device) | |
| act_cond_curr = t_emb + self.action_t_mlp(sigma_curr) | |
| v1 = self.action_decoder(z, z_vis, act_cond_curr) | |
| z_euler = z + v1 * dt | |
| act_cond_next = t_emb + self.action_t_mlp(sigma_next) | |
| v2 = self.action_decoder(z_euler, z_vis, act_cond_next) | |
| z = z + 0.5 * dt * (v1 + v2) | |
| else: | |
| raise ValueError(f"Unsupported ode_solver: {ode_solver}") | |
| return z.clamp(-1.0, 1.0) | |
| def predict_next_frame( | |
| self, | |
| obs: torch.Tensor, | |
| text_emb: Optional[torch.Tensor] = None, | |
| texts: Optional[Sequence[str]] = None, | |
| num_ode_steps: int = 10, | |
| x_init: Optional[torch.Tensor] = None, | |
| ) -> torch.Tensor: | |
| if self.video_decoder is None: | |
| raise RuntimeError("video_decoder is not initialized. Pass use_video_decoder=True when loading model.") | |
| b = obs.shape[0] | |
| device = obs.device | |
| t_emb = self._text_cond(texts, text_emb, b, device) | |
| skips, _ = self.encoder(obs, t_emb) | |
| if x_init is not None: | |
| x = x_init.clone() | |
| else: | |
| x = torch.randn(b, 6, 256, 256, device=device) | |
| dt = 1.0 / num_ode_steps | |
| for step in range(num_ode_steps): | |
| tau = torch.full((b,), step * dt, device=device) | |
| cond = t_emb + self.video_t_mlp(tau) | |
| v_pred = self.video_decoder(skips, cond) | |
| x = x + v_pred * dt | |
| return x.clamp(-1.0, 1.0) | |
| def predict_change( | |
| self, | |
| obs: torch.Tensor, | |
| text_emb: Optional[torch.Tensor] = None, | |
| texts: Optional[Sequence[str]] = None, | |
| ) -> torch.Tensor: | |
| if self.change_decoder is None: | |
| raise RuntimeError("change_decoder is not initialized. Pass use_change_decoder=True when loading model.") | |
| b = obs.shape[0] | |
| device = obs.device | |
| t_emb = self._text_cond(texts, text_emb, b, device) | |
| skips, _ = self.encoder(obs, t_emb) | |
| return self.change_decoder(skips, t_emb) | |
| def reconstruct_observation( | |
| self, | |
| obs: torch.Tensor, | |
| text_emb: Optional[torch.Tensor] = None, | |
| texts: Optional[Sequence[str]] = None, | |
| ) -> torch.Tensor: | |
| if self.self_decoder is None: | |
| raise RuntimeError("self_decoder is not initialized. Pass use_self_decoder=True when loading model.") | |
| b = obs.shape[0] | |
| device = obs.device | |
| t_emb = self._text_cond(texts, text_emb, b, device) | |
| skips, _ = self.encoder(obs, t_emb) | |
| return self.self_decoder(skips, t_emb) | |
| def from_pretrained( | |
| cls, | |
| pretrained_path: Union[str, Path] = ".", | |
| load_auxiliary: bool = False, | |
| load_text_encoder: bool = True, | |
| device: Union[str, torch.device] = "cpu", | |
| ) -> "ParvulusModel": | |
| root = Path(pretrained_path) | |
| cfg_file = root / "config.json" | |
| if cfg_file.exists(): | |
| with open(cfg_file, "r") as f: | |
| cfg = json.load(f) | |
| else: | |
| cfg = {} | |
| text_enc = None | |
| if load_text_encoder: | |
| from text_encoder import ParvulusTextEncoder | |
| t5_cfg = cfg.get("text_encoder", {}) | |
| text_enc = ParvulusTextEncoder( | |
| cond_dim=t5_cfg.get("cond_dim", 256), | |
| model_name=t5_cfg.get("model_name", "google-t5/t5-small"), | |
| revision=t5_cfg.get("revision", "df1b051c49625cf57a3d0d8d3863ed4d13564fe4"), | |
| ) | |
| model = cls( | |
| base_channels=cfg.get("base_channels", 48), | |
| channel_mults=cfg.get("channel_mults", (1, 2, 2, 4)), | |
| num_res_blocks=cfg.get("num_res_blocks", 1), | |
| cond_dim=cfg.get("cond_dim", 256), | |
| action_dim=cfg.get("action_dim", 7), | |
| chunk_size=cfg.get("chunk_size", 8), | |
| text_encoder=text_enc, | |
| use_video_decoder=load_auxiliary, | |
| use_self_decoder=load_auxiliary, | |
| use_change_decoder=load_auxiliary, | |
| ) | |
| if load_auxiliary and (root / "parvulus_full.safetensors").exists(): | |
| state = load_file(str(root / "parvulus_full.safetensors")) | |
| model.load_state_dict(state, strict=False) | |
| else: | |
| core_weights_file = root / "parvulus.safetensors" | |
| if not core_weights_file.exists(): | |
| core_weights_file = root / "model.safetensors" | |
| if not core_weights_file.exists(): | |
| raise FileNotFoundError(f"Neither parvulus.safetensors nor model.safetensors found in {root}") | |
| state = load_file(str(core_weights_file)) | |
| model.load_state_dict(state, strict=False) | |
| if load_auxiliary: | |
| aux_weights_file = root / "parvulus_auxiliary.safetensors" | |
| if aux_weights_file.exists(): | |
| aux_state = load_file(str(aux_weights_file)) | |
| model.load_state_dict(aux_state, strict=False) | |
| else: | |
| print(f"[Warning] Auxiliary weights requested but {aux_weights_file} was not found.") | |
| model = model.to(device) | |
| model.eval() | |
| return model | |