code-HFD / src /psi /models /psi0.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
162 kB
import os import json import copy import contextlib from tqdm import tqdm from typing import List, Optional, Tuple, Union, Dict, Any from typing import Any, Dict, List, Optional, Tuple, Union from typing_extensions import Self import torch import torch.nn as nn import torch.nn.functional as F import numpy as np from PIL import Image from dataclasses import dataclass from diffusers.utils.outputs import BaseOutput from diffusers.configuration_utils import ConfigMixin, FrozenDict, register_to_config # from diffusers.utils.torch_utils import maybe_allow_in_graph from diffusers.models.modeling_utils import ModelMixin # from diffusers.models.transformers.transformer_sd3 import SD3Transformer2DModel from diffusers.models.embeddings import CombinedTimestepTextProjEmbeddings # from diffusers.models.normalization import AdaLayerNorm, AdaLayerNormZero, AdaLayerNormContinuous from diffusers.models.embeddings import CombinedTimestepLabelEmbeddings from diffusers.models.normalization import FP32LayerNorm from diffusers.models.attention_processor import Attention, JointAttnProcessor2_0 from diffusers.models.attention import FeedForward, _chunked_feed_forward # from dit_policy.data4robotics.models.diffusion import _TimeNetwork from psi.config.config import LaunchConfig from transformers import ( Qwen3VLForConditionalGeneration, Qwen2_5OmniThinkerForConditionalGeneration, Qwen2_5OmniProcessor, AutoConfig, AutoProcessor, Qwen2TokenizerFast, Qwen3VLProcessor, ) from qwen_vl_utils import process_vision_info from psi.utils import initialize_overwatch, count_parameters # from InternVLA.model.modules.action_model.DiT_modules.models import DiT # from InternVLA.model.modules.projector.QFormer import CrossAttentionBlock overwatch = initialize_overwatch(__name__) # from psi.config.model import ModelConfig, InternVLA_M1_ModelConfig # from psi.learn.vlms import _QWen_VL_Interface from transformers.models.siglip import SiglipModel from transformers.models.dinov2 import Dinov2Model # from vlt.transformers.vla import ActionTransformerModel from diffusers.utils import logging logger = logging.get_logger(__name__) # pylint: disable=invalid-name FIXME why this not works any more? QWEN3VL_VARIANT = "Qwen/Qwen3-VL-2B-Instruct" OMNI_DEFAULT_SYSTEM_PROMPT = ( "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, " "capable of perceiving auditory and visual inputs, as well as generating text and speech." ) def _validate_rtc_inference_contract( *, action_horizon: int, action_exec_horizon: int, inference_delay: int, max_delay: int, ) -> int: """Return the configured RTC overlap or fail on train/deploy mismatch.""" horizon = int(action_horizon) execute = int(action_exec_horizon) delay = int(inference_delay) delay_limit = int(max_delay) if not 0 < execute < horizon: raise ValueError( "RTC requires 0 < action_exec_horizon < action_horizon, " f"got execute={execute}, horizon={horizon}" ) expected_overlap = horizon - execute if delay != expected_overlap: raise ValueError( "RTC inference_delay must equal the overlap used by training and " "deployment: " f"got inference_delay={delay}, expected={expected_overlap} " f"(action_horizon={horizon}, action_exec_horizon={execute})" ) if expected_overlap >= delay_limit: raise ValueError( "RTC overlap must be smaller than max_delay, " f"got overlap={expected_overlap}, max_delay={delay_limit}" ) return expected_overlap @dataclass class HumanFoundationModelOutput(BaseOutput): action: "torch.Tensor" # noqa: F821 def to_tuple(self): return (None, self.action) @dataclass class ActionTransformerModelOutput(BaseOutput): action: "torch.Tensor" # noqa: F821 def to_tuple(self): return (None, self.action) class _TimeNetwork(nn.Module): def __init__(self, time_dim, out_dim, learnable_w=False): assert time_dim % 2 == 0, "time_dim must be even!" half_dim = int(time_dim // 2) super().__init__() w = np.log(10000) / (half_dim - 1) w = torch.exp(torch.arange(half_dim) * -w).float() # self.register_parameter("w", nn.Parameter(w, requires_grad=learnable_w)) self.w = nn.Parameter(w, requires_grad=learnable_w) self.out_net = nn.Sequential( nn.Linear(time_dim, out_dim), nn.SiLU(), nn.Linear(out_dim, out_dim) ) def forward(self, x): # assert len(x.shape) == 1, "assumes 1d input timestep array" # RTC: support 2D timesteps (B,Tp) x = x[..., None] * self.w x = torch.cat((torch.cos(x), torch.sin(x)), dim=-1) return self.out_net(x) class RMSNorm(nn.Module): def __init__(self, dim, eps: float, elementwise_affine: bool = True): super().__init__() self.eps = eps if isinstance(dim, int): dim = (dim,) self.dim = torch.Size(dim) if elementwise_affine: self.weight = nn.Parameter(torch.ones(dim)) else: self.weight = None def forward(self, hidden_states): input_dtype = hidden_states.dtype variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True) hidden_states = hidden_states * torch.rsqrt(variance + self.eps) if self.weight is not None: # convert into half-precision if necessary if self.weight.dtype in [torch.float16, torch.bfloat16]: hidden_states = hidden_states.to(self.weight.dtype) hidden_states = hidden_states * self.weight else: hidden_states = hidden_states.to(input_dtype) return hidden_states if torch.__version__ >= "2.1.0": LayerNorm = nn.LayerNorm # type:ignore else: # Has optional bias parameter compared to torch layer norm # TODO: replace with torch layernorm once min required torch version >= 2.1 class LayerNorm(nn.Module): def __init__(self, dim, eps: float = 1e-5, elementwise_affine: bool = True, bias: bool = True): super().__init__() self.eps = eps if isinstance(dim, int): dim = (dim,) self.dim = torch.Size(dim) if elementwise_affine: self.weight = nn.Parameter(torch.ones(dim)) self.bias = nn.Parameter(torch.zeros(dim)) if bias else None else: self.weight = None self.bias = None def forward(self, input): return F.layer_norm(input, self.dim, self.weight, self.bias, self.eps) class AdaLayerNormContinuous(nn.Module): def __init__( self, embedding_dim: int, conditioning_embedding_dim: int, # NOTE: It is a bit weird that the norm layer can be configured to have scale and shift parameters # because the output is immediately scaled and shifted by the projected conditioning embeddings. # Note that AdaLayerNorm does not let the norm layer have scale and shift parameters. # However, this is how it was implemented in the original code, and it's rather likely you should # set `elementwise_affine` to False. elementwise_affine=True, eps=1e-5, bias=True, norm_type="layer_norm", ): super().__init__() self.silu = nn.SiLU() self.linear = nn.Linear(conditioning_embedding_dim, embedding_dim * 2, bias=bias) if norm_type == "layer_norm": self.norm = LayerNorm(embedding_dim, eps, elementwise_affine, bias) elif norm_type == "rms_norm": self.norm = RMSNorm(embedding_dim, eps, elementwise_affine) else: raise ValueError(f"unknown norm_type {norm_type}") def forward(self, x: torch.Tensor, conditioning_embedding: torch.Tensor) -> torch.Tensor: # convert back to the original dtype in case `conditioning_embedding`` is upcasted to float32 (needed for hunyuanDiT) if len(conditioning_embedding.shape) == 2: # (B, D) conditioning_embedding = conditioning_embedding.unsqueeze(1) # (B, 1, D) elif len(conditioning_embedding.shape) == 3: # (B, T, D) pass else: raise ValueError(f"Invalid shape of conditioning_embedding: {conditioning_embedding.shape}") emb = self.linear(self.silu(conditioning_embedding).to(x.dtype)) scale, shift = torch.chunk(emb, 2, dim=-1) x = self.norm(x) * (1 + scale) + shift return x class AdaLayerNormZero(nn.Module): r""" Norm layer adaptive layer norm zero (adaLN-Zero). Parameters: embedding_dim (`int`): The size of each embedding vector. num_embeddings (`int`): The size of the embeddings dictionary. """ def __init__(self, embedding_dim: int, num_embeddings: Optional[int] = None, norm_type="layer_norm", bias=True): super().__init__() if num_embeddings is not None: self.emb = CombinedTimestepLabelEmbeddings(num_embeddings, embedding_dim) else: self.emb = None self.silu = nn.SiLU() self.linear = nn.Linear(embedding_dim, 6 * embedding_dim, bias=bias) # This module is adaLN-Zero, so every residual modulation must start as # an exact no-op. The previous implementation used PyTorch's default # random Linear initialization despite the class name, opening all # attention/MLP residual gates from the first optimizer step. nn.init.zeros_(self.linear.weight) if self.linear.bias is not None: nn.init.zeros_(self.linear.bias) if norm_type == "layer_norm": self.norm = nn.LayerNorm(embedding_dim, elementwise_affine=False, eps=1e-6) elif norm_type == "fp32_layer_norm": self.norm = FP32LayerNorm(embedding_dim, elementwise_affine=False, bias=False) else: raise ValueError( f"Unsupported `norm_type` ({norm_type}) provided. Supported ones are: 'layer_norm', 'fp32_layer_norm'." ) def forward( self, x: torch.Tensor, timestep: Optional[torch.Tensor] = None, class_labels: Optional[torch.LongTensor] = None, hidden_dtype: Optional[torch.dtype] = None, emb: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: if self.emb is not None: emb = self.emb(timestep, class_labels, hidden_dtype=hidden_dtype) assert False, "Not implemented with emb shape == 3, (B, T, D)" if len(emb.shape) == 2: # (B, D) emb = emb.unsqueeze(1) # (B, 1, D) elif len(emb.shape) == 3: # (B, T, D) pass else: raise ValueError(f"Invalid shape of emb: {emb.shape}") emb = self.linear(self.silu(emb)) shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = emb.chunk(6, dim=-1) x = self.norm(x) * (1 + scale_msa) + shift_msa return x, gate_msa, shift_mlp, scale_mlp, gate_mlp class _PositionalEncoding(nn.Module): def __init__(self, d_model, max_len=5000): super().__init__() # Compute the positional encodings once in log space pe = torch.zeros(max_len, d_model) position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) div_term = torch.exp( torch.arange(0, d_model, 2, dtype=torch.float) * -(np.log(10000.0) / d_model) ) pe[:, 0::2] = torch.sin(position * div_term) pe[:, 1::2] = torch.cos(position * div_term) pe = pe.unsqueeze(0).transpose(0, 1) # self.register_buffer("pe", torch.clone(pe)) # <-- SONGLIN: this will cause problems when accelerate.prepare # self.register_parameter("pe", nn.Parameter(pe, requires_grad=False)) self.pe = nn.Parameter(pe, requires_grad=False) def forward(self, x, valid_mask=None): """ Args: x: Tensor of shape (seq_len, batch_size, d_model) Returns: Tensor of shape (seq_len, batch_size, d_model) with positional encodings added """ if valid_mask is not None: if tuple(valid_mask.shape) != (x.shape[1], x.shape[0]): raise ValueError("Position mask shape must be (batch, sequence)") # Appended control/state tokens follow each source's valid AV # tokens, not a neighbouring source's padding. No GPU scalar read. positions = (valid_mask.to(dtype=torch.long).cumsum(-1) - 1).clamp_min(0) return self.pe[:, 0][positions].transpose(0, 1).detach() pe = self.pe[: x.shape[0]] pe = pe.repeat((1, x.shape[1], 1)) return pe.detach().clone() class FilmConditioning(nn.Module): """Layer that adds FiLM conditioning. This is intended to be applied after a convolutional layer. It will learn a multiplicative and an additive factor to be applied to each channel of the convolution's output. Conv layer can be rank 2 or 4. For further details, see: https://arxiv.org/abs/1709.07871 """ def __init__(self, in_dim: int, num_channels: int): """Constructs a FiLM conditioning layer. Args: num_channels: Number of filter channels to expect in the input. """ super(FilmConditioning, self).__init__() # Note that we initialize with zeros because empirically we have found # this works better than initializing with glorot. self._projection_add = nn.Linear(in_dim, num_channels) self._projection_mult = nn.Linear(in_dim, num_channels) for m in self.modules(): if isinstance(m, nn.Linear): nn.init.zeros_(m.weight) nn.init.zeros_(m.bias) def forward(self, conv_filters, conditioning): assert len(conditioning.shape) == 2 projected_cond_add = self._projection_add(conditioning) projected_cond_mult = self._projection_mult(conditioning) if len(conv_filters.shape) == 4: # [B, D] -> [B, D, 1, 1] projected_cond_add = projected_cond_add.unsqueeze(-1).unsqueeze(-1) projected_cond_mult = projected_cond_mult.unsqueeze(-1).unsqueeze(-1) elif len(conv_filters.shape) == 3: # [B, D] -> [B, 1, D] projected_cond_add = projected_cond_add.unsqueeze(1) projected_cond_mult = projected_cond_mult.unsqueeze(1) else: assert len(conv_filters.shape) == 2 # Original FiLM paper argues that 1 + gamma centers the initialization at # identity transform. result = (1 + projected_cond_mult) * conv_filters + projected_cond_add return result class JointVLAAttnProcessor: #(nn.Module): """Attention processor used typically in processing the SD3-like self-attention projections.""" def __init__( self, query_dim: int, heads: int = 8, kv_heads: Optional[int] = None, dim_head: int = 64, out_dim: Optional[int] = None, dropout: float = 0.0, bias: bool = False, out_bias: bool = True, context_pre_only=None, ): super().__init__() if not hasattr(F, "scaled_dot_product_attention"): raise ImportError("JointAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.") # self.inner_dim = out_dim if out_dim is not None else dim_head * heads # self.inner_kv_dim = self.inner_dim if kv_heads is None else dim_head * kv_heads # self.out_dim = out_dim if out_dim is not None else query_dim # self.to_q_lang = nn.Linear(query_dim, self.inner_dim, bias=bias) # self.to_k_lang = nn.Linear(query_dim, self.inner_kv_dim, bias=bias) # self.to_v_lang = nn.Linear(query_dim, self.inner_kv_dim, bias=bias) def __call__( self, attn: Attention, hidden_states: torch.Tensor, encoder_hidden_states: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, *args, **kwargs, ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: residual = hidden_states batch_size = hidden_states.shape[0] # `sample` projections. query = attn.to_q(hidden_states) assert attn.to_k is not None key = attn.to_k(hidden_states) assert attn.to_v is not None value = attn.to_v(hidden_states) inner_dim = key.shape[-1] head_dim = inner_dim // attn.heads query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) if attn.norm_q is not None: query = attn.norm_q(query) if attn.norm_k is not None: key = attn.norm_k(key) # `context` projections. if encoder_hidden_states is not None: encoder_hidden_states_query_proj = attn.add_q_proj(encoder_hidden_states) encoder_hidden_states_key_proj = attn.add_k_proj(encoder_hidden_states) encoder_hidden_states_value_proj = attn.add_v_proj(encoder_hidden_states) encoder_hidden_states_query_proj = encoder_hidden_states_query_proj.view( batch_size, -1, attn.heads, head_dim ).transpose(1, 2) encoder_hidden_states_key_proj = encoder_hidden_states_key_proj.view( batch_size, -1, attn.heads, head_dim ).transpose(1, 2) encoder_hidden_states_value_proj = encoder_hidden_states_value_proj.view( batch_size, -1, attn.heads, head_dim ).transpose(1, 2) if attn.norm_added_q is not None: encoder_hidden_states_query_proj = attn.norm_added_q(encoder_hidden_states_query_proj) if attn.norm_added_k is not None: encoder_hidden_states_key_proj = attn.norm_added_k(encoder_hidden_states_key_proj) query = torch.cat([query, encoder_hidden_states_query_proj], dim=2) key = torch.cat([key, encoder_hidden_states_key_proj], dim=2) value = torch.cat([value, encoder_hidden_states_value_proj], dim=2) if attention_mask is not None: assert attention_mask.dtype == torch.float32 attn_mask = torch.cat([ torch.ones(hidden_states.shape[0], 1, 1, hidden_states.shape[1], device=attention_mask.device).to(torch.bool), # (B, 1, 1, S1) (attention_mask == 1)[:, None, None, :] # (B, 1, 1, S2) ], dim=-1) else: attn_mask = None hidden_states = F.scaled_dot_product_attention(query, key, value, dropout_p=0.0, is_causal=False, attn_mask=attn_mask) # type: ignore hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) # type: ignore hidden_states = hidden_states.to(query.dtype) # type: ignore # Split the attention outputs. hidden_states, encoder_hidden_states = ( hidden_states[:, : residual.shape[1]], hidden_states[:, residual.shape[1] :], ) # type: ignore if not attn.context_pre_only: encoder_hidden_states = attn.to_add_out(encoder_hidden_states) # linear proj hidden_states = attn.to_out[0](hidden_states) # dropout hidden_states = attn.to_out[1](hidden_states) return hidden_states, encoder_hidden_states class ObservationProjection(nn.Module): # FIXME naming """ VLT observation projector Args: ... """ def __init__( self, action_pred_horizon: int = 50, # Tp action_dim: int = 7, # in dim output_dim: int = 512, # out dim, hidden_dim: int = 1280, n_conditions: int = 1, # num image of contitions timestep_in_dim: int = 320, timestep_flip_sin_to_cos: bool = True, timestep_freq_shift: int = 0, token_fusion: str = "concat", resnet_store_path: str = "cache/visual_features/resnet18/IN_1M_resnet18.pth", odim: int = 32, view_feature_dim: int = 1920, # VLM输出hidden_dim维度 use_film: bool = False ): super().__init__() # TODO 1. compare # self.time_net = _TimeNetwork(time_dim=256, out_dim=512) # self.time_proj = Timesteps(timestep_in_dim, timestep_flip_sin_to_cos, timestep_freq_shift) # self.time_embedding = TimestepEmbedding(timestep_in_dim, output_dim, act_fn="silu") self.action_pred_horizon = action_pred_horizon self.action_dim = action_dim # self.ac_proj = nn.Sequential( # nn.Linear(action_dim, action_dim), # nn.GELU(approximate="tanh"), # nn.Linear(action_dim, output_dim), # ) # _DiTNoiseNet.__init__ self.enc_pos = _PositionalEncoding(d_model=output_dim) # Opt in through the saved model config; old checkpoints keep their # original padding-sensitive positions for reproducible inference. self.mask_aware_positions = False # agent.__init__ n_cams = 0 # 1 imgs_per_cam = 1 self.camera_indices = [] # [0] self._n_cams = n_cams self._share_cam_features = False # model_cfg.share_cam_features early_fusion = False text_dim = 768 # odim = 15 self.token_fusion = token_fusion if self._share_cam_features: ... else: if n_conditions > 0: from dit_policy.data4robotics.models.resnet import ResNet features = ResNet( size=18, weights="IMAGENET1K_V1", restore_path=resnet_store_path, avg_pool=False, conv_repeat=0, norm_cfg={ "name": "group_norm", "num_groups": 16, } ) else: features = None feat_list = [copy.deepcopy(features) for _ in range(0, n_conditions)] # + [copy.deepcopy(features) for _ in range(1, n_cams)] + [copy.deepcopy(features) for _ in range(0, n_conditions)] self.visual_features = nn.ModuleList(feat_list) self.views_proj = nn.Linear(view_feature_dim, output_dim, bias=True) # TODO configure 1920 # boqian_fix # TODO abl: more layers # view proj # self.fc1 = nn.Linear(self.vision_dim, initial_projection_dim, bias=True) # self.fc2 = nn.Linear(initial_projection_dim, self.llm_dim, bias=True) # self.fc3 = nn.Linear(self.llm_dim, self.llm_dim, bias=True) # self.act_fn1 = nn.GELU() # self.act_fn2 = nn.GELU() # self.views_proj = nn.Sequential( # nn.Conv2d(16, output_dim, kernel_size=(2, 2), stride=2, bias=True), # nn.ReLU(), # nn.Conv2d(output_dim, output_dim, kernel_size=(2, 2), stride=2, bias=True) # ) # self.traj2d_proj = nn.ModuleList([ # nn.Conv2d(16, 512, kernel_size=(2, 2), stride=2, bias=True), # nn.ReLU(), # nn.Conv2d(512, output_dim, kernel_size=(2, 2), stride=2, bias=True), # nn.ReLU(), # nn.Linear(output_dim, output_dim, bias=True), # nn.LayerNorm(output_dim) # ]) self.early_fusion = early_fusion self.imgs_per_cam = imgs_per_cam imgs_per_cam = 1 if early_fusion else imgs_per_cam self._n_conditions = n_conditions self.embed_proj = nn.Identity() if self.token_fusion != "perceiver" and self._n_conditions > 0: assert features is not None self._token_dim = features.embed_dim if output_dim != self._token_dim: self.embed_proj = nn.Linear(self._token_dim, output_dim, bias=False) else: self._token_dim = 0 self._n_tokens = imgs_per_cam * n_cams * self._token_dim + n_conditions * self._token_dim self.traj_n_token = self._token_dim self.feat_norm = nn.LayerNorm(output_dim, elementwise_affine=False, eps=1e-6) # feat_norm = None self.feat_traj_norm = nn.LayerNorm(output_dim, elementwise_affine=False, eps=1e-6) # feat_norm = None # self.traj2d_proj = nn.ModuleList([ # nn.Linear(output_dim, output_dim) for _ in range(2) # ]) # self.traj2d_proj = nn.Conv2d( # 16, output_dim, kernel_size=(2, 2), stride=2, bias=True # ) if self.token_fusion == "cross": self.attn = Attention( query_dim=self._token_dim, cross_attention_dim=1920, dim_head=64, out_dim=output_dim, heads=output_dim//64, bias=True, # pre_only=True, eps=1e-6 ) elif self.token_fusion == "perceiver": num_queries = 256 self.latents1 = nn.Parameter(torch.randn(1, num_queries, hidden_dim) / hidden_dim**0.5) # self.proj_in1 = nn.Linear(1920, hidden_dim) -> views_proj (see above) self.cross_attn1 = Attention( query_dim=hidden_dim, cross_attention_dim=hidden_dim, dim_head=64, out_dim=output_dim, heads=output_dim//64, bias=True, # pre_only=True, eps=1e-6 ) self.latents2 = nn.Parameter(torch.randn(1, num_queries, hidden_dim) / hidden_dim**0.5) self.proj_in2 = nn.Linear(self._token_dim, hidden_dim) self.cross_attn2 = Attention( query_dim=hidden_dim, cross_attention_dim=hidden_dim, dim_head=64, out_dim=output_dim, heads=output_dim//64, bias=True, # pre_only=True, eps=1e-6 ) self.film = FilmConditioning(2048, output_dim) if use_film else None self._obs_strat = "add_token" self._n_tokens += 1 self._obs_proc = nn.Sequential( nn.Dropout(p=0.2), nn.Linear(odim, output_dim) ) dropout = 0.1 # @see model_cfg.dropout linear_proj = nn.Identity() # build (optional) token feature projection layer norm = nn.Identity() # feat_norm = None self.post_proc = nn.Sequential(linear_proj, norm, nn.Dropout(dropout)) def forward(self, # noisy_actions: torch.Tensor, # timestep: torch.Tensor, # temp views, obs, traj2ds=None, # ac_flat = None, # mask_flat = None, text_embeddings = None, vlm_attn_mask=None ) -> Tuple[torch.Tensor, torch.Tensor]: """Forward pass. Args: noisy_actions (`torch.Tensor`): Image features. timestep (`torch.Tensor`): Timestep in denoising process. Returns: `Tuple`[`torch.Tensor`, `torch.Tensor`]: The pair (latents, timestep_emb). """ # DiffusionTransformerAgent.forward s_t, vlm_attn_mask = self.tokenize_obs(views, obs, traj2ds, text_embeddings=text_embeddings, vlm_attn_mask=vlm_attn_mask) # (B, S, d_model) # _DiTNoiseNet.forward_enc s_t = s_t.transpose(0, 1) # (S, B, d_model) pos = self.enc_pos( s_t, vlm_attn_mask if self.mask_aware_positions else None ) # (S, B, d_model) return (s_t + pos).transpose(0,1), vlm_attn_mask # (B,S,d_model) def tokenize_obs(self, views, obs, traj2ds = None, flatten=False, text_embeddings=None, vlm_attn_mask=None): view_tokens = self.views_proj(views) # V:views,相机、视角数量 # S:每个view的token数(VLM长度) # D:token维度 B,V,S,D = view_tokens.shape if traj2ds is not None: traj_tokens = self.embed({}, {f"cond{i}": traj2ds[:, i] for i in range(traj2ds.shape[1])}) if self.token_fusion == "concat": traj_tokens = self.embed_proj(traj_tokens) #(B,S,d_model) tokens = torch.cat([view_tokens.view(B,V*S,D), traj_tokens], dim=1) # (B, tok1+tok2, 1536) elif self.token_fusion == "cross": traj_tokens_list, view_tokens_list = [], [] for v_idx in range(traj2ds.shape[1]): # traj2d --> cross attend --> view features view_tokens = self.attn( hidden_states=traj_tokens[:, v_idx*self.traj_n_token, (v_idx+1)*self.traj_n_token], encoder_hidden_states=views[:, v_idx], ) traj_tokens_list.append(traj_tokens) view_tokens_list.append(view_tokens) traj_tokens = torch.cat(traj_tokens_list, dim=1) traj_tokens = self.embed_proj(traj_tokens) view_tokens = torch.cat(view_tokens_list, dim=1) tokens = torch.cat([view_tokens, traj_tokens], dim=1) # (B, tok1+tok2, 1536) elif self.token_fusion == "perceiver": latents1 = self.latents1.repeat(B, 1, 1) # views = self.proj_in1(views) # views = views + text_embeddings[:, None] tokens1 = self.cross_attn1( hidden_states=latents1, encoder_hidden_states=view_tokens.view(B, V*S, D), ) latents2 = self.latents2.repeat(B, 1, 1) traj_tokens = self.proj_in2(traj_tokens) # traj2ds = traj2ds + text_embeddings[:, None] tokens2 = self.cross_attn2( hidden_states=latents2, encoder_hidden_states=traj_tokens, ) tokens = torch.cat([tokens1, tokens2], dim=1) # (B, tok1+tok2, 1536) else: raise ValueError else: assert self._n_conditions == 0, "inconsistent confg" tokens = view_tokens.view(B,V*S,D) # 用FiLM把text_embeddings融进token if (self.film is not None) and (text_embeddings is not None): tokens = self.film(tokens, text_embeddings) if self._obs_strat == "add_token": # <modify_states> 多帧state时,每一帧都作为一个proprio token参与注意力。 obs_token = self._obs_proc(obs)#[:, None] tokens = torch.cat((tokens, obs_token), 1) # 原逻辑: # vlm_attn_mask = torch.cat([vlm_attn_mask, torch.ones((vlm_attn_mask.shape[0], 1), device=vlm_attn_mask.device)], 1) if vlm_attn_mask is not None else None if vlm_attn_mask is not None: obs_token_count = obs_token.shape[1] if len(obs_token.shape) == 3 else 1 # <modify_states> 保持与原始训练代码一致:新增obs token mask继续使用默认float32, # 这样拼接后整体attention mask仍会提升成float32,满足后续断言。 obs_token_mask = torch.ones( (vlm_attn_mask.shape[0], obs_token_count), device=vlm_attn_mask.device, ) vlm_attn_mask = torch.cat([vlm_attn_mask, obs_token_mask], 1) elif self._obs_strat == "pad_img_tokens": obs = self._obs_proc(obs) obs = obs[:, None].repeat((1, tokens.shape[1], 1)) tokens = torch.cat((obs, tokens), 2) vlm_attn_mask = None else: assert self._obs_strat is None tokens = self.post_proc(tokens) if flatten: return tokens.reshape((tokens.shape[0], -1)), vlm_attn_mask return tokens, vlm_attn_mask def embed(self, imgs, conditions=None): """ SONGLIN: DOWNSCALE == 2**5 = 32, total 5 conv2d with stride(2) """ def embed_helper(net, im): # with torch.autocast(device_type='cuda', dtype=torch.bfloat16): # TODO check if this is needed if self.early_fusion and len(im.shape) == 5: T = im.shape[1] im = torch.cat([im[:, t] for t in range(T)], 1) embeds = net(im) elif len(im.shape) == 5: B, T, C, H, W = im.shape embeds = net(im.reshape((B * T, C, H, W))) embeds = embeds.reshape((B, -1, net.embed_dim)) else: assert len(im.shape) == 4 embeds = net(im) return embeds if self._share_cam_features: embeds = [ embed_helper(self.visual_features, imgs[f"cam{i}"]) for i in self.camera_indices ] if conditions is not None: assert self._n_conditions > 0 embeds += [ embed_helper(self.visual_features, conditions[f"cond{i}"]) for i in range(self._n_conditions) ] else: assert conditions is not None # img_encoders = self.visual_features[:self._n_cams] # sum([v.sum().item() for k, v in img_encoders.named_parameters()]) # cond_encoders = self.visual_features[self._n_cams:] cond_encoders = self.visual_features # [ # embed_helper(net, imgs[f"cam{i}"]) # for i, net in zip(self.camera_indices, img_encoders) # ] + embeds = [ embed_helper(net, conditions[f"cond{0}"]) for i, net in enumerate(cond_encoders) ] """ if self.training: # FIXME better handle this thing return torch.cat(embeds, dim=1) """ assert conditions is not None in_dtype = next(iter(imgs.values())).dtype if len(imgs) > 0 else next(iter(conditions.values())).dtype embeds = torch.cat(embeds, dim=1).to(dtype=in_dtype) # (B, S, d_model) return embeds class ActionProjectionIn(nn.Module): def __init__( self, action_pred_horizon, action_dim, output_dim, ): super().__init__() self.action_pred_horizon = action_pred_horizon self.action_dim = action_dim # Opt-in numerical policy. Keep legacy checkpoints bitwise unchanged. self.force_fp32 = False self.ac_proj = nn.Sequential( nn.Linear(action_dim, action_dim), nn.GELU(approximate="tanh"), nn.Linear(action_dim, output_dim), ) # self.time_net = _TimeNetwork(time_dim=256, out_dim=output_dim) # self.register_parameter( # "dec_pos", # nn.Parameter(torch.empty(action_pred_horizon, output_dim), requires_grad=True), # ) # (1, Tp, hidden_dim) self.dec_pos = nn.Parameter(torch.empty(action_pred_horizon, output_dim), requires_grad=True) nn.init.xavier_uniform_(self.dec_pos.data) def forward( self, noisy_actions: torch.Tensor, position_offset: int = 0, ) -> torch.Tensor: B = noisy_actions.shape[0] # action_dim = self._dit_obs_proj.action_dim # time_enc = self.time_net(timestep) noise_acs = noisy_actions.reshape(B, -1, self.action_dim) # DiffusionTransformerAgent.forward if self.force_fp32: if any(p.dtype != torch.float32 for p in self.ac_proj.parameters()): raise RuntimeError("FP32 action input projection requires FP32 master weights") # BF16 quantization here can erase small inter-frame angle changes # before denoising. Only this small input MLP leaves mixed precision; # attention, output head and the rest of the model are unchanged. with torch.autocast(device_type=noise_acs.device.type, enabled=False): ac_tokens = self.ac_proj(noise_acs.float()) else: ac_tokens = self.ac_proj(noise_acs) # ac_tokens = ac_tokens.transpose(0, 1) # (B, L, d_model) _DiTNoiseNet.forward_dec token_count = ac_tokens.shape[1] position_end = int(position_offset) + token_count if position_offset < 0 or position_end > self.action_pred_horizon: raise ValueError( "Action token positions exceed the configured horizon: " f"offset={position_offset}, tokens={token_count}, " f"horizon={self.action_pred_horizon}" ) action_hidden_states = ( ac_tokens + self.dec_pos[int(position_offset):position_end].unsqueeze(0) ) return action_hidden_states#, time_enc class ActionProjectionOut(nn.Module): # _FinalLayer def __init__(self, hidden_size, action_dim): super().__init__() self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) self.linear = nn.Linear(hidden_size, action_dim, bias=True) self.adaLN_modulation = nn.Sequential( nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True) ) self.reset_parameters() def reset_parameters(self): # Standard DiT final-layer initialization: start with zero velocity # prediction and let the output head open gradually. This keeps the # first updates from backpropagating an untrained random output through # the complete transformer stack. modulation = self.adaLN_modulation[-1] assert isinstance(modulation, nn.Linear) nn.init.zeros_(modulation.weight) nn.init.zeros_(modulation.bias) nn.init.zeros_(self.linear.weight) nn.init.zeros_(self.linear.bias) def forward(self, x, t, cond=None): # process the conditioning vector first # cond = torch.mean(cond, dim=1) # cond = cond + t if len(t.shape) == 2: # (B, D) t = t.unsqueeze(1) # (B, 1, D) elif len(t.shape) == 3: # (B, T, D) pass else: raise ValueError(f"Invalid shape of t: {t.shape}") shift, scale = self.adaLN_modulation(t).chunk(2, dim=-1) # cond -> t # x = self.norm_final(x) * (1+scale[None]) + shift[None] # TODO be careful! # x = x * scale + shift x = self.norm_final(x) * (1 + scale) + shift x = self.linear(x) # return x.transpose(0, 1) return x # @maybe_allow_in_graph class VLATransformerBlock(nn.Module): def __init__( self, dim: int, num_attention_heads: int, attention_head_dim: int, context_pre_only: bool = False, qk_norm: Optional[str] = "fp32_layer_norm", use_dual_attention: bool = False, training_phase: str = "joint", # "joint", "action", "traj2d", "action_mini" ): super().__init__() # self.training_phase = training_phase # from models.dit_policy.data4robotics.models.diffusion import _SelfAttnEncoder #_TransformerEncoder, # self.encoder_module = _SelfAttnEncoder( # d_model=dim, # nhead=num_attention_heads, # dim_feedforward=2048, # dropout=0.1, # activation="gelu", # ) # self.encoder_module.reset_parameters() # # decoder blocks # from models.dit_policy.data4robotics.models.diffusion import _DiTDecoder #_TransformerDecoder, # self.decoder_module = _DiTDecoder( # d_model=dim, # nhead=num_attention_heads, # dim_feedforward=2048, # dropout=0.1, # activation="gelu", # ) # self.decoder_module.reset_parameters() # _TransformerDecoder.__init__ self.context_pre_only = context_pre_only context_norm_type = "ada_norm_continous" if context_pre_only else "ada_norm_zero" # self.norm1 = AdaLayerNormZero(dim) self.norm1_act = AdaLayerNormZero(dim) logger.debug(f"AdaLN act parameters: {count_parameters(self.norm1_act):,}") if context_norm_type == "ada_norm_continous": """ self.norm1_lang = AdaLayerNormContinuous(dim, dim, elementwise_affine=False, eps=1e-6, bias=True, norm_type="layer_norm") """ self.norm1_obs = AdaLayerNormContinuous(dim, dim, elementwise_affine=False, eps=1e-6, bias=True, norm_type="layer_norm") logger.debug(f"AdaLN obs parameters: {count_parameters(self.norm1_obs):,}") elif context_norm_type == "ada_norm_zero": """ self.norm1_lang = AdaLayerNormZero(dim) """ self.norm1_obs = AdaLayerNormZero(dim) logger.debug(f"AdaLN obs parameters: {count_parameters(self.norm1_obs):,}") else: raise ValueError( f"Unknown context_norm_type: {context_norm_type}, currently only support `ada_norm_continous`, `ada_norm_zero`" ) if hasattr(F, "scaled_dot_product_attention"): processor = JointVLAAttnProcessor( query_dim=dim, dim_head=attention_head_dim, heads=num_attention_heads, out_dim=dim, bias=True, context_pre_only=context_pre_only, ) else: raise ValueError( "The current PyTorch version does not support the `scaled_dot_product_attention` function." ) self.attn = Attention( query_dim=dim, cross_attention_dim=None, added_kv_proj_dim=dim, dim_head=attention_head_dim, heads=num_attention_heads, out_dim=dim, context_pre_only=context_pre_only, bias=True, processor=processor, # type: ignore qk_norm=qk_norm, eps=1e-6, ) logger.debug(f"Joint Attention parameters: {count_parameters(self.attn):,}") self.norm2_act = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) self.ff_act = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate") logger.debug(f"FF act parameters: {count_parameters(self.ff_act):,}") if not context_pre_only: """ self.norm2_lang = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) self.ff_lang = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate") """ self.norm2_obs = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) self.ff_obs = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate") logger.debug(f"FF obs parameters: {count_parameters(self.ff_obs):,}") else: """ self.norm2_lang = None self.ff_lang = None """ self.norm2_obs= None self.ff_obs = None # let chunk size default to None self._chunk_size = None self._chunk_dim = 0 def forward( self, # hidden_states: torch.FloatTensor, # V # lang_hidden_states: torch.FloatTensor, # L action_hidden_states: torch.Tensor, # A obs_hidden_states: torch.Tensor, # O temb: torch.Tensor, obs_token_mask: Optional[torch.Tensor] = None, # obs_pos_emb: Optional[torch.FloatTensor] = None, # time_enc: Optional[torch.FloatTensor] = None, joint_attention_kwargs=None, ): # obs_hidden_states = self.encoder_module(src=obs_hidden_states, pos=obs_pos_emb) # action_hidden_states = self.decoder_module(x=action_hidden_states, t=time_enc, cond=obs_hidden_states) # recursive: _TransformerDecoder.forward norm_action_hidden_states, gate_msa_act, shift_mlp_act, scale_mlp_act, gate_mlp_act = self.norm1_act(action_hidden_states, emb=temb) if self.context_pre_only: """ norm_lang_hidden_states = self.norm1_lang(lang_hidden_states, temb) """ norm_obs_hidden_states = self.norm1_obs(obs_hidden_states, temb[:,-1] if len(temb.shape) > 2 else temb) gate_msa_obs, shift_mlp_obs, scale_mlp_obs, gate_mlp_obs = None, None, None, None else: """ norm_lang_hidden_states, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.norm1_lang( lang_hidden_states, emb=temb ) """ norm_obs_hidden_states, gate_msa_obs, shift_mlp_obs, scale_mlp_obs, gate_mlp_obs = self.norm1_obs(obs_hidden_states, emb=temb[:,-1] if len(temb.shape) > 2 else temb) act_attn_output, obs_attn_output = self.attn( ## indirectly calls JointAttnProcessor2_0.__call__, VLAAttnProcessor.__call__ # hidden_states=None, # lang_hidden_states=norm_lang_hidden_states, hidden_states=norm_action_hidden_states, encoder_hidden_states=norm_obs_hidden_states, attention_mask=obs_token_mask # **joint_attention_kwargs, ) # Action feed forward act_attn_output = gate_msa_act * act_attn_output action_hidden_states = action_hidden_states + act_attn_output norm_action_hidden_states = self.norm2_act(action_hidden_states) norm_action_hidden_states = norm_action_hidden_states * (1 + scale_mlp_act) + shift_mlp_act ff_action_output = self.ff_act(norm_action_hidden_states) ff_action_output = gate_mlp_act * ff_action_output action_hidden_states = action_hidden_states + ff_action_output # attention outputs for the `obs_hidden_states`. if self.context_pre_only: # encoder_hidden_states = None obs_hidden_states = None # type:ignore else: assert gate_msa_obs is not None and \ shift_mlp_obs is not None and \ scale_mlp_obs is not None and \ gate_mlp_obs is not None """ # lang forward lang_attn_output = c_gate_msa.unsqueeze(1) * lang_attn_output lang_hidden_states = lang_hidden_states + lang_attn_output norm_lang_hidden_states = self.norm2_lang(lang_hidden_states) norm_lang_hidden_states = norm_lang_hidden_states * (1 + c_scale_mlp[:, None]) + c_shift_mlp[:, None] if self._chunk_size is not None: # "feed_forward_chunk_size" can be used to save memory context_ff_output = _chunked_feed_forward( self.ff_lang, norm_lang_hidden_states, self._chunk_dim, self._chunk_size ) else: context_ff_output = self.ff_lang(norm_lang_hidden_states) lang_hidden_states = lang_hidden_states + c_gate_mlp.unsqueeze(1) * context_ff_output """ # obs feedforward assert self.norm2_obs is not None obs_attn_output = gate_msa_obs * obs_attn_output obs_hidden_states = obs_hidden_states + obs_attn_output norm_obs_hidden_states = self.norm2_obs(obs_hidden_states) norm_obs_hidden_states = norm_obs_hidden_states * (1 + scale_mlp_obs) + shift_mlp_obs assert self.ff_obs is not None ff_obs_output = self.ff_obs(norm_obs_hidden_states) ff_obs_output = gate_mlp_obs * ff_obs_output obs_hidden_states = obs_hidden_states + ff_obs_output return action_hidden_states, obs_hidden_states class ActionTransformerModel( ModelMixin, ConfigMixin #, VLTTransformerActionMixin # PeftAdapterMixin, FromOriginalModelMixin, ReprodMixin, SD3Transformer2DLoadersMixin, ): """ An action transformer model """ _supports_gradient_checkpointing = True _no_split_modules = ["ActionTransformerBlock"] # _skip_layerwise_casting_patterns = ["pos_embed", "norm"] @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs): # model = cls.from_config(config=None, **kwargs) model = ActionTransformerModel() torch_dtype = kwargs.pop("torch_dtype", torch.float32) if pretrained_model_name_or_path is not None and "checkpoints" in str(pretrained_model_name_or_path): if os.path.exists(pretrained_model_name_or_path): # NOTE use accelerator.checkpointing from safetensors.torch import load_file file_path = f"{pretrained_model_name_or_path}/model.safetensors" assert os.path.exists(file_path), f"Checkpoint {file_path} does not exist." model_state_dict = load_file(file_path) missing_keys, unexpected_keys = model.load_state_dict(model_state_dict, strict=False) """ # load all checkpoint data, including weights, optimizer, etc. load_dict = torch.load(pretrained_model_name_or_path, weights_only=False) from torch.nn.parallel import DistributedDataParallel as DDP model = model.module if isinstance(model, DDP) else model missing_keys, unexpected_keys = model.load_state_dict(load_dict["model"], strict=False) """ if len(unexpected_keys) > 0: logger.info(f"Unexpected keys in the checkpoint: {unexpected_keys}. ") if len(missing_keys) > 0: logger.info(f"Missing keys in the checkpoint: {missing_keys}.") logger.info(f"Loaded model from {pretrained_model_name_or_path}") else: raise ValueError(f"Checkpoint {pretrained_model_name_or_path} does not exist. ") model.to(torch_dtype) return model @register_to_config def __init__( self, # sample_size: int = 128, # patch_size: int = 2, # in_channels: int = 16, # num_layers: int = 24, attention_head_dim: int = 64, num_attention_heads: int = 24, # joint_attention_dim: int = 4096, caption_projection_dim: int = 1536, pooled_projection_dim: int = 2048, # out_channels: int = 16, pos_embed_max_size: int = 192, dual_attention_layers: Tuple[ int, ... ] = (), # () for sd3.0; (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12) for sd3.5 qk_norm: Optional[str] = "fp32_layer_norm", # VLA parameters training_phase: str = "joint", # "joint", "action", "traj2d", "action_mini" # action head parameters action_num_blocks:int = 6, action_pred_horizon: int = 6, action_dim: int = 7, action_hidden_dim: int = 1536, action_nheads: int = 24, # action_attention_head_dim: Optional[int] = 64, n_conditions: int = 0, token_fusion: str = "concat", # "concat", "cross" resnet_store_path: str = "cache/visual_features/resnet18/IN_1M_resnet18.pth", odim: int = 32, view_feature_dim: int = 1920, use_film: bool = False, combined_temb: bool = False, use_future_progress_embedding: bool = False, future_progress_dim: int = 5, ): super().__init__() # self.inner_dim = num_attention_heads * attention_head_dim self.inner_dim = num_attention_heads * attention_head_dim # TODO-yth: 扩散时间步embedding是否融合文本语义 self.combined_temb = combined_temb if self.combined_temb: self.time_ins_embed = CombinedTimestepTextProjEmbeddings( embedding_dim=self.inner_dim, pooled_projection_dim=pooled_projection_dim ) # used else: self.time_ins_embed = _TimeNetwork(time_dim=256, out_dim=action_hidden_dim) """ self.lang_embedder = nn.Linear(joint_attention_dim, caption_projection_dim) """ # Set the observation projection self.obs_proj = ObservationProjection( action_pred_horizon=action_pred_horizon, action_dim=action_dim, hidden_dim=self.inner_dim, # not used output_dim=action_hidden_dim, n_conditions=n_conditions, token_fusion=token_fusion, resnet_store_path=resnet_store_path, odim=odim, view_feature_dim=view_feature_dim, use_film=use_film ) total_params = sum(p.numel() for p in self.obs_proj.parameters() if p.requires_grad) logger.debug(f"ObservationEncoder parameters: {total_params:,}") # Set the action projection self.action_proj_in = ActionProjectionIn( action_pred_horizon=action_pred_horizon, action_dim=action_dim, output_dim=action_hidden_dim, ) self.use_future_progress_embedding = bool(use_future_progress_embedding) self.future_progress_dim = int(future_progress_dim) if self.use_future_progress_embedding: self.future_progress_mlp = nn.Sequential( nn.Linear(self.future_progress_dim, action_hidden_dim), nn.SiLU(), nn.Linear(action_hidden_dim, action_hidden_dim), ) # future_progress is an additive residual condition. Start it as # an exact no-op for the same reason as adaLN-Zero and learn the # conditioning path gradually. nn.init.zeros_(self.future_progress_mlp[-1].weight) nn.init.zeros_(self.future_progress_mlp[-1].bias) else: self.future_progress_mlp = None total_params = sum(p.numel() for p in self.action_proj_in.parameters() if p.requires_grad) logger.debug(f"ActionProjectionIn parameters: {total_params:,}") self.transformer_blocks = nn.ModuleList( [ # IdentityBlock() VLATransformerBlock( # JointTransformerBlock dim=action_hidden_dim, num_attention_heads=action_nheads, attention_head_dim=attention_head_dim, context_pre_only=i == action_num_blocks - 1, qk_norm=qk_norm, # use_dual_attention=True if i in dual_attention_layers else False, training_phase=training_phase, # "joint", "action", "traj2d", "action_mini" ) for i in range(action_num_blocks) ] ) total_params = sum(p.numel() for p in self.transformer_blocks[0].parameters() if p.requires_grad) logger.debug(f"ActionTransformerBlock parameters: {total_params:,}") self.action_proj_out = ActionProjectionOut( hidden_size=action_hidden_dim, action_dim=action_dim ) total_params = sum(p.numel() for p in self.action_proj_out.parameters()) logger.debug(f"ActionProjectionOut parameters: {total_params:,}") def forward( self, hidden_states: torch.Tensor, encoder_hidden_states: Optional[torch.Tensor] = None, pooled_projections: Optional[torch.Tensor] = None, timestep: Optional[torch.LongTensor] = None, block_controlnet_hidden_states: Optional[List] = None, joint_attention_kwargs: Optional[Dict[str, Any]] = None, vlm_attn_mask: Optional[torch.Tensor] = None, return_dict: bool = True, skip_layers: Optional[List[int]] = None, ) -> Union[List[torch.Tensor], ActionTransformerModelOutput]: assert joint_attention_kwargs is not None if self.config.n_conditions > 0: # type: ignore assert joint_attention_kwargs["traj2ds"].shape[1] == self.config.n_conditions, "model n_conditions does not match" # type: ignore # L if self.combined_temb: temb = self.time_ins_embed(timestep, pooled_projections) else: temb = self.time_ins_embed(timestep) """ lang_hidden_states = self.lang_embedder(encoder_hidden_states) """ # A. For RTC, only the unknown postfix lives on the action/AdaLN # branch. The known overlap is encoded with the existing action # projection (no new parameters), then appended to observation context. noisy_action = joint_attention_kwargs.pop("action_hidden_embeds") known_action_prefix = joint_attention_kwargs.pop("known_action_prefix", None) prefix_len = 0 if known_action_prefix is not None: if known_action_prefix.ndim != noisy_action.ndim: raise ValueError( "known_action_prefix and action_hidden_embeds must have " f"the same rank, got {known_action_prefix.ndim} and {noisy_action.ndim}" ) if ( known_action_prefix.shape[0] != noisy_action.shape[0] or known_action_prefix.shape[-1] != noisy_action.shape[-1] ): raise ValueError( "known_action_prefix batch/action dimensions must match " "action_hidden_embeds" ) prefix_len = int(known_action_prefix.shape[1]) if prefix_len + noisy_action.shape[1] != self.config.action_pred_horizon: raise ValueError( "RTC prefix plus postfix must fill the configured action horizon: " f"prefix={prefix_len}, postfix={noisy_action.shape[1]}, " f"horizon={self.config.action_pred_horizon}" ) action_hidden_states = self.action_proj_in( noisy_action, position_offset=prefix_len ) prefix_hidden_states = None if known_action_prefix is not None: prefix_hidden_states = self.action_proj_in( known_action_prefix, position_offset=0 ) if self.use_future_progress_embedding: future_progress = joint_attention_kwargs.get("future_progress", None) if future_progress is None: raise ValueError("use_future_progress_embedding=True requires future_progress tensor") assert self.future_progress_mlp is not None future_progress = future_progress.to( device=action_hidden_states.device, dtype=action_hidden_states.dtype, ) expected_progress_shape = ( action_hidden_states.shape[0], prefix_len + action_hidden_states.shape[1], ) if future_progress.shape[:2] != expected_progress_shape: raise ValueError( f"future_progress shape {future_progress.shape} does not match full action " f"timeline {expected_progress_shape}" ) if prefix_hidden_states is not None: prefix_hidden_states = prefix_hidden_states + self.future_progress_mlp( future_progress[:, :prefix_len] ) action_hidden_states = action_hidden_states + self.future_progress_mlp( future_progress[:, prefix_len:] ) # Obs: V+Proprio obs_hidden_states, obs_token_mask = self.obs_proj( views=joint_attention_kwargs["views"], obs=joint_attention_kwargs["obs"], traj2ds=joint_attention_kwargs["traj2ds"], text_embeddings=pooled_projections, vlm_attn_mask=vlm_attn_mask ) # S, B, d_model if prefix_hidden_states is not None: # Prefix is context-only: it is a K/V condition alongside VLM and # proprio tokens. It never enters action AdaLN or the output head, # while postfix queries can still attend to it in every block. obs_hidden_states = torch.cat( [prefix_hidden_states, obs_hidden_states], dim=1 ) if obs_token_mask is not None: prefix_token_mask = torch.ones( (obs_token_mask.shape[0], prefix_len), device=obs_token_mask.device, dtype=obs_token_mask.dtype, ) obs_token_mask = torch.cat( [prefix_token_mask, obs_token_mask], dim=1 ) for index_block, block in enumerate(self.transformer_blocks): is_skip = True if skip_layers is not None and index_block in skip_layers else False if is_skip: continue action_hidden_states, obs_hidden_states = block( # hidden_states=hidden_states, # V # lang_hidden_states=lang_hidden_states, # L action_hidden_states=action_hidden_states, # A obs_hidden_states=obs_hidden_states, # O temb=temb, obs_token_mask=obs_token_mask # time_enc=time_enc, # obs_pos_emb=obs_pos_emb, # joint_attention_kwargs=joint_attention_kwargs, ) action_output = self.action_proj_out( x=action_hidden_states, t=temb, # cond=obs_hidden_states, ) return ActionTransformerModelOutput(action=action_output) class DiTActionTransformerModel( ModelMixin, ConfigMixin #, VLTTransformerActionMixin # PeftAdapterMixin, FromOriginalModelMixin, ReprodMixin, SD3Transformer2DLoadersMixin, ): """ An action transformer model """ _supports_gradient_checkpointing = True # _no_split_modules = ["ActionTransformerBlock"] # _skip_layerwise_casting_patterns = ["pos_embed", "norm"] @register_to_config def __init__( self, # sample_size: int = 128, # patch_size: int = 2, # in_channels: int = 16, # num_layers: int = 24, attention_head_dim: int = 64, num_attention_heads: int = 24, # joint_attention_dim: int = 4096, caption_projection_dim: int = 1536, pooled_projection_dim: int = 2048, # out_channels: int = 16, pos_embed_max_size: int = 192, dual_attention_layers: Tuple[ int, ... ] = (), # () for sd3.0; (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12) for sd3.5 qk_norm: Optional[str] = None, # VLA parameters training_phase: str = "joint", # "joint", "action", "traj2d", "action_mini" # action head parameters action_num_blocks:int = 6, action_pred_horizon: int = 6, action_dim: int = 7, action_hidden_dim: int = 1536, #this action_nheads: int = 24, # action_attention_head_dim: Optional[int] = 64, n_conditions: int = 0, token_fusion: str = "concat", # "concat", "cross" resnet_store_path: str = "cache/visual_features/resnet18/IN_1M_resnet18.pth", odim: int = 32, view_feature_dim: int = 1920, use_film: bool = False, combined_temb: bool = False, use_future_progress_embedding: bool = False, future_progress_dim: int = 5, ): super().__init__() # self.inner_dim = num_attention_heads * attention_head_dim self.inner_dim = num_attention_heads * attention_head_dim if use_future_progress_embedding: raise ValueError("DiTActionTransformerModel does not support future progress embedding") # Set the observation projection self.obs_proj = ObservationProjection( action_pred_horizon=action_pred_horizon, action_dim=action_dim, hidden_dim=self.inner_dim, # not used output_dim=action_hidden_dim, n_conditions=n_conditions, token_fusion="concat", resnet_store_path=resnet_store_path, odim=odim, view_feature_dim=view_feature_dim, use_film=use_film ) total_params = sum(p.numel() for p in self.obs_proj.parameters() if p.requires_grad) logger.debug(f"ObservationEncoder parameters: {total_params:,}") # q_former: # TODO: boqian: hardcode here num_query_tokens = 256 num_heads_ = 16 self.query_tokens = nn.Parameter(torch.randn(num_query_tokens, action_hidden_dim)) self.cross_attention = CrossAttentionBlock(action_hidden_dim, num_heads_) # assert action_hidden_dim == 1024 self.DiT = DiT( in_channels=action_dim, depth=24, num_heads=16, token_size=action_hidden_dim, learn_sigma=False, class_dropout_prob=0, future_action_window_size=action_pred_horizon-1, past_action_window_size=0, n_conditon_token=num_query_tokens ) total_params = sum(p.numel() for p in self.DiT.parameters() if p.requires_grad) logger.debug(f"DiT parameters: {total_params:,}") def forward( self, hidden_states: torch.Tensor, encoder_hidden_states: Optional[torch.Tensor] = None, pooled_projections: Optional[torch.Tensor] = None, timestep: Optional[torch.LongTensor] = None, block_controlnet_hidden_states: Optional[List] = None, joint_attention_kwargs: Optional[Dict[str, Any]] = None, vlm_attn_mask: Optional[torch.Tensor] = None, return_dict: bool = True, skip_layers: Optional[List[int]] = None, ) -> Union[List[torch.Tensor], ActionTransformerModelOutput]: assert joint_attention_kwargs is not None if joint_attention_kwargs.get("known_action_prefix", None) is not None: raise ValueError( "Context-only RTC is implemented for ActionTransformerModel; " "DiTActionTransformerModel cannot consume known_action_prefix." ) if self.config.n_conditions > 0: # type: ignore assert joint_attention_kwargs["traj2ds"].shape[1] == self.config.n_conditions, "model n_conditions does not match" # type: ignore """ lang_hidden_states = self.lang_embedder(encoder_hidden_states) """ # A noisy_action = joint_attention_kwargs.pop("action_hidden_embeds") # Obs: V+Proprio condition_tokens, vlm_attn_mask = self.obs_proj( views=joint_attention_kwargs["views"], obs=joint_attention_kwargs["obs"], traj2ds=joint_attention_kwargs["traj2ds"], text_embeddings=pooled_projections, vlm_attn_mask=vlm_attn_mask ) # B, S, d_model # print("condition_tokens.shape:", condition_tokens.shape) # [B, 106, 1024]) # q_former # TODO: forward vlm_attn_mask here query = self.query_tokens.unsqueeze(0).expand(condition_tokens.shape[0], -1, -1) # (B, Q, D) queried = self.cross_attention(query, condition_tokens) # (B, Q, D) # DiT action_output = self.DiT(noisy_action, timestep, queried) return ActionTransformerModelOutput(action=action_output) class HumanFoundationModel(nn.Module): def __init__(self, model_cfg, feature_extractors: dict[str, str]): super().__init__() # self.qwen = _QWen_VL_Interface(config=model_cfg) # self.layer_qformer = get_layerwise_qformer(config=self.config) # self.action_model = get_action_model(config=self.config) action_nheads = int(model_cfg.nhead) if model_cfg.hidden_dim % action_nheads != 0: raise ValueError( f"Action hidden dim {model_cfg.hidden_dim} must be divisible by nhead {action_nheads}" ) action_head_dim = model_cfg.hidden_dim // action_nheads self.action_header = ActionTransformerModel( attention_head_dim=action_head_dim, num_attention_heads=action_nheads, action_num_blocks=model_cfg.num_blocks, action_nheads=action_nheads, resnet_store_path=model_cfg.resnet_store_path, odim=model_cfg.odim, action_dim=model_cfg.action_dim, action_pred_horizon=model_cfg.action_chunk_size, view_feature_dim=model_cfg.view_feature_dim, use_film=model_cfg.use_film, combined_temb=model_cfg.combined_temb, action_hidden_dim=model_cfg.hidden_dim, qk_norm=getattr(model_cfg, "qk_norm", "fp32_layer_norm"), use_future_progress_embedding=getattr(model_cfg, "use_future_progress_embedding", False), future_progress_dim=getattr(model_cfg, "future_progress_dim", 5), ) total_params = sum(p.numel() for p in self.action_header.parameters()) overwatch.info(f"Total ActionTransformerModel parameters: {total_params:,}") vision_backbones = {} for backbone, model_path in feature_extractors.items(): if backbone == "siglip": siglip = SiglipModel.from_pretrained( model_path ) # , resume_download=None, local_files_only=True vision_backbones[backbone] = siglip.vision_model vision_backbones[backbone].head = nn.Identity() # remove classification head del siglip elif backbone == "dinov2": vision_backbones[backbone] = Dinov2Model.from_pretrained( model_path ) # , resume_download=None, local_files_only=True vision_backbones[backbone].embeddings.mask_token = None else: raise ValueError(f"Unsupported vision backbone: {backbone}") self.vision_backbones = nn.ModuleDict(vision_backbones) for name, backbone in self.vision_backbones.items(): num_parameters = sum(p.numel() for p in backbone.parameters() if p.requires_grad) overwatch.info(f"Vision backbone {name} has {num_parameters*1e-6:,}M parameters") def tune_vision_backbones(self, backbone_names: List[str]): for name, backbone in self.vision_backbones.items(): if name in backbone_names: overwatch.info(f"Tuning vision backbone: {name}") for param in backbone.parameters(): param.requires_grad = True else: overwatch.info(f"Freezing vision backbone: {name}") for param in backbone.parameters(): param.requires_grad = False def extract_vision_features(self, batch): features = [] for vision_backbone, model in self.vision_backbones.items(): assert vision_backbone in batch B, V, C, H, W = batch[vision_backbone].shape if vision_backbone == "siglip": vision_outputs = model( pixel_values=batch[vision_backbone].reshape(B * V, C, H, W), return_dict=True, ) siglip_features = vision_outputs.last_hidden_state features.append(siglip_features.view(B, V, *siglip_features.shape[1:])) elif vision_backbone == "dinov2": dinov2_outputs = model( pixel_values=batch[vision_backbone].view(B * V, C, H, W) ) dinov2_features = dinov2_outputs.last_hidden_state[:, 1:, :] features.append(dinov2_features.view(B, V, *dinov2_features.shape[1:])) else: raise ValueError(f"Unsupported vision backbone: {vision_backbone}") return torch.cat(features, dim=3) # (B,V,N,D) def forward( self, action_samples: torch.Tensor, views: dict[str, torch.Tensor], states: torch.Tensor, # hidden_states: torch.Tensor, lang_hidden_states: torch.Tensor, pooled_projections: torch.Tensor, timestep: torch.LongTensor, # block_controlnet_hidden_states: List, # joint_attention_kwargs: Optional[Dict[str, Any]], traj2ds: Optional[torch.Tensor] = None, return_dict: bool = True, skip_layers: Optional[List[int]] = None, ) -> Union[List[torch.Tensor], HumanFoundationModelOutput]: view_features = self.extract_vision_features(views) model_output = self.action_header( hidden_states=None, encoder_hidden_states=lang_hidden_states, # (B, 333, 4096) timestep=timestep, pooled_projections=pooled_projections, # (B, 2048) joint_attention_kwargs=dict( action_hidden_embeds=action_samples, # (B,Tp,Da) views=view_features, # (B,V,N,D) obs=states, # (B,1,M) traj2ds=traj2ds, # (B, C, 3, H, W) # text_embeddings=text_embeddings ), return_dict=return_dict, ) return HumanFoundationModelOutput(action=model_output.action) @torch.inference_mode() def predict_action( self, observations: List[List[Image.Image]], # B * List of PIL Image as [view1, view2] states: List[Any], instructions: List[str], **kwargs: str ) -> np.ndarray: # with torch.autocast("cuda", dtype=torch.float32): ... @torch.inference_mode() def chat(self): ... class Psi0Model(nn.Module): # 这里改不改2.5无所谓,外部会传参(来自posttrain.py的init_model) def __init__(self, model_cfg, vlm_model: Optional[nn.Module]): super().__init__() self.action_horizon = int(model_cfg.action_chunk_size) self.action_dim = int(model_cfg.action_dim) self.action_exec_horizon = int(model_cfg.action_exec_horizon) self.rtc_max_delay = int(model_cfg.max_delay) self.rtc_expected_overlap = ( self.action_horizon - self.action_exec_horizon ) if bool(getattr(model_cfg, "rtc", False)): _validate_rtc_inference_contract( action_horizon=self.action_horizon, action_exec_horizon=self.action_exec_horizon, inference_delay=self.rtc_expected_overlap, max_delay=self.rtc_max_delay, ) if model_cfg.use_dit: self.action_header = DiTActionTransformerModel( resnet_store_path=model_cfg.resnet_store_path, odim=model_cfg.odim, action_dim=model_cfg.action_dim, action_pred_horizon=model_cfg.action_chunk_size, view_feature_dim=model_cfg.view_feature_dim, use_film=model_cfg.use_film, combined_temb=model_cfg.combined_temb, action_hidden_dim=model_cfg.hidden_dim, qk_norm=getattr(model_cfg, "qk_norm", "fp32_layer_norm"), use_future_progress_embedding=getattr(model_cfg, "use_future_progress_embedding", False), future_progress_dim=getattr(model_cfg, "future_progress_dim", 5), ) else: action_nheads = int(model_cfg.nhead) if model_cfg.hidden_dim % action_nheads != 0: raise ValueError( f"Action hidden dim {model_cfg.hidden_dim} must be divisible by nhead {action_nheads}" ) action_head_dim = model_cfg.hidden_dim // action_nheads self.action_header = ActionTransformerModel( attention_head_dim=action_head_dim, num_attention_heads=action_nheads, action_num_blocks=model_cfg.num_blocks, action_nheads=action_nheads, resnet_store_path=model_cfg.resnet_store_path, odim=model_cfg.odim, action_dim=model_cfg.action_dim, action_pred_horizon=model_cfg.action_chunk_size, view_feature_dim=model_cfg.view_feature_dim, use_film=model_cfg.use_film, combined_temb=model_cfg.combined_temb, action_hidden_dim=model_cfg.hidden_dim, qk_norm=getattr(model_cfg, "qk_norm", "fp32_layer_norm"), use_future_progress_embedding=getattr(model_cfg, "use_future_progress_embedding", False), future_progress_dim=getattr(model_cfg, "future_progress_dim", 5), ) self.action_header.obs_proj.mask_aware_positions = bool( getattr(model_cfg, "action_mask_aware_positions", False) ) if bool(getattr(model_cfg, "action_input_projection_fp32", False)): if not hasattr(self.action_header, "action_proj_in"): raise ValueError("FP32 action input projection requires the masked action transformer") self.action_header.action_proj_in.force_fp32 = True total_params = sum(p.numel() for p in self.action_header.parameters()) overwatch.info(f"Total ActionTransformerModel parameters: {total_params:,}") self.vlm_model = vlm_model if self.vlm_model is None: overwatch.info("Total VLM Backbone parameters: 0 (cached hidden states only)") else: total_params = sum(p.numel() for p in self.vlm_model.parameters()) overwatch.info(f"Total VLM Backbone parameters: {total_params:,}") # load pretrained vlm+action head, and all the modules needed for predict_action # TODO-yth: 暂时没看怎么改加载预训练 @classmethod def from_pretrained(cls, run_dir, ckpt_step, launch_config:LaunchConfig, device): if not os.path.exists(run_dir): raise ValueError(f"Pretrained model path {run_dir} does not exist.") from safetensors.torch import load_file # <modify> 支持infer脚本传入best_ckpt目录名,用于读取checkpoints/best_ckpt。 # 原逻辑: # ckpt_dir = f"{run_dir}/checkpoints/ckpt_{ckpt_step}" if str(ckpt_step) in {"best", "best_ckpt"}: ckpt_dir = f"{run_dir}/checkpoints/best_ckpt" else: ckpt_dir = f"{run_dir}/checkpoints/ckpt_{ckpt_step}" # <modify> 兼容训练中间保存到run根目录的ckpt_xxx,例如用户当前的.../2604192034/ckpt_8000。 # 原逻辑只读取run_dir/checkpoints/ckpt_xxx。 if not os.path.exists(ckpt_dir): root_ckpt_dir = f"{run_dir}/ckpt_{ckpt_step}" if os.path.exists(root_ckpt_dir): ckpt_dir = root_ckpt_dir # ckpt_dir = f"{run_dir}/checkpoints/ckpt_800" file_path = f"{ckpt_dir}/model.safetensors" index_path = f"{ckpt_dir}/model.safetensors.index.json" if os.path.exists(file_path): state_dict = load_file(file_path, device="cpu") elif os.path.exists(index_path): with open(index_path, "r") as f: index_obj = json.load(f) weight_map = index_obj.get("weight_map", {}) shard_files = sorted(set(weight_map.values())) state_dict = {} for shard_file in shard_files: shard_path = os.path.join(ckpt_dir, shard_file) if not os.path.exists(shard_path): raise ValueError(f"Checkpoint shard {shard_path} does not exist.") state_dict.update(load_file(shard_path, device="cpu")) else: raise ValueError(f"Checkpoint file {file_path} does not exist.") configured_model_name_or_path = str(launch_config.model.model_name_or_path) model_path_override = os.getenv("PSI_PRETRAIN_BACKBONE_PATH", "").strip() model_name_or_path = os.path.expanduser( model_path_override or configured_model_name_or_path ) if model_path_override: overwatch.info( "Using PSI_PRETRAIN_BACKBONE_PATH override for inference: " f"{model_name_or_path} (run_config={configured_model_name_or_path})" ) # Transformers treats a missing local absolute path as a Hub repo id, # which otherwise produces a misleading HFValidationError. Fail early # with the actual configuration problem instead. if os.path.isabs(model_name_or_path) and not os.path.isfile( os.path.join(model_name_or_path, "config.json") ): source = "PSI_PRETRAIN_BACKBONE_PATH" if model_path_override else "run_config.json" raise FileNotFoundError( f"VLM backbone from {source} is not a valid local model directory: " f"{model_name_or_path} (missing config.json). " "Set PSI_PRETRAIN_BACKBONE_PATH to a complete local Qwen checkpoint." ) is_omni = "omni" in model_name_or_path.lower() # <modify_float> 增加float32推理开关;开启后避免FlashAttention2(fp16/bf16-only)报错。 # 原逻辑: # attn_impl = "flash_attention_2" if str(device).startswith("cuda") else "sdpa" force_float32_infer = os.getenv("PSI_INFER_FLOAT32", "0") == "1" if force_float32_infer: attn_impl = "sdpa" vlm_dtype = torch.float32 else: attn_impl = "flash_attention_2" if str(device).startswith("cuda") else "sdpa" vlm_dtype = torch.bfloat16 if is_omni: vlm_model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained( model_name_or_path, attn_implementation=attn_impl, dtype=vlm_dtype, ) else: # init empty vlm backbone from config only (skip loading base pretrained weights) vlm_config = AutoConfig.from_pretrained(model_name_or_path) vlm_config._attn_implementation = attn_impl vlm_model = Qwen3VLForConditionalGeneration(vlm_config) # <modify_float> float32开关打开时,VLM保持float32;默认保持bf16。 # 原逻辑: # vlm_model = vlm_model.to(dtype=torch.bfloat16) # type: ignore vlm_model = vlm_model.to(dtype=vlm_dtype) # type: ignore vlm_state_dict = {} for k, v in state_dict.items(): if k.startswith("action_header."): continue elif k.startswith('vlm_model.'): vlm_state_dict[k.replace("vlm_model.", "")] = v elif k.startswith("motion_encoder."): continue else: assert False, "check here" if vlm_state_dict: if "lm_head.weight" not in vlm_state_dict: if "model.language_model.embed_tokens.weight" in vlm_state_dict: vlm_state_dict["lm_head.weight"] = vlm_state_dict["model.language_model.embed_tokens.weight"] elif "model.embed_tokens.weight" in vlm_state_dict: vlm_state_dict["lm_head.weight"] = vlm_state_dict["model.embed_tokens.weight"] if "lm_head.weight" not in vlm_state_dict: raise KeyError( "Checkpoint contains VLM weights but neither lm_head.weight nor a supported " "embed_tokens.weight key was found." ) if vlm_state_dict["lm_head.weight"].shape[0] != vlm_model.lm_head.weight.shape[0]: vlm_model.resize_token_embeddings( vlm_state_dict["lm_head.weight"].shape[0], pad_to_multiple_of=192, mean_resizing=True, ) overwatch.info(f"Resized model token embeddings to {vlm_model.lm_head.weight.shape[0]}") vlm_model.load_state_dict(vlm_state_dict, strict=True) overwatch.info("loaded vlm_backbone checkpoint successfully.") else: if not is_omni: raise ValueError( "Checkpoint contains no VLM weights, but this non-Omni inference path only " "initialized the VLM from config. A pretrained base VLM is required." ) # Cached-hidden-state training with PSI_SKIP_VLM_MODEL=1 saves an # action-only checkpoint. The Omni backbone above was already loaded # from model_name_or_path, so keep those frozen base Qwen weights. overwatch.info( f"Checkpoint {ckpt_dir} contains no VLM weights; retaining pretrained " f"VLM backbone from {model_name_or_path}." ) # init hfm-together model with vlm backbone model = cls( model_cfg=launch_config.model, vlm_model = vlm_model, ) # load action head action_head_state_dict = {} for k, v in state_dict.items(): if k.startswith("vlm_model."): continue elif k.startswith('action_header.'): action_head_state_dict[k.replace("action_header.", "")] = v elif k.startswith("motion_encoder."): continue else: assert False, "check here" model.action_header.load_state_dict(action_head_state_dict, strict=True) overwatch.info("loaded action head checkpoint successfully.") if hasattr(model, "motion_encoder"): motion_encoder_state_dict = { k.replace("motion_encoder.", ""): v for k, v in state_dict.items() if k.startswith("motion_encoder.") } if motion_encoder_state_dict: model.motion_encoder.load_state_dict(motion_encoder_state_dict, strict=True) # type: ignore[attr-defined] overwatch.info("loaded motion encoder checkpoint successfully.") # load necessary modules if is_omni: model.vlm_processor = Qwen2_5OmniProcessor.from_pretrained(model_name_or_path) else: model.vlm_processor = AutoProcessor.from_pretrained(model_name_or_path) # model.tokenizer = model.vlm_processor.tokenizer if launch_config.model.noise_scheduler == "ddpm": from diffusers.schedulers.scheduling_ddim import DDIMScheduler scheduler = DDIMScheduler( num_train_timesteps=launch_config.model.train_diffusion_steps, beta_start=0.0001, beta_end=0.02, beta_schedule="squaredcos_cap_v2", clip_sample=True, set_alpha_to_one=True, steps_offset=0, prediction_type="epsilon", ) elif launch_config.model.noise_scheduler == "flow": from diffusers.schedulers.scheduling_flow_match_euler_discrete import (FlowMatchEulerDiscreteScheduler) scheduler= FlowMatchEulerDiscreteScheduler( num_train_timesteps=launch_config.model.train_diffusion_steps, # MUST be 1000 as per pretrained SD3 ) else: raise ValueError(f"Unsupported noise scheduler: {launch_config.model.noise_scheduler}") model.noise_scheduler = scheduler model.action_horizon = launch_config.model.action_chunk_size model.action_dim = launch_config.model.action_dim model.text_only = bool(getattr(launch_config.data.transform.model, "text_only", False)) model.audio_only = bool(getattr(launch_config.data.transform.model, "audio_only", False)) model.video_only = bool(getattr(launch_config.data.transform.model, "video_only", False)) model.video_size_pixels = int(getattr(launch_config.data.transform.model, "video_size_pixels", 50176)) model.use_delta_actions = bool(getattr(launch_config.data.transform.repack, "use_delta_actions", False)) # <modify_jerk> 记录hybrid action模式,方便infer/调试区分root-delta checkpoint。 model.root_delta_actions = bool(getattr(launch_config.data.transform.repack, "root_delta_actions", False)) # 记录TextOp动作checkpoint,推理脚本据此走一致的decode路径。 model.textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False)) # <modify_float> 记录推理精度配置,供predict_action构造autocast上下文。 model.infer_autocast_dtype = vlm_dtype model.device = device return model def _action_condition_hidden_states( self, vlm_hidden_states: torch.Tensor, input_ids: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, ) -> torch.Tensor: del input_ids, attention_mask return vlm_hidden_states.unsqueeze(1) def _omni_hidden_states_without_lm_head(self, vlm_kwargs: dict[str, Any]) -> torch.Tensor: """Run Qwen2.5-Omni thinker up to hidden states without materializing logits.""" input_ids = vlm_kwargs.get("input_ids", None) attention_mask = vlm_kwargs.get("attention_mask", None) input_features = vlm_kwargs.get("input_features", None) pixel_values = vlm_kwargs.get("pixel_values", None) pixel_values_videos = vlm_kwargs.get("pixel_values_videos", None) image_grid_thw = vlm_kwargs.get("image_grid_thw", None) video_grid_thw = vlm_kwargs.get("video_grid_thw", None) feature_attention_mask = vlm_kwargs.get("feature_attention_mask", None) audio_feature_lengths = vlm_kwargs.get("audio_feature_lengths", None) video_second_per_grid = vlm_kwargs.get("video_second_per_grid", None) inputs_embeds = vlm_kwargs.get("inputs_embeds", None) position_ids = vlm_kwargs.get("position_ids", None) cache_position = vlm_kwargs.get("cache_position", None) use_audio_in_video = vlm_kwargs.get("use_audio_in_video", None) if inputs_embeds is None: inputs_embeds = self.vlm_model.get_input_embeddings()(input_ids) if input_features is not None: audio_features = self.vlm_model.get_audio_features( input_features, feature_attention_mask=feature_attention_mask, audio_feature_lengths=audio_feature_lengths, ) audio_features = audio_features.to(inputs_embeds.device, inputs_embeds.dtype) _, _, audio_mask = self.vlm_model.get_placeholder_mask(input_ids, inputs_embeds=inputs_embeds) inputs_embeds = inputs_embeds.masked_scatter(audio_mask, audio_features) if pixel_values is not None: image_embeds = self.vlm_model.get_image_features(pixel_values, image_grid_thw) image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype) image_mask, _, _ = self.vlm_model.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds ) inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds) if pixel_values_videos is not None: video_embeds = self.vlm_model.get_video_features(pixel_values_videos, video_grid_thw) video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype) _, video_mask, _ = self.vlm_model.get_placeholder_mask( input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds ) inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds) if feature_attention_mask is not None: audio_feature_lengths = torch.sum(feature_attention_mask, dim=1) else: audio_feature_lengths = None if attention_mask is not None and position_ids is None: if cache_position is None or (cache_position is not None and cache_position[0] == 0) or self.vlm_model.rope_deltas is None: delta0 = (1 - attention_mask).sum(dim=-1).unsqueeze(1) position_ids, rope_deltas = self.vlm_model.get_rope_index( input_ids, image_grid_thw, video_grid_thw, attention_mask, use_audio_in_video, audio_feature_lengths, video_second_per_grid, ) self.vlm_model.rope_deltas = rope_deltas - delta0 else: batch_size, seq_length = input_ids.shape delta = cache_position[0] + self.vlm_model.rope_deltas if cache_position is not None else 0 position_ids = torch.arange(seq_length, device=input_ids.device) position_ids = position_ids.view(1, -1).expand(batch_size, -1) position_ids = position_ids.add(delta) position_ids = position_ids.unsqueeze(0).expand(3, -1, -1) outputs = self.vlm_model.model( attention_mask=attention_mask, position_ids=position_ids, past_key_values=vlm_kwargs.get("past_key_values", None), inputs_embeds=inputs_embeds, use_cache=False, output_attentions=vlm_kwargs.get("output_attentions", None), output_hidden_states=True, return_dict=True, cache_position=cache_position, ) return outputs.hidden_states[-1] def _extract_vlm_hidden_states(self, vlm_kwargs: dict[str, Any]) -> torch.Tensor: if self.vlm_model.__class__.__name__ == "Qwen2_5OmniThinkerForConditionalGeneration": return self._omni_hidden_states_without_lm_head(vlm_kwargs) return self.vlm_model(**vlm_kwargs).hidden_states[-1] # TODO-yth: forward入口,vlm(Qwen3VLForConditionalGeneration)输出hidden最后一层到action_head,得到action_pred和目标噪声算MSE def forward( self, input_ids, attention_mask, pixel_values, image_grid_thw, action_samples: torch.Tensor, states: torch.Tensor, timestep: torch.LongTensor, # <modify_audio> 支持Qwen2.5-Omni audio-only processor输出的音频特征。 input_features=None, feature_attention_mask=None, audio_feature_lengths=None, vlm_hidden_states=None, pixel_values_videos=None, video_grid_thw=None, video_second_per_grid=None, traj2ds: Optional[torch.Tensor] = None, progress: Optional[torch.Tensor] = None, future_progress: Optional[torch.Tensor] = None, return_dict: bool = True, skip_layers: Optional[List[int]] = None, known_action_prefix: Optional[torch.Tensor] = None, ) -> Union[List[torch.Tensor], HumanFoundationModelOutput]: # <modify> # qwen2.5里面有一行delta0 = (1 - attention_mask),而qwen3这边接收的mask是bool,所以这里要转成long vlm_attention_mask = attention_mask if vlm_attention_mask is not None and vlm_attention_mask.dtype == torch.bool: vlm_attention_mask = vlm_attention_mask.to(dtype=torch.long) # <modify_audio> 原逻辑固定传pixel_values/image_grid_thw;现在按实际batch动态拼VLM输入,避免Qwen3VL收到音频空参数。 # 原逻辑: # vlm_hidden_states = self.vlm_model( # input_ids=input_ids, # attention_mask=vlm_attention_mask, # pixel_values=pixel_values, # image_grid_thw=image_grid_thw, # output_hidden_states=True, # return_dict=True # ).hidden_states[-1] vlm_kwargs = { "input_ids": input_ids, "attention_mask": vlm_attention_mask, "output_hidden_states": True, "return_dict": True, "use_cache": False, } if pixel_values is not None: vlm_kwargs["pixel_values"] = pixel_values if image_grid_thw is not None: vlm_kwargs["image_grid_thw"] = image_grid_thw if input_features is not None: vlm_kwargs["input_features"] = input_features if feature_attention_mask is not None: vlm_kwargs["feature_attention_mask"] = feature_attention_mask if audio_feature_lengths is not None: vlm_kwargs["audio_feature_lengths"] = audio_feature_lengths if pixel_values_videos is not None: vlm_kwargs["pixel_values_videos"] = pixel_values_videos if video_grid_thw is not None: vlm_kwargs["video_grid_thw"] = video_grid_thw if video_second_per_grid is not None: vlm_kwargs["video_second_per_grid"] = video_second_per_grid # extract vision/audio + language features, or reuse precomputed Qwen-Omni hidden states. if vlm_hidden_states is None: if self.vlm_model is None: raise RuntimeError( "VLM backbone is not loaded because PSI_SKIP_VLM_MODEL=1; " "this training run requires every sample to provide cached vlm_hidden_states." ) vlm_hidden_states = self._extract_vlm_hidden_states(vlm_kwargs) else: vlm_hidden_states = vlm_hidden_states.to(device=states.device, dtype=states.dtype) # print(vlm_hidden_states.shape) model_output = self.action_header( hidden_states=None, timestep=timestep, joint_attention_kwargs=dict( action_hidden_embeds=action_samples, known_action_prefix=known_action_prefix, views=vlm_hidden_states.unsqueeze(1), # (B,V,N,D) obs=states, # (B,1,M) traj2ds=traj2ds, # (B, C, 3, H, W) progress=progress, future_progress=future_progress, ), vlm_attn_mask=attention_mask, # (B, seq_len) return_dict=return_dict, ) return HumanFoundationModelOutput(action=model_output.action) def _build_video_only_inputs( self, texts, video=None, video_embed=None, video_size_pixels: int = 50176, video_sample_fps: float = 1.0, video_pixel_cache_dtype: str = "none", ): if video_embed is not None: pixel_values_videos = torch.as_tensor( video_embed["pixel_values_videos"], dtype=torch.float32, device=self.device, ) video_grid_thw = torch.as_tensor( video_embed["video_grid_thw"], dtype=torch.long, device=self.device, ) if video_grid_thw.dim() == 1: video_grid_thw = video_grid_thw.unsqueeze(0) if "video_second_per_grid" in video_embed: video_second_per_grid = torch.as_tensor( video_embed["video_second_per_grid"], dtype=torch.float32, device=self.device, ).reshape(-1) else: video_second_per_grid = torch.tensor( [float(self.vlm_processor.video_processor.temporal_patch_size) / float(video_sample_fps)], dtype=torch.float32, device=self.device, ) texts = self.vlm_processor.replace_multimodal_special_tokens( texts, iter([]), iter([]), iter([video_grid_thw[0].detach().cpu()]), iter([float(video_second_per_grid.reshape(-1)[0].item())]), False, 25, 2.0, ) text_inputs = self.vlm_processor.tokenizer( texts, padding=True, return_tensors="pt", ) inputs = {key: value.to(self.device) for key, value in text_inputs.items()} inputs["pixel_values_videos"] = pixel_values_videos inputs["video_grid_thw"] = video_grid_thw inputs["video_second_per_grid"] = video_second_per_grid return inputs inputs = self.vlm_processor( text=texts, videos=[np.asarray(video, dtype=np.uint8)], padding=True, return_tensors="pt", use_audio_in_video=False, videos_kwargs={ "size": { "shortest_edge": int(video_size_pixels), "longest_edge": int(video_size_pixels), }, "fps": float(video_sample_fps), }, ).to(self.device) # The v23 video-clip training cache stores Qwen pixel_values_videos as # FP16 before exporting BF16 hidden states. Reproduce that quantization # for raw-video inference so the frozen Qwen input matches training. cache_dtype = str(video_pixel_cache_dtype).strip().lower() if cache_dtype not in {"", "none"}: quant_dtype = { "fp16": torch.float16, "float16": torch.float16, "bf16": torch.bfloat16, "bfloat16": torch.bfloat16, }.get(cache_dtype) if quant_dtype is None: raise ValueError(f"Unsupported video_pixel_cache_dtype={video_pixel_cache_dtype!r}") inputs["pixel_values_videos"] = ( inputs["pixel_values_videos"].to(dtype=quant_dtype).to(dtype=torch.float32) ) return inputs @torch.inference_mode() def predict_action( self, observations: Optional[List[List[Image.Image]]], # B * List of PIL Image as [view1, view2] states: torch.Tensor, # (B, Ts, Ds) instructions: List[str], # (B,) num_inference_steps: int, traj2ds, **kwargs: str ) -> torch.Tensor: bsz = states.shape[0] progress = kwargs.get("progress", None) future_progress = kwargs.get("future_progress", None) text_only = bool(getattr(self, "text_only", False)) audio_only = bool(getattr(self, "audio_only", False)) video_only = bool(getattr(self, "video_only", False)) batch_input_ids = [] batch_attention_mask = [] batch_pixel_values = [] if (not text_only and not audio_only and not video_only) else None batch_image_grid_thw = [] if (not text_only and not audio_only and not video_only) else None # <modify_audio> audio推理时从kwargs接收每个batch样本的waveform片段,可同时使用文本指令。 batch_input_features = [] if audio_only else None batch_feature_attention_mask = [] if audio_only else None batch_pixel_values_videos = [] if video_only else None batch_video_grid_thw = [] if video_only else None batch_video_second_per_grid = [] if video_only else None if audio_only: audios = kwargs.get("audios", None) assert audios is not None, "audios must be provided for audio-only inference" audio_items = list(audios) audio_instructions = list(instructions or [""] * len(audio_items)) assert len(audio_instructions) == len(audio_items), "instructions must match audios for audio-only inference" items = list(zip(audio_items, audio_instructions)) elif video_only: video_embeds = kwargs.get("video_embeds", None) if video_embeds is not None: video_paths = kwargs.get("video_paths", ["video.mp4"] * len(video_embeds)) items = list(zip(video_embeds, video_paths, instructions)) else: videos = kwargs.get("videos", None) assert videos is not None, "videos must be provided for video-only inference" video_paths = kwargs.get("video_paths", ["video.mp4"] * len(videos)) items = list(zip(videos, video_paths, instructions)) elif text_only: items = [(None, instruction) for instruction in instructions] else: assert observations is not None, "observations must be provided for non-text-only inference" items = list(zip(observations, instructions)) for item in items: if audio_only: audio, instruction = item content = [{"type": "audio", "audio": "audio.wav"}] if str(instruction).strip(): content.append({"type": "text", "text": str(instruction).strip()}) messages = [[ {"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]}, {"role": "user", "content": content}, ]] elif video_only: video_item, video_path, instruction = item content = [{"type": "video", "video": str(video_path or "video.mp4")}] if str(instruction).strip(): content.append({"type": "text", "text": str(instruction).strip()}) messages = [[ {"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]}, {"role": "user", "content": content}, ]] elif text_only: _, instruction = item messages = [[ {"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]}, {"role": "user", "content": [{"type": "text", "text": instruction}]}, ]] else: observation, instruction = item assert observation is not None messages = [] content = [{"type": "image", "image": img} for img in observation] content.append({"type": "text", "text": instruction}) user_msg = {"role": "user", "content": content} messages.append([user_msg]) texts = [ self.vlm_processor.apply_chat_template( m, tokenize=False, add_generation_prompt=True ) for m in messages ] if audio_only: inputs = self.vlm_processor( text=texts, audio=[np.asarray(audio, dtype=np.float32)], padding=True, return_tensors="pt", ).to(self.device) elif video_only: video_size_pixels = int(kwargs.get("video_size_pixels", getattr(self, "video_size_pixels", 50176))) video_sample_fps = float(kwargs.get("video_sample_fps", 1.0)) inputs = self._build_video_only_inputs( texts, video=None if isinstance(video_item, dict) else video_item, video_embed=video_item if isinstance(video_item, dict) else None, video_size_pixels=video_size_pixels, video_sample_fps=video_sample_fps, video_pixel_cache_dtype=str(kwargs.get("video_pixel_cache_dtype", "none")), ) elif text_only: inputs = self.vlm_processor( text=texts, padding=True, return_tensors="pt", ).to(self.device) else: image_inputs, video_inputs = process_vision_info(messages, image_patch_size=16) inputs = self.vlm_processor( text=texts, images=image_inputs, videos=video_inputs, padding=True, return_tensors="pt", ).to(self.device) batch_input_ids.append(inputs['input_ids'].squeeze(0)) batch_attention_mask.append(inputs['attention_mask'].squeeze(0)) if audio_only: assert batch_input_features is not None and batch_feature_attention_mask is not None batch_input_features.append(inputs["input_features"].squeeze(0)) batch_feature_attention_mask.append(inputs["feature_attention_mask"].squeeze(0)) if video_only: assert batch_pixel_values_videos is not None and batch_video_grid_thw is not None assert batch_video_second_per_grid is not None batch_pixel_values_videos.append(inputs["pixel_values_videos"].squeeze(0)) batch_video_grid_thw.append(inputs["video_grid_thw"].squeeze(0)) batch_video_second_per_grid.append(inputs["video_second_per_grid"].reshape(-1)) if not text_only and not audio_only and not video_only: assert batch_pixel_values is not None and batch_image_grid_thw is not None batch_pixel_values.append(inputs['pixel_values']) batch_image_grid_thw.append(inputs['image_grid_thw'].squeeze(0)) pad_token_id = int(getattr(self.vlm_processor.tokenizer, "pad_token_id", 0)) batch_input_ids = torch.nn.utils.rnn.pad_sequence( batch_input_ids, batch_first=True, padding_value=pad_token_id ) batch_attention_mask = torch.nn.utils.rnn.pad_sequence( batch_attention_mask, batch_first=True, padding_value=0 ) if audio_only: assert batch_input_features is not None and batch_feature_attention_mask is not None # <modify_full_audio> 整段音频推理时不同样本audio feature长度可能不同,按batch最长补零。 max_audio_len = max(int(feature.shape[-1]) for feature in batch_input_features) padded_input_features = [] padded_feature_attention_mask = [] for feature, feature_mask in zip(batch_input_features, batch_feature_attention_mask): pad_len = max_audio_len - int(feature.shape[-1]) if pad_len > 0: feature_pad_shape = list(feature.shape) feature_pad_shape[-1] = pad_len feature = torch.cat( [feature, torch.zeros(feature_pad_shape, dtype=feature.dtype, device=feature.device)], dim=-1, ) mask_pad_shape = list(feature_mask.shape) mask_pad_shape[-1] = pad_len feature_mask = torch.cat( [feature_mask, torch.zeros(mask_pad_shape, dtype=feature_mask.dtype, device=feature_mask.device)], dim=-1, ) padded_input_features.append(feature) padded_feature_attention_mask.append(feature_mask) batch_input_features = torch.stack(padded_input_features) batch_feature_attention_mask = torch.stack(padded_feature_attention_mask) if not text_only and not audio_only and not video_only: assert batch_pixel_values is not None and batch_image_grid_thw is not None batch_pixel_values = torch.stack(batch_pixel_values) # (B, 256, 1536) batch_image_grid_thw = torch.stack(batch_image_grid_thw) # (B, 3) if video_only: assert batch_pixel_values_videos is not None and batch_video_grid_thw is not None assert batch_video_second_per_grid is not None batch_pixel_values_videos = torch.cat(batch_pixel_values_videos, dim=0) batch_video_grid_thw = torch.stack(batch_video_grid_thw) batch_video_second_per_grid = torch.cat(batch_video_second_per_grid, dim=0) # <modify_float> 推理时按infer_autocast_dtype选择精度;float32时不启用autocast。 # 原逻辑: # autocast_ctx = ( # torch.autocast("cuda", dtype=torch.bfloat16) # if str(self.device).startswith("cuda") # else contextlib.nullcontext() # ) infer_dtype = getattr(self, "infer_autocast_dtype", torch.bfloat16) if str(self.device).startswith("cuda") and infer_dtype in (torch.float16, torch.bfloat16): autocast_ctx = torch.autocast("cuda", dtype=infer_dtype) else: autocast_ctx = contextlib.nullcontext() with autocast_ctx: # extract vision + language features if audio_only: output = self.vlm_model( input_ids=batch_input_ids, attention_mask=batch_attention_mask, input_features=batch_input_features, # type: ignore feature_attention_mask=batch_feature_attention_mask, # type: ignore output_hidden_states=True, return_dict=True ) elif video_only: output = self.vlm_model( input_ids=batch_input_ids, attention_mask=batch_attention_mask, pixel_values_videos=batch_pixel_values_videos, # type: ignore video_grid_thw=batch_video_grid_thw, # type: ignore video_second_per_grid=batch_video_second_per_grid, # type: ignore output_hidden_states=True, return_dict=True, ) elif text_only: output = self.vlm_model( input_ids=batch_input_ids, attention_mask=batch_attention_mask, output_hidden_states=True, return_dict=True ) else: output = self.vlm_model( input_ids=batch_input_ids, attention_mask=batch_attention_mask, pixel_values=batch_pixel_values, # type: ignore image_grid_thw=batch_image_grid_thw, # type: ignore output_hidden_states=True, return_dict=True ) vlm_hidden_states_ = output.hidden_states # len(vlm_hidden_states_) == 29 # use hidden states from the last layer vlm_hidden_states = vlm_hidden_states_[-1] # shape (B, seq_len, D_h) shape(16, 80, 2048) vlm_hidden_states = self._action_condition_hidden_states( vlm_hidden_states.to(dtype=states.dtype), input_ids=batch_input_ids, attention_mask=batch_attention_mask, ) # shape (B, 1, seq_len, D_h) or subclass-specific latent tokens # generate action from noise action_samples = torch.randn( bsz, self.action_horizon, self.action_dim, device=self.device ) self.noise_scheduler.set_timesteps(num_inference_steps) for timestep in self.noise_scheduler.timesteps: batched_timestep = timestep.expand(bsz).to(self.device) model_pred = self.action_header( hidden_states=None, timestep=batched_timestep, joint_attention_kwargs=dict( action_hidden_embeds=action_samples, # (B,Tp,Da) views=vlm_hidden_states, # (B,V,N,D) obs=states, # (B,1,M) traj2ds=traj2ds, # (B, C, 3, H, W) progress=progress, future_progress=future_progress, ), return_dict=True, ).action action_samples = self.noise_scheduler.step( model_output=model_pred, timestep=timestep, sample=action_samples # type: ignore ).prev_sample return action_samples.float() @torch.inference_mode() def predict_action_with_training_rtc_flow( self, observations: Optional[List[List[Image.Image]]], # B * List of PIL Image as [view1, view2] states: torch.Tensor, # (B, Ts, Ds) instructions: List[str], # (B,) num_inference_steps: int, traj2ds, prev_actions: torch.Tensor = None, # (1, H, D) inference_delay: int = 0, max_delay: int = 0, **kwargs: str ) -> torch.Tensor: """Predict a full RTC chunk with v18-style clean overlap positions. Callers shift the unexecuted tail of the previous chunk into the first ``inference_delay`` positions. All 30 positions use the same action forward; the overlap positions are reset to those clean actions before every denoising step and in the returned chunk. """ # RTC overlap: [known tail from previous chunk | newly generated tail] H = self.action_horizon bsz = states.shape[0] if prev_actions is None or inference_delay <= 0 or max_delay <= 0: raise ValueError( "prev_actions, a positive inference_delay, and max_delay " "must be provided for RTC inference" ) if prev_actions.ndim != 3 or prev_actions.shape != (bsz, H, self.action_dim): raise ValueError( "prev_actions must have shape " f"({bsz}, {H}, {self.action_dim}), got {tuple(prev_actions.shape)}" ) prev_actions = prev_actions.to(device=self.device, dtype=torch.float32) if not hasattr(self, "action_exec_horizon") or not hasattr(self, "rtc_max_delay"): raise RuntimeError( "RTC model is missing action_exec_horizon/max_delay deployment metadata" ) if int(max_delay) != int(self.rtc_max_delay): raise ValueError( "RTC caller max_delay does not match the loaded model config: " f"caller={max_delay}, model={self.rtc_max_delay}" ) d = _validate_rtc_inference_contract( action_horizon=H, action_exec_horizon=int(self.action_exec_horizon), inference_delay=int(inference_delay), max_delay=int(self.rtc_max_delay), ) progress = kwargs.get("progress", None) future_progress = kwargs.get("future_progress", None) text_only = bool(getattr(self, "text_only", False)) audio_only = bool(getattr(self, "audio_only", False)) video_only = bool(getattr(self, "video_only", False)) batch_input_ids = [] batch_attention_mask = [] batch_pixel_values = [] if (not text_only and not audio_only and not video_only) else None batch_image_grid_thw = [] if (not text_only and not audio_only and not video_only) else None # <modify_audio> audio RTC推理同样从kwargs接收waveform片段,可同时使用文本指令。 batch_input_features = [] if audio_only else None batch_feature_attention_mask = [] if audio_only else None batch_pixel_values_videos = [] if video_only else None batch_video_grid_thw = [] if video_only else None batch_video_second_per_grid = [] if video_only else None if audio_only: audios = kwargs.get("audios", None) assert audios is not None, "audios must be provided for audio-only inference" audio_items = list(audios) audio_instructions = list(instructions or [""] * len(audio_items)) assert len(audio_instructions) == len(audio_items), "instructions must match audios for audio-only inference" items = list(zip(audio_items, audio_instructions)) elif video_only: video_embeds = kwargs.get("video_embeds", None) if video_embeds is not None: video_paths = kwargs.get("video_paths", ["video.mp4"] * len(video_embeds)) items = list(zip(video_embeds, video_paths, instructions)) else: videos = kwargs.get("videos", None) assert videos is not None, "videos must be provided for video-only inference" video_paths = kwargs.get("video_paths", ["video.mp4"] * len(videos)) items = list(zip(videos, video_paths, instructions)) elif text_only: items = [(None, instruction) for instruction in instructions] else: assert observations is not None, "observations must be provided for non-text-only inference" items = list(zip(observations, instructions)) for item in items: if audio_only: audio, instruction = item content = [{"type": "audio", "audio": "audio.wav"}] if str(instruction).strip(): content.append({"type": "text", "text": str(instruction).strip()}) messages = [[ {"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]}, {"role": "user", "content": content}, ]] elif video_only: video_item, video_path, instruction = item content = [{"type": "video", "video": str(video_path or "video.mp4")}] if str(instruction).strip(): content.append({"type": "text", "text": str(instruction).strip()}) messages = [[ {"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]}, {"role": "user", "content": content}, ]] elif text_only: _, instruction = item messages = [[ {"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]}, {"role": "user", "content": [{"type": "text", "text": instruction}]}, ]] else: observation, instruction = item assert observation is not None messages = [] content = [{"type": "image", "image": img} for img in observation] content.append({"type": "text", "text": instruction}) user_msg = {"role": "user", "content": content} messages.append([user_msg]) texts = [ self.vlm_processor.apply_chat_template( m, tokenize=False, add_generation_prompt=True ) for m in messages ] if audio_only: inputs = self.vlm_processor( text=texts, audio=[np.asarray(audio, dtype=np.float32)], padding=True, return_tensors="pt", ).to(self.device) elif video_only: video_size_pixels = int(kwargs.get("video_size_pixels", getattr(self, "video_size_pixels", 50176))) video_sample_fps = float(kwargs.get("video_sample_fps", 1.0)) inputs = self._build_video_only_inputs( texts, video=None if isinstance(video_item, dict) else video_item, video_embed=video_item if isinstance(video_item, dict) else None, video_size_pixels=video_size_pixels, video_sample_fps=video_sample_fps, video_pixel_cache_dtype=str(kwargs.get("video_pixel_cache_dtype", "none")), ) elif text_only: inputs = self.vlm_processor( text=texts, padding=True, return_tensors="pt", ).to(self.device) else: image_inputs, video_inputs = process_vision_info(messages, image_patch_size=16) inputs = self.vlm_processor( text=texts, images=image_inputs, videos=video_inputs, padding=True, return_tensors="pt", ).to(self.device) batch_input_ids.append(inputs['input_ids'].squeeze(0)) batch_attention_mask.append(inputs['attention_mask'].squeeze(0)) if audio_only: assert batch_input_features is not None and batch_feature_attention_mask is not None batch_input_features.append(inputs["input_features"].squeeze(0)) batch_feature_attention_mask.append(inputs["feature_attention_mask"].squeeze(0)) if video_only: assert batch_pixel_values_videos is not None and batch_video_grid_thw is not None assert batch_video_second_per_grid is not None batch_pixel_values_videos.append(inputs["pixel_values_videos"].squeeze(0)) batch_video_grid_thw.append(inputs["video_grid_thw"].squeeze(0)) batch_video_second_per_grid.append(inputs["video_second_per_grid"].reshape(-1)) if not text_only and not audio_only and not video_only: assert batch_pixel_values is not None and batch_image_grid_thw is not None batch_pixel_values.append(inputs['pixel_values']) batch_image_grid_thw.append(inputs['image_grid_thw'].squeeze(0)) pad_token_id = int(getattr(self.vlm_processor.tokenizer, "pad_token_id", 0)) batch_input_ids = torch.nn.utils.rnn.pad_sequence( batch_input_ids, batch_first=True, padding_value=pad_token_id ) batch_attention_mask = torch.nn.utils.rnn.pad_sequence( batch_attention_mask, batch_first=True, padding_value=0 ) if audio_only: assert batch_input_features is not None and batch_feature_attention_mask is not None # <modify_full_audio> 整段音频RTC推理同样补齐batch内audio feature长度。 max_audio_len = max(int(feature.shape[-1]) for feature in batch_input_features) padded_input_features = [] padded_feature_attention_mask = [] for feature, feature_mask in zip(batch_input_features, batch_feature_attention_mask): pad_len = max_audio_len - int(feature.shape[-1]) if pad_len > 0: feature_pad_shape = list(feature.shape) feature_pad_shape[-1] = pad_len feature = torch.cat( [feature, torch.zeros(feature_pad_shape, dtype=feature.dtype, device=feature.device)], dim=-1, ) mask_pad_shape = list(feature_mask.shape) mask_pad_shape[-1] = pad_len feature_mask = torch.cat( [feature_mask, torch.zeros(mask_pad_shape, dtype=feature_mask.dtype, device=feature_mask.device)], dim=-1, ) padded_input_features.append(feature) padded_feature_attention_mask.append(feature_mask) batch_input_features = torch.stack(padded_input_features) batch_feature_attention_mask = torch.stack(padded_feature_attention_mask) if not text_only and not audio_only and not video_only: assert batch_pixel_values is not None and batch_image_grid_thw is not None batch_pixel_values = torch.stack(batch_pixel_values) # (B, 256, 1536) batch_image_grid_thw = torch.stack(batch_image_grid_thw) # (B, 3) if video_only: assert batch_pixel_values_videos is not None and batch_video_grid_thw is not None assert batch_video_second_per_grid is not None batch_pixel_values_videos = torch.cat(batch_pixel_values_videos, dim=0) batch_video_grid_thw = torch.stack(batch_video_grid_thw) batch_video_second_per_grid = torch.cat(batch_video_second_per_grid, dim=0) # <modify_float> RTC推理同样按infer_autocast_dtype选择精度;float32时不启用autocast。 # 原逻辑: # autocast_ctx = ( # torch.autocast("cuda", dtype=torch.bfloat16) # if str(self.device).startswith("cuda") # else contextlib.nullcontext() # ) infer_dtype = getattr(self, "infer_autocast_dtype", torch.bfloat16) if str(self.device).startswith("cuda") and infer_dtype in (torch.float16, torch.bfloat16): autocast_ctx = torch.autocast("cuda", dtype=infer_dtype) else: autocast_ctx = contextlib.nullcontext() with autocast_ctx: # extract vision + language features if audio_only: output = self.vlm_model( input_ids=batch_input_ids, attention_mask=batch_attention_mask, input_features=batch_input_features, # type: ignore feature_attention_mask=batch_feature_attention_mask, # type: ignore output_hidden_states=True, return_dict=True ) elif video_only: output = self.vlm_model( input_ids=batch_input_ids, attention_mask=batch_attention_mask, pixel_values_videos=batch_pixel_values_videos, # type: ignore video_grid_thw=batch_video_grid_thw, # type: ignore video_second_per_grid=batch_video_second_per_grid, # type: ignore output_hidden_states=True, return_dict=True, ) elif text_only: output = self.vlm_model( input_ids=batch_input_ids, attention_mask=batch_attention_mask, output_hidden_states=True, return_dict=True ) else: output = self.vlm_model( input_ids=batch_input_ids, attention_mask=batch_attention_mask, pixel_values=batch_pixel_values, # type: ignore image_grid_thw=batch_image_grid_thw, # type: ignore output_hidden_states=True, return_dict=True ) vlm_hidden_states_ = output.hidden_states # len(vlm_hidden_states_) == 29 # use hidden states from the last layer vlm_hidden_states = vlm_hidden_states_[-1] # shape (B, seq_len, D_h) shape(16, 80, 2048) vlm_hidden_states = self._action_condition_hidden_states( vlm_hidden_states.to(dtype=states.dtype), input_ids=batch_input_ids, attention_mask=batch_attention_mask, ) # shape (B, 1, seq_len, D_h) or subclass-specific latent tokens # v18-style RTC: keep the complete 30-frame action path. The # known prefix is overwritten with clean previous actions before # every denoising step and receives timestep zero. action_samples = torch.randn( bsz, self.action_horizon, self.action_dim, device=self.device ) prefix_mask = ( torch.arange(H, device=self.device)[None, :] < d ).expand(bsz, -1) self.noise_scheduler.set_timesteps(num_inference_steps) # self.noise_scheduler.timesteps: tensor([1000., 889., 778., 667., 556., 445., 334., 223., 112., 1.]) # self.noise_scheduler.sigmas: tensor([1.0000, 0.8890, 0.7780, 0.6670, 0.5560, 0.4450, 0.3340, 0.2230, 0.1120, 0.0010, 0.0000]) for timestep in self.noise_scheduler.timesteps: batched_timestep = torch.where( prefix_mask, torch.zeros((), device=self.device, dtype=timestep.dtype), timestep, ) action_samples = torch.where( prefix_mask[:, :, None], prev_actions, action_samples ) model_pred = self.action_header( hidden_states=None, timestep=batched_timestep, joint_attention_kwargs=dict( action_hidden_embeds=action_samples, known_action_prefix=None, views=vlm_hidden_states, # (B,V,N,D) obs=states, # (B,1,M) traj2ds=traj2ds, # (B, C, 3, H, W) progress=progress, future_progress=future_progress, ), return_dict=True, ).action action_samples = self.noise_scheduler.step( model_output=model_pred, timestep=timestep, sample=action_samples, ).prev_sample # The old implementation overwrote the prefix at the start of each # solver step. Re-apply it once after the final step as well so the # returned overlap is exact rather than solver-perturbed. action_samples = torch.where( prefix_mask[:, :, None], prev_actions, action_samples ) return action_samples.float() def predict_action_with_rtc_flow( self, observations: List[List[Image.Image]], # B * List of PIL Image as [view1, view2] states: torch.Tensor, # (B, Ts, Ds) instructions: List[str], # (B,) num_inference_steps: int, traj2ds, prev_actions: torch.Tensor = None, # (1, H, D) inference_delay: int = 0, execution_horizon: int = 0, mask_schedule: str = "exponential", guidance_weight: float = 5.0, **kwargs: str ) -> np.ndarray: with torch.no_grad(): ## RTC related ## H = self.action_horizon assert prev_actions is not None and inference_delay > 0 and execution_horizon > 0, "prev_actions, inference_delay and execution_horizon must be provided" assert prev_actions.shape[0] == 1 prev_actions = prev_actions.to(device=self.device, dtype=torch.float32) # Create soft mask for inpainting d = inference_delay s = execution_horizon # Validate constraint from paper: d ≤ s ≤ H - d assert d <= s and s <= H - d, f"Constraint violated: d={d}, s={s}, H={H}. Need d ≤ s ≤ H-d" # Create soft mask [H] according to Equation 5 mask = self._create_soft_mask( H, d, s, schedule=mask_schedule, device=self.device ) mask_expanded = mask.view(1, -1, 1).expand(1, self.action_horizon, self.action_dim) ## ## bsz = states.shape[0] progress = kwargs.get("progress", None) future_progress = kwargs.get("future_progress", None) batch_input_ids = [] batch_attention_mask = [] batch_pixel_values = [] batch_image_grid_thw = [] for observation, instruction in zip(observations, instructions): messages = [] content = [{"type": "image", "image": img} for img in observation] content.append({"type": "text", "text": instruction}) user_msg = {"role": "user", "content": content} messages.append([user_msg]) texts = [ self.vlm_processor.apply_chat_template( m, tokenize=False, add_generation_prompt=True ) for m in messages ] image_inputs, video_inputs = process_vision_info(messages, image_patch_size=16) inputs = self.vlm_processor( text=texts, images=image_inputs, videos=video_inputs, padding=True, return_tensors="pt", ).to(self.device) # input_ids torch.Size([1, 80]) # attention_mask torch.Size([1, 80]) # pixel_values torch.Size([256, 1536]) # image_grid_thw torch.Size([1, 3]) batch_input_ids.append(inputs['input_ids'].squeeze(0)) batch_attention_mask.append(inputs['attention_mask'].squeeze(0)) batch_pixel_values.append(inputs['pixel_values']) batch_image_grid_thw.append(inputs['image_grid_thw'].squeeze(0)) batch_input_ids = torch.stack(batch_input_ids) # (B, 80) batch_attention_mask = torch.stack(batch_attention_mask) # (B, 80) batch_pixel_values = torch.stack(batch_pixel_values) # (B, 256, 1536) batch_image_grid_thw = torch.stack(batch_image_grid_thw) # (B, 3) with torch.autocast("cuda", dtype=torch.bfloat16): # extract vision + language features output = self.vlm_model( input_ids=batch_input_ids, attention_mask=batch_attention_mask, pixel_values=batch_pixel_values, image_grid_thw=batch_image_grid_thw, output_hidden_states=True, return_dict=True ) vlm_hidden_states_ = output.hidden_states # len(vlm_hidden_states_) == 29 # use hidden states from the last layer vlm_hidden_states = vlm_hidden_states_[-1] # shape (B, seq_len, D_h) shape(16, 80, 2048) vlm_hidden_states = vlm_hidden_states.unsqueeze(1) # shape (B, 1, seq_len, D_h) (16, 1, 80, 2048) # generate action from noise action_samples = torch.randn( bsz, self.action_horizon, self.action_dim, device=self.device ) # target_noise = action_samples.clone().detach() self.noise_scheduler.set_timesteps(num_inference_steps) # self.noise_scheduler.timesteps: tensor([1000., 889., 778., 667., 556., 445., 334., 223., 112., 1.]) # self.noise_scheduler.sigmas: tensor([1.0000, 0.8890, 0.7780, 0.6670, 0.5560, 0.4450, 0.3340, 0.2230, 0.1120, 0.0010, 0.0000]) # self.noise_scheduler.timesteps: tensor([1000., 889., 778., 667., 556., 445., 334., 223., 112., 1.]) # self.noise_scheduler.sigmas: tensor([1.0000, 0.8890, 0.7780, 0.6670, 0.5560, 0.4450, 0.3340, 0.2230, 0.1120, 0.0010, 0.0000]) for timestep in self.noise_scheduler.timesteps: batched_timestep = timestep.expand(bsz).to(self.device).detach() ### pseudo inverse (with gradient): # Enable gradient only for this step with torch.enable_grad(): # Detach and enable gradient only for sample_actions in this step action_samples_grad = action_samples.detach().requires_grad_(True) model_pred = self.action_header( hidden_states=None, timestep=batched_timestep, joint_attention_kwargs=dict( action_hidden_embeds=action_samples_grad, # (B,Tp,Da) views=vlm_hidden_states, # (B,V,N,D) obs=states, # (B,1,M) traj2ds=traj2ds, # (B, C, 3, H, W) progress=progress, future_progress=future_progress, ), return_dict=True, ).action # 1. 外推干净data X_0 tau = self.noise_scheduler.sigmas[self.noise_scheduler.index_for_timestep(timestep)].detach() # DEBUG test inpainting # noisy_prev_actions = (1 - tau) * prev_actions + tau * target_noise # action_samples[:, :d, 14:28] = noisy_prev_actions[:, :d, 14:28] # print("tau:", tau.item()) # print("timestep:", timestep) # print("self.scheduler.sigmas:", self.scheduler.sigmas) # print("self.scheduler.index_for_timestep(timestep):", self.scheduler.index_for_timestep(timestep)) pred_x0 = action_samples_grad - tau * model_pred # pred_x0 = epsilon - model_pred # print("pred_x0:", pred_x0) # 2. 计算(Y - pred_X_0) * weights error = (prev_actions - pred_x0.detach()) * mask_expanded # print("error:", error) # print("error.abs().mean():", error.abs().mean()) # 3. vjp pinv_correction = torch.autograd.grad( outputs=pred_x0, # [1,T,D] inputs=action_samples_grad, # [1,T,D] grad_outputs=error, # 匹配形状 [1,T,D] retain_graph=False, create_graph=False )[0] # 输出 [1,T,D] # print("pinv_correction:", pinv_correction) # 4. 计算tau相关参数,得到修正项,修正v inv_r2 = (tau**2 + (1 - tau) ** 2) / (tau**2) c = torch.nan_to_num(tau / (1 - tau), posinf=guidance_weight) g = torch.clamp(c * inv_r2, max=guidance_weight) # print("c * inv_r2:", c * inv_r2) # print("model_pred_before_correction:", model_pred) model_pred = model_pred - g * pinv_correction # print("pinv_correction.abs().mean():", pinv_correction.abs().mean()) ### action_samples = self.noise_scheduler.step( model_output=model_pred, timestep=timestep, sample=action_samples # type: ignore ).prev_sample # DEBUG test inpainting # if i == len(self.noise_scheduler.timesteps) - 1: # action_samples[:, :d, 14:28] = prev_actions[:, :d, 14:28] return action_samples def predict_action_with_rtc_flow_naive_inpaint( self, observations: List[List[Image.Image]], # B * List of PIL Image as [view1, view2] states: torch.Tensor, # (B, Ts, Ds) instructions: List[str], # (B,) num_inference_steps: int, traj2ds, prev_actions: torch.Tensor = None, # (1, H, D) inference_delay: int = 0, execution_horizon: int = 0, mask_schedule: str = "exponential", guidance_weight: float = 5.0, **kwargs: str ) -> np.ndarray: with torch.no_grad(): ## RTC related ## H = self.action_horizon assert prev_actions is not None and inference_delay > 0 and execution_horizon > 0, "prev_actions, inference_delay and execution_horizon must be provided" assert prev_actions.shape[0] == 1 prev_actions = prev_actions.to(device=self.device, dtype=torch.float32) # Create soft mask for inpainting d = inference_delay s = execution_horizon # Validate constraint from paper: d ≤ s ≤ H - d assert d <= s and s <= H - d, f"Constraint violated: d={d}, s={s}, H={H}. Need d ≤ s ≤ H-d" # Create soft mask [H] according to Equation 5 mask = self._create_soft_mask( H, d, s, schedule=mask_schedule, device=self.device ) mask_expanded = mask.view(1, -1, 1).expand(1, self.action_horizon, self.action_dim) ## ## bsz = states.shape[0] progress = kwargs.get("progress", None) future_progress = kwargs.get("future_progress", None) batch_input_ids = [] batch_attention_mask = [] batch_pixel_values = [] batch_image_grid_thw = [] for observation, instruction in zip(observations, instructions): messages = [] content = [{"type": "image", "image": img} for img in observation] content.append({"type": "text", "text": instruction}) user_msg = {"role": "user", "content": content} messages.append([user_msg]) texts = [ self.vlm_processor.apply_chat_template( m, tokenize=False, add_generation_prompt=True ) for m in messages ] image_inputs, video_inputs = process_vision_info(messages, image_patch_size=16) inputs = self.vlm_processor( text=texts, images=image_inputs, videos=video_inputs, padding=True, return_tensors="pt", ).to(self.device) # input_ids torch.Size([1, 80]) # attention_mask torch.Size([1, 80]) # pixel_values torch.Size([256, 1536]) # image_grid_thw torch.Size([1, 3]) batch_input_ids.append(inputs['input_ids'].squeeze(0)) batch_attention_mask.append(inputs['attention_mask'].squeeze(0)) batch_pixel_values.append(inputs['pixel_values']) batch_image_grid_thw.append(inputs['image_grid_thw'].squeeze(0)) batch_input_ids = torch.stack(batch_input_ids) # (B, 80) batch_attention_mask = torch.stack(batch_attention_mask) # (B, 80) batch_pixel_values = torch.stack(batch_pixel_values) # (B, 256, 1536) batch_image_grid_thw = torch.stack(batch_image_grid_thw) # (B, 3) with torch.autocast("cuda", dtype=torch.bfloat16): # extract vision + language features output = self.vlm_model( input_ids=batch_input_ids, attention_mask=batch_attention_mask, pixel_values=batch_pixel_values, image_grid_thw=batch_image_grid_thw, output_hidden_states=True, return_dict=True ) vlm_hidden_states_ = output.hidden_states # len(vlm_hidden_states_) == 29 # use hidden states from the last layer vlm_hidden_states = vlm_hidden_states_[-1] # shape (B, seq_len, D_h) shape(16, 80, 2048) vlm_hidden_states = vlm_hidden_states.unsqueeze(1) # shape (B, 1, seq_len, D_h) (16, 1, 80, 2048) # generate action from noise action_samples = torch.randn( bsz, self.action_horizon, self.action_dim, device=self.device ) self.noise_scheduler.set_timesteps(num_inference_steps) # Fixed noise for consistency in inpainting # Use the same noise as action_samples to ensure the whole sequence starts from the same latent x_1 target_noise = action_samples.clone() for i, timestep in enumerate(self.noise_scheduler.timesteps): batched_timestep = timestep.expand(bsz).to(self.device).detach() ### Naive inpainting (Repaint method): # Step 1: Add noise to prev_actions matching current timestep's noise level # This ensures consistency in noise levels across the entire action sequence tau = self.noise_scheduler.sigmas[self.noise_scheduler.index_for_timestep(timestep)] noisy_prev_actions = (1 - tau) * prev_actions + tau * target_noise # Step 2: Replace first d positions with noisy prev_actions (hard mask) action_samples[:, :d, :] = noisy_prev_actions[:, :d, :] # Step 3: Model prediction model_pred = self.action_header( hidden_states=None, timestep=batched_timestep, joint_attention_kwargs=dict( action_hidden_embeds=action_samples, # (B,Tp,Da) views=vlm_hidden_states, # (B,V,N,D) obs=states, # (B,1,M) traj2ds=traj2ds, # (B, C, 3, H, W) progress=progress, future_progress=future_progress, ), return_dict=True, ).action # Step 4: Denoise one step (updates all positions including first d) action_samples = self.noise_scheduler.step( model_output=model_pred, timestep=timestep, sample=action_samples # type: ignore ).prev_sample # Step 5: Re-inpaint with noise level matching the denoised state # TODO: do this or not??? boqian if i == len(self.noise_scheduler.timesteps) - 1: # Last step: use clean prev_actions (tau = 0) action_samples[:, :d, :] = prev_actions[:, :d, :] return action_samples # TODO-yth: 重要 def predict_action_with_rtc_flow_based_gradient( self, observations: List[List[Image.Image]], # B * List of PIL Image as [view1, view2] states: torch.Tensor, # (B, Ts, Ds) instructions: List[str], # (B,) num_inference_steps: int, traj2ds, prev_actions: torch.Tensor | None = None, # (1, H, D) inference_delay: int = 0, execution_horizon: int = 0, mask_schedule: str = "exponential", guidance_weight: float = 5.0, **kwargs: str ) -> np.ndarray: with torch.no_grad(): ## RTC related ## H = self.action_horizon assert prev_actions is not None and inference_delay > 0 and execution_horizon > 0, "prev_actions, inference_delay and execution_horizon must be provided" assert prev_actions.shape[0] == 1 prev_actions = prev_actions.to(device=self.device, dtype=torch.float32) # Create soft mask for inpainting d = inference_delay s = execution_horizon # Validate constraint from paper: d ≤ s ≤ H - d assert d <= s and s <= H - d, f"Constraint violated: d={d}, s={s}, H={H}. Need d ≤ s ≤ H-d" # Create soft mask [H] according to Equation 5 mask = self._create_soft_mask( H, d, s, schedule=mask_schedule, device=self.device ) mask_expanded = mask.view(1, -1, 1).expand(1, self.action_horizon, self.action_dim) ## ## bsz = states.shape[0] progress = kwargs.get("progress", None) future_progress = kwargs.get("future_progress", None) batch_input_ids = [] batch_attention_mask = [] batch_pixel_values = [] batch_image_grid_thw = [] for observation, instruction in zip(observations, instructions): messages = [] content = [{"type": "image", "image": img} for img in observation] content.append({"type": "text", "text": instruction}) user_msg = {"role": "user", "content": content} messages.append([user_msg]) texts = [ self.vlm_processor.apply_chat_template( m, tokenize=False, add_generation_prompt=True ) for m in messages ] image_inputs, video_inputs = process_vision_info(messages, image_patch_size=16) inputs = self.vlm_processor( text=texts, images=image_inputs, videos=video_inputs, padding=True, return_tensors="pt", ).to(self.device) # input_ids torch.Size([1, 80]) # attention_mask torch.Size([1, 80]) # pixel_values torch.Size([256, 1536]) # image_grid_thw torch.Size([1, 3]) batch_input_ids.append(inputs['input_ids'].squeeze(0)) batch_attention_mask.append(inputs['attention_mask'].squeeze(0)) batch_pixel_values.append(inputs['pixel_values']) batch_image_grid_thw.append(inputs['image_grid_thw'].squeeze(0)) batch_input_ids = torch.stack(batch_input_ids) # (B, 80) batch_attention_mask = torch.stack(batch_attention_mask) # (B, 80) batch_pixel_values = torch.stack(batch_pixel_values) # (B, 256, 1536) batch_image_grid_thw = torch.stack(batch_image_grid_thw) # (B, 3) with torch.autocast("cuda", dtype=torch.bfloat16): # extract vision + language features output = self.vlm_model( input_ids=batch_input_ids, attention_mask=batch_attention_mask, pixel_values=batch_pixel_values, image_grid_thw=batch_image_grid_thw, output_hidden_states=True, return_dict=True ) vlm_hidden_states_ = output.hidden_states # len(vlm_hidden_states_) == 29 # use hidden states from the last layer vlm_hidden_states = vlm_hidden_states_[-1] # shape (B, seq_len, D_h) shape(16, 80, 2048) vlm_hidden_states = vlm_hidden_states.unsqueeze(1) # shape (B, 1, seq_len, D_h) (16, 1, 80, 2048) # 初始化 action_samples action_samples = torch.randn( bsz, self.action_horizon, self.action_dim, device=self.device ) # 使用同样的 noise 构建目标,保证一致性 target_noise = action_samples.clone() self.noise_scheduler.set_timesteps(num_inference_steps) for i, timestep in enumerate(self.noise_scheduler.timesteps): # 1. 计算当前步的目标 (Noisy Target) # --------------------------------------------------------- tau = self.noise_scheduler.sigmas[self.noise_scheduler.index_for_timestep(timestep)] # 这是一个移动的目标:随着 tau 变小,它越来越接近真实的 prev_actions noisy_prev_actions = (1 - tau) * prev_actions + tau * target_noise # 2. 基于梯度的 Guidance (核心部分) # --------------------------------------------------------- # 我们希望 action_samples 在 mask 区域尽可能接近 noisy_prev_actions # 通过梯度下降来修改 action_samples # 定义 Guidance 的强度衰减: # 早期(噪声大)需要强引导来定型,晚期(噪声小)减弱引导以允许模型平滑过渡 current_scale = guidance_weight * (tau ** 0.5) # 这里的衰减策略可以调整,例如用 linear # 临时开启梯度计算 with torch.enable_grad(): # Detach 并开启梯度,创建一个新的叶子节点 x_in = action_samples.detach().requires_grad_(True) # 计算 Loss: 只在 mask 有值的地方计算 MSE # 这里的 mask_expanded 充当了权重的角色 # 重点:只计算前 s 步的差异 (inference_delay + execution_horizon) diff = (x_in - noisy_prev_actions) # 使用 mask 加权平方误差 # 形状: (B, H, D) * (1, H, D) loss = (diff ** 2 * mask_expanded).sum() # 计算梯度 grad = torch.autograd.grad(loss, x_in)[0] # 更新 action_samples (朝着 loss 减小的方向) # 注意:这里我们是在 Latent 空间进行微调 print("grad[:, :, 0:3].abs().mean():", grad[:, :, 0:3].abs().mean()) print("grad[:, :, 15:17].abs().mean():", grad[:, :, 15:17].abs().mean()) action_samples = action_samples - current_scale * grad # 3. 常规的 Flow Matching / Diffusion Step # --------------------------------------------------------- batched_timestep = timestep.expand(bsz).to(self.device).detach() model_pred = self.action_header( hidden_states=None, timestep=batched_timestep, joint_attention_kwargs=dict( action_hidden_embeds=action_samples, views=vlm_hidden_states, obs=states, traj2ds=traj2ds, progress=progress, future_progress=future_progress, ), return_dict=True, ).action # 4. Denoise one step action_samples = self.noise_scheduler.step( model_output=model_pred, timestep=timestep, sample=action_samples ).prev_sample # 5. (可选) Soft Blending 作为最后的保险 # 如果单纯靠梯度还是压不住,可以在最后一步稍微融合一下 # 但如果 guidance_weight 足够,通常不需要这步 # if i == len(self.noise_scheduler.timesteps) - 1: # # 最后一步不再有噪声,直接做一次软混合确保开头完全吻合 # action_samples = prev_actions * mask_expanded + action_samples * (1 - mask_expanded) return action_samples def _create_soft_mask(self, H, d, s, schedule="exponential", device="cpu"): """ Create soft mask for RTC inpainting (Equation 5 in paper). Paper: "Real-Time Execution of Action Chunking Flow Policies" Figure 3 and Equation 5: W_i = { 1 if i < d (frozen region) { c_i * (e^(c_i) - 1)/(e - 1) if d ≤ i < H - s (intermediate region) { 0 if i ≥ H - s (free region) where c_i = (H - s - i)/(H - s - d + 1) Args: H: Prediction horizon (total sequence length) d: Inference delay (number of frozen steps, already executed) s: Execution horizon (non-overlapping steps at end). schedule: 'exponential' (paper default), 'linear', 'hard', or 'simple' Returns: mask: [H] tensor with values in [0, 1] Guidance weights for each timestep """ mask = torch.zeros(H, device=device) if schedule == "hard": # Hard mask: 1 for frozen, 0 for rest mask[:d] = 1.0 elif schedule == "linear": # Linear decay mask[:d] = 1.0 # Three-region version (full paper) overlap_end = H - s if d < overlap_end: # Linear decay from 1 to 0 in intermediate region indices = torch.arange(d, overlap_end, device=device).float() mask[d:overlap_end] = 1.0 - (indices - d) / (overlap_end - d) # mask[overlap_end:] remains 0 elif schedule == "exponential": # Exponential decay (paper default, Equation 5) mask[:d] = 1.0 # Three-region version (full paper implementation) overlap_end = H - s if d < overlap_end: indices = torch.arange(d, overlap_end, device=device).float() # c_i = (H - s - i) / (H - s - d + 1) c_i = (overlap_end - indices) / (overlap_end - d + 1) # W_i = c_i * (e^(c_i) - 1) / (e - 1) e = torch.tensor(torch.e, device=device) mask[d:overlap_end] = c_i * (torch.exp(c_i) - 1) / (e - 1) # mask[overlap_end:] remains 0 elif schedule == "simple": # Simplified exponential (for backward compatibility) mask[:d] = 1.0 if d < H: indices = torch.arange(d, H, device=device).float() mask[d:] = torch.exp(-5.0 * (indices - d) / (H - d)) else: raise ValueError(f"Unknown mask schedule: {schedule}") return mask @torch.inference_mode() def chat(self): ...
import os
import json
import copy
import contextlib
from tqdm import tqdm
from typing import List, Optional, Tuple, Union, Dict, Any
from typing import Any, Dict, List, Optional, Tuple, Union
from typing_extensions import Self
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from PIL import Image
from dataclasses import dataclass
from diffusers.utils.outputs import BaseOutput
from diffusers.configuration_utils import ConfigMixin, FrozenDict, register_to_config
# from diffusers.utils.torch_utils import maybe_allow_in_graph
from diffusers.models.modeling_utils import ModelMixin
# from diffusers.models.transformers.transformer_sd3 import SD3Transformer2DModel
from diffusers.models.embeddings import CombinedTimestepTextProjEmbeddings
# from diffusers.models.normalization import AdaLayerNorm, AdaLayerNormZero, AdaLayerNormContinuous
from diffusers.models.embeddings import CombinedTimestepLabelEmbeddings
from diffusers.models.normalization import FP32LayerNorm
from diffusers.models.attention_processor import Attention, JointAttnProcessor2_0
from diffusers.models.attention import FeedForward, _chunked_feed_forward
# from dit_policy.data4robotics.models.diffusion import _TimeNetwork
from psi.config.config import LaunchConfig
from transformers import (
Qwen3VLForConditionalGeneration,
Qwen2_5OmniThinkerForConditionalGeneration,
Qwen2_5OmniProcessor,
AutoConfig,
AutoProcessor,
Qwen2TokenizerFast,
Qwen3VLProcessor,
)
from qwen_vl_utils import process_vision_info
from psi.utils import initialize_overwatch, count_parameters
# from InternVLA.model.modules.action_model.DiT_modules.models import DiT
# from InternVLA.model.modules.projector.QFormer import CrossAttentionBlock
overwatch = initialize_overwatch(__name__)
# from psi.config.model import ModelConfig, InternVLA_M1_ModelConfig
# from psi.learn.vlms import _QWen_VL_Interface
from transformers.models.siglip import SiglipModel
from transformers.models.dinov2 import Dinov2Model
# from vlt.transformers.vla import ActionTransformerModel
from diffusers.utils import logging
logger = logging.get_logger(__name__) # pylint: disable=invalid-name FIXME why this not works any more?
QWEN3VL_VARIANT = "Qwen/Qwen3-VL-2B-Instruct"
OMNI_DEFAULT_SYSTEM_PROMPT = (
"You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
"capable of perceiving auditory and visual inputs, as well as generating text and speech."
)
def _validate_rtc_inference_contract(
*,
action_horizon: int,
action_exec_horizon: int,
inference_delay: int,
max_delay: int,
) -> int:
"""Return the configured RTC overlap or fail on train/deploy mismatch."""
horizon = int(action_horizon)
execute = int(action_exec_horizon)
delay = int(inference_delay)
delay_limit = int(max_delay)
if not 0 < execute < horizon:
raise ValueError(
"RTC requires 0 < action_exec_horizon < action_horizon, "
f"got execute={execute}, horizon={horizon}"
)
expected_overlap = horizon - execute
if delay != expected_overlap:
raise ValueError(
"RTC inference_delay must equal the overlap used by training and "
"deployment: "
f"got inference_delay={delay}, expected={expected_overlap} "
f"(action_horizon={horizon}, action_exec_horizon={execute})"
)
if expected_overlap >= delay_limit:
raise ValueError(
"RTC overlap must be smaller than max_delay, "
f"got overlap={expected_overlap}, max_delay={delay_limit}"
)
return expected_overlap
@dataclass
class HumanFoundationModelOutput(BaseOutput):
action: "torch.Tensor" # noqa: F821
def to_tuple(self):
return (None, self.action)
@dataclass