Download src/psi/models/psi0.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 162 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/models/psi0.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/models/psi0.py
-
curl -L -o psi0.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/models/psi0.py
162 kB
import os
import json
import copy
import contextlib
from tqdm import tqdm
from typing import List, Optional, Tuple, Union, Dict, Any
from typing import Any, Dict, List, Optional, Tuple, Union
from typing_extensions import Self
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from PIL import Image
from dataclasses import dataclass
from diffusers.utils.outputs import BaseOutput
from diffusers.configuration_utils import ConfigMixin, FrozenDict, register_to_config
# from diffusers.utils.torch_utils import maybe_allow_in_graph
from diffusers.models.modeling_utils import ModelMixin
# from diffusers.models.transformers.transformer_sd3 import SD3Transformer2DModel
from diffusers.models.embeddings import CombinedTimestepTextProjEmbeddings
# from diffusers.models.normalization import AdaLayerNorm, AdaLayerNormZero, AdaLayerNormContinuous
from diffusers.models.embeddings import CombinedTimestepLabelEmbeddings
from diffusers.models.normalization import FP32LayerNorm
from diffusers.models.attention_processor import Attention, JointAttnProcessor2_0
from diffusers.models.attention import FeedForward, _chunked_feed_forward
# from dit_policy.data4robotics.models.diffusion import _TimeNetwork
from psi.config.config import LaunchConfig
from transformers import (
Qwen3VLForConditionalGeneration,
Qwen2_5OmniThinkerForConditionalGeneration,
Qwen2_5OmniProcessor,
AutoConfig,
AutoProcessor,
Qwen2TokenizerFast,
Qwen3VLProcessor,
)
from qwen_vl_utils import process_vision_info
from psi.utils import initialize_overwatch, count_parameters
# from InternVLA.model.modules.action_model.DiT_modules.models import DiT
# from InternVLA.model.modules.projector.QFormer import CrossAttentionBlock
overwatch = initialize_overwatch(__name__)
# from psi.config.model import ModelConfig, InternVLA_M1_ModelConfig
# from psi.learn.vlms import _QWen_VL_Interface
from transformers.models.siglip import SiglipModel
from transformers.models.dinov2 import Dinov2Model
# from vlt.transformers.vla import ActionTransformerModel
from diffusers.utils import logging
logger = logging.get_logger(__name__) # pylint: disable=invalid-name FIXME why this not works any more?
QWEN3VL_VARIANT = "Qwen/Qwen3-VL-2B-Instruct"
OMNI_DEFAULT_SYSTEM_PROMPT = (
"You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
"capable of perceiving auditory and visual inputs, as well as generating text and speech."
)
def _validate_rtc_inference_contract(
*,
action_horizon: int,
action_exec_horizon: int,
inference_delay: int,
max_delay: int,
) -> int:
"""Return the configured RTC overlap or fail on train/deploy mismatch."""
horizon = int(action_horizon)
execute = int(action_exec_horizon)
delay = int(inference_delay)
delay_limit = int(max_delay)
if not 0 < execute < horizon:
raise ValueError(
"RTC requires 0 < action_exec_horizon < action_horizon, "
f"got execute={execute}, horizon={horizon}"
)
expected_overlap = horizon - execute
if delay != expected_overlap:
raise ValueError(
"RTC inference_delay must equal the overlap used by training and "
"deployment: "
f"got inference_delay={delay}, expected={expected_overlap} "
f"(action_horizon={horizon}, action_exec_horizon={execute})"
)
if expected_overlap >= delay_limit:
raise ValueError(
"RTC overlap must be smaller than max_delay, "
f"got overlap={expected_overlap}, max_delay={delay_limit}"
)
return expected_overlap
@dataclass
class HumanFoundationModelOutput(BaseOutput):
action: "torch.Tensor" # noqa: F821
def to_tuple(self):
return (None, self.action)
@dataclass
class ActionTransformerModelOutput(BaseOutput):
action: "torch.Tensor" # noqa: F821
def to_tuple(self):
return (None, self.action)
class _TimeNetwork(nn.Module):
def __init__(self, time_dim, out_dim, learnable_w=False):
assert time_dim % 2 == 0, "time_dim must be even!"
half_dim = int(time_dim // 2)
super().__init__()
w = np.log(10000) / (half_dim - 1)
w = torch.exp(torch.arange(half_dim) * -w).float()
# self.register_parameter("w", nn.Parameter(w, requires_grad=learnable_w))
self.w = nn.Parameter(w, requires_grad=learnable_w)
self.out_net = nn.Sequential(
nn.Linear(time_dim, out_dim), nn.SiLU(), nn.Linear(out_dim, out_dim)
)
def forward(self, x):
# assert len(x.shape) == 1, "assumes 1d input timestep array"
# RTC: support 2D timesteps (B,Tp)
x = x[..., None] * self.w
x = torch.cat((torch.cos(x), torch.sin(x)), dim=-1)
return self.out_net(x)
class RMSNorm(nn.Module):
def __init__(self, dim, eps: float, elementwise_affine: bool = True):
super().__init__()
self.eps = eps
if isinstance(dim, int):
dim = (dim,)
self.dim = torch.Size(dim)
if elementwise_affine:
self.weight = nn.Parameter(torch.ones(dim))
else:
self.weight = None
def forward(self, hidden_states):
input_dtype = hidden_states.dtype
variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + self.eps)
if self.weight is not None:
# convert into half-precision if necessary
if self.weight.dtype in [torch.float16, torch.bfloat16]:
hidden_states = hidden_states.to(self.weight.dtype)
hidden_states = hidden_states * self.weight
else:
hidden_states = hidden_states.to(input_dtype)
return hidden_states
if torch.__version__ >= "2.1.0":
LayerNorm = nn.LayerNorm # type:ignore
else:
# Has optional bias parameter compared to torch layer norm
# TODO: replace with torch layernorm once min required torch version >= 2.1
class LayerNorm(nn.Module):
def __init__(self, dim, eps: float = 1e-5, elementwise_affine: bool = True, bias: bool = True):
super().__init__()
self.eps = eps
if isinstance(dim, int):
dim = (dim,)
self.dim = torch.Size(dim)
if elementwise_affine:
self.weight = nn.Parameter(torch.ones(dim))
self.bias = nn.Parameter(torch.zeros(dim)) if bias else None
else:
self.weight = None
self.bias = None
def forward(self, input):
return F.layer_norm(input, self.dim, self.weight, self.bias, self.eps)
class AdaLayerNormContinuous(nn.Module):
def __init__(
self,
embedding_dim: int,
conditioning_embedding_dim: int,
# NOTE: It is a bit weird that the norm layer can be configured to have scale and shift parameters
# because the output is immediately scaled and shifted by the projected conditioning embeddings.
# Note that AdaLayerNorm does not let the norm layer have scale and shift parameters.
# However, this is how it was implemented in the original code, and it's rather likely you should
# set `elementwise_affine` to False.
elementwise_affine=True,
eps=1e-5,
bias=True,
norm_type="layer_norm",
):
super().__init__()
self.silu = nn.SiLU()
self.linear = nn.Linear(conditioning_embedding_dim, embedding_dim * 2, bias=bias)
if norm_type == "layer_norm":
self.norm = LayerNorm(embedding_dim, eps, elementwise_affine, bias)
elif norm_type == "rms_norm":
self.norm = RMSNorm(embedding_dim, eps, elementwise_affine)
else:
raise ValueError(f"unknown norm_type {norm_type}")
def forward(self, x: torch.Tensor, conditioning_embedding: torch.Tensor) -> torch.Tensor:
# convert back to the original dtype in case `conditioning_embedding`` is upcasted to float32 (needed for hunyuanDiT)
if len(conditioning_embedding.shape) == 2: # (B, D)
conditioning_embedding = conditioning_embedding.unsqueeze(1) # (B, 1, D)
elif len(conditioning_embedding.shape) == 3: # (B, T, D)
pass
else:
raise ValueError(f"Invalid shape of conditioning_embedding: {conditioning_embedding.shape}")
emb = self.linear(self.silu(conditioning_embedding).to(x.dtype))
scale, shift = torch.chunk(emb, 2, dim=-1)
x = self.norm(x) * (1 + scale) + shift
return x
class AdaLayerNormZero(nn.Module):
r"""
Norm layer adaptive layer norm zero (adaLN-Zero).
Parameters:
embedding_dim (`int`): The size of each embedding vector.
num_embeddings (`int`): The size of the embeddings dictionary.
"""
def __init__(self, embedding_dim: int, num_embeddings: Optional[int] = None, norm_type="layer_norm", bias=True):
super().__init__()
if num_embeddings is not None:
self.emb = CombinedTimestepLabelEmbeddings(num_embeddings, embedding_dim)
else:
self.emb = None
self.silu = nn.SiLU()
self.linear = nn.Linear(embedding_dim, 6 * embedding_dim, bias=bias)
# This module is adaLN-Zero, so every residual modulation must start as
# an exact no-op. The previous implementation used PyTorch's default
# random Linear initialization despite the class name, opening all
# attention/MLP residual gates from the first optimizer step.
nn.init.zeros_(self.linear.weight)
if self.linear.bias is not None:
nn.init.zeros_(self.linear.bias)
if norm_type == "layer_norm":
self.norm = nn.LayerNorm(embedding_dim, elementwise_affine=False, eps=1e-6)
elif norm_type == "fp32_layer_norm":
self.norm = FP32LayerNorm(embedding_dim, elementwise_affine=False, bias=False)
else:
raise ValueError(
f"Unsupported `norm_type` ({norm_type}) provided. Supported ones are: 'layer_norm', 'fp32_layer_norm'."
)
def forward(
self,
x: torch.Tensor,
timestep: Optional[torch.Tensor] = None,
class_labels: Optional[torch.LongTensor] = None,
hidden_dtype: Optional[torch.dtype] = None,
emb: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
if self.emb is not None:
emb = self.emb(timestep, class_labels, hidden_dtype=hidden_dtype)
assert False, "Not implemented with emb shape == 3, (B, T, D)"
if len(emb.shape) == 2: # (B, D)
emb = emb.unsqueeze(1) # (B, 1, D)
elif len(emb.shape) == 3: # (B, T, D)
pass
else:
raise ValueError(f"Invalid shape of emb: {emb.shape}")
emb = self.linear(self.silu(emb))
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = emb.chunk(6, dim=-1)
x = self.norm(x) * (1 + scale_msa) + shift_msa
return x, gate_msa, shift_mlp, scale_mlp, gate_mlp
class _PositionalEncoding(nn.Module):
def __init__(self, d_model, max_len=5000):
super().__init__()
# Compute the positional encodings once in log space
pe = torch.zeros(max_len, d_model)
position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
div_term = torch.exp(
torch.arange(0, d_model, 2, dtype=torch.float)
* -(np.log(10000.0) / d_model)
)
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
pe = pe.unsqueeze(0).transpose(0, 1)
# self.register_buffer("pe", torch.clone(pe)) # <-- SONGLIN: this will cause problems when accelerate.prepare
# self.register_parameter("pe", nn.Parameter(pe, requires_grad=False))
self.pe = nn.Parameter(pe, requires_grad=False)
def forward(self, x, valid_mask=None):
"""
Args:
x: Tensor of shape (seq_len, batch_size, d_model)
Returns:
Tensor of shape (seq_len, batch_size, d_model) with positional encodings added
"""
if valid_mask is not None:
if tuple(valid_mask.shape) != (x.shape[1], x.shape[0]):
raise ValueError("Position mask shape must be (batch, sequence)")
# Appended control/state tokens follow each source's valid AV
# tokens, not a neighbouring source's padding. No GPU scalar read.
positions = (valid_mask.to(dtype=torch.long).cumsum(-1) - 1).clamp_min(0)
return self.pe[:, 0][positions].transpose(0, 1).detach()
pe = self.pe[: x.shape[0]]
pe = pe.repeat((1, x.shape[1], 1))
return pe.detach().clone()
class FilmConditioning(nn.Module):
"""Layer that adds FiLM conditioning.
This is intended to be applied after a convolutional layer. It will learn a
multiplicative and an additive factor to be applied to each channel of the
convolution's output.
Conv layer can be rank 2 or 4.
For further details, see: https://arxiv.org/abs/1709.07871
"""
def __init__(self,
in_dim: int,
num_channels: int):
"""Constructs a FiLM conditioning layer.
Args:
num_channels: Number of filter channels to expect in the input.
"""
super(FilmConditioning, self).__init__()
# Note that we initialize with zeros because empirically we have found
# this works better than initializing with glorot.
self._projection_add = nn.Linear(in_dim, num_channels)
self._projection_mult = nn.Linear(in_dim, num_channels)
for m in self.modules():
if isinstance(m, nn.Linear):
nn.init.zeros_(m.weight)
nn.init.zeros_(m.bias)
def forward(self, conv_filters, conditioning):
assert len(conditioning.shape) == 2
projected_cond_add = self._projection_add(conditioning)
projected_cond_mult = self._projection_mult(conditioning)
if len(conv_filters.shape) == 4:
# [B, D] -> [B, D, 1, 1]
projected_cond_add = projected_cond_add.unsqueeze(-1).unsqueeze(-1)
projected_cond_mult = projected_cond_mult.unsqueeze(-1).unsqueeze(-1)
elif len(conv_filters.shape) == 3:
# [B, D] -> [B, 1, D]
projected_cond_add = projected_cond_add.unsqueeze(1)
projected_cond_mult = projected_cond_mult.unsqueeze(1)
else:
assert len(conv_filters.shape) == 2
# Original FiLM paper argues that 1 + gamma centers the initialization at
# identity transform.
result = (1 + projected_cond_mult) * conv_filters + projected_cond_add
return result
class JointVLAAttnProcessor: #(nn.Module):
"""Attention processor used typically in processing the SD3-like self-attention projections."""
def __init__(
self,
query_dim: int,
heads: int = 8,
kv_heads: Optional[int] = None,
dim_head: int = 64,
out_dim: Optional[int] = None,
dropout: float = 0.0,
bias: bool = False,
out_bias: bool = True,
context_pre_only=None,
):
super().__init__()
if not hasattr(F, "scaled_dot_product_attention"):
raise ImportError("JointAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")
# self.inner_dim = out_dim if out_dim is not None else dim_head * heads
# self.inner_kv_dim = self.inner_dim if kv_heads is None else dim_head * kv_heads
# self.out_dim = out_dim if out_dim is not None else query_dim
# self.to_q_lang = nn.Linear(query_dim, self.inner_dim, bias=bias)
# self.to_k_lang = nn.Linear(query_dim, self.inner_kv_dim, bias=bias)
# self.to_v_lang = nn.Linear(query_dim, self.inner_kv_dim, bias=bias)
def __call__(
self,
attn: Attention,
hidden_states: torch.Tensor,
encoder_hidden_states: Optional[torch.Tensor] = None,
attention_mask: Optional[torch.Tensor] = None,
*args,
**kwargs,
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
residual = hidden_states
batch_size = hidden_states.shape[0]
# `sample` projections.
query = attn.to_q(hidden_states)
assert attn.to_k is not None
key = attn.to_k(hidden_states)
assert attn.to_v is not None
value = attn.to_v(hidden_states)
inner_dim = key.shape[-1]
head_dim = inner_dim // attn.heads
query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
if attn.norm_q is not None:
query = attn.norm_q(query)
if attn.norm_k is not None:
key = attn.norm_k(key)
# `context` projections.
if encoder_hidden_states is not None:
encoder_hidden_states_query_proj = attn.add_q_proj(encoder_hidden_states)
encoder_hidden_states_key_proj = attn.add_k_proj(encoder_hidden_states)
encoder_hidden_states_value_proj = attn.add_v_proj(encoder_hidden_states)
encoder_hidden_states_query_proj = encoder_hidden_states_query_proj.view(
batch_size, -1, attn.heads, head_dim
).transpose(1, 2)
encoder_hidden_states_key_proj = encoder_hidden_states_key_proj.view(
batch_size, -1, attn.heads, head_dim
).transpose(1, 2)
encoder_hidden_states_value_proj = encoder_hidden_states_value_proj.view(
batch_size, -1, attn.heads, head_dim
).transpose(1, 2)
if attn.norm_added_q is not None:
encoder_hidden_states_query_proj = attn.norm_added_q(encoder_hidden_states_query_proj)
if attn.norm_added_k is not None:
encoder_hidden_states_key_proj = attn.norm_added_k(encoder_hidden_states_key_proj)
query = torch.cat([query, encoder_hidden_states_query_proj], dim=2)
key = torch.cat([key, encoder_hidden_states_key_proj], dim=2)
value = torch.cat([value, encoder_hidden_states_value_proj], dim=2)
if attention_mask is not None:
assert attention_mask.dtype == torch.float32
attn_mask = torch.cat([
torch.ones(hidden_states.shape[0], 1, 1, hidden_states.shape[1], device=attention_mask.device).to(torch.bool), # (B, 1, 1, S1)
(attention_mask == 1)[:, None, None, :] # (B, 1, 1, S2)
], dim=-1)
else:
attn_mask = None
hidden_states = F.scaled_dot_product_attention(query, key, value, dropout_p=0.0, is_causal=False, attn_mask=attn_mask) # type: ignore
hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) # type: ignore
hidden_states = hidden_states.to(query.dtype) # type: ignore
# Split the attention outputs.
hidden_states, encoder_hidden_states = (
hidden_states[:, : residual.shape[1]],
hidden_states[:, residual.shape[1] :],
) # type: ignore
if not attn.context_pre_only:
encoder_hidden_states = attn.to_add_out(encoder_hidden_states)
# linear proj
hidden_states = attn.to_out[0](hidden_states)
# dropout
hidden_states = attn.to_out[1](hidden_states)
return hidden_states, encoder_hidden_states
class ObservationProjection(nn.Module): # FIXME naming
""" VLT observation projector
Args:
...
"""
def __init__(
self,
action_pred_horizon: int = 50, # Tp
action_dim: int = 7, # in dim
output_dim: int = 512, # out dim,
hidden_dim: int = 1280,
n_conditions: int = 1, # num image of contitions
timestep_in_dim: int = 320,
timestep_flip_sin_to_cos: bool = True,
timestep_freq_shift: int = 0,
token_fusion: str = "concat",
resnet_store_path: str = "cache/visual_features/resnet18/IN_1M_resnet18.pth",
odim: int = 32,
view_feature_dim: int = 1920, # VLM输出hidden_dim维度
use_film: bool = False
):
super().__init__()
# TODO 1. compare
# self.time_net = _TimeNetwork(time_dim=256, out_dim=512)
# self.time_proj = Timesteps(timestep_in_dim, timestep_flip_sin_to_cos, timestep_freq_shift)
# self.time_embedding = TimestepEmbedding(timestep_in_dim, output_dim, act_fn="silu")
self.action_pred_horizon = action_pred_horizon
self.action_dim = action_dim
# self.ac_proj = nn.Sequential(
# nn.Linear(action_dim, action_dim),
# nn.GELU(approximate="tanh"),
# nn.Linear(action_dim, output_dim),
# )
# _DiTNoiseNet.__init__
self.enc_pos = _PositionalEncoding(d_model=output_dim)
# Opt in through the saved model config; old checkpoints keep their
# original padding-sensitive positions for reproducible inference.
self.mask_aware_positions = False
# agent.__init__
n_cams = 0 # 1
imgs_per_cam = 1
self.camera_indices = [] # [0]
self._n_cams = n_cams
self._share_cam_features = False # model_cfg.share_cam_features
early_fusion = False
text_dim = 768
# odim = 15
self.token_fusion = token_fusion
if self._share_cam_features:
...
else:
if n_conditions > 0:
from dit_policy.data4robotics.models.resnet import ResNet
features = ResNet(
size=18,
weights="IMAGENET1K_V1",
restore_path=resnet_store_path,
avg_pool=False,
conv_repeat=0,
norm_cfg={
"name": "group_norm",
"num_groups": 16,
}
)
else:
features = None
feat_list = [copy.deepcopy(features) for _ in range(0, n_conditions)] # + [copy.deepcopy(features) for _ in range(1, n_cams)] + [copy.deepcopy(features) for _ in range(0, n_conditions)]
self.visual_features = nn.ModuleList(feat_list)
self.views_proj = nn.Linear(view_feature_dim, output_dim, bias=True) # TODO configure 1920 # boqian_fix
# TODO abl: more layers # view proj
# self.fc1 = nn.Linear(self.vision_dim, initial_projection_dim, bias=True)
# self.fc2 = nn.Linear(initial_projection_dim, self.llm_dim, bias=True)
# self.fc3 = nn.Linear(self.llm_dim, self.llm_dim, bias=True)
# self.act_fn1 = nn.GELU()
# self.act_fn2 = nn.GELU()
# self.views_proj = nn.Sequential(
# nn.Conv2d(16, output_dim, kernel_size=(2, 2), stride=2, bias=True),
# nn.ReLU(),
# nn.Conv2d(output_dim, output_dim, kernel_size=(2, 2), stride=2, bias=True)
# )
# self.traj2d_proj = nn.ModuleList([
# nn.Conv2d(16, 512, kernel_size=(2, 2), stride=2, bias=True),
# nn.ReLU(),
# nn.Conv2d(512, output_dim, kernel_size=(2, 2), stride=2, bias=True),
# nn.ReLU(),
# nn.Linear(output_dim, output_dim, bias=True),
# nn.LayerNorm(output_dim)
# ])
self.early_fusion = early_fusion
self.imgs_per_cam = imgs_per_cam
imgs_per_cam = 1 if early_fusion else imgs_per_cam
self._n_conditions = n_conditions
self.embed_proj = nn.Identity()
if self.token_fusion != "perceiver" and self._n_conditions > 0:
assert features is not None
self._token_dim = features.embed_dim
if output_dim != self._token_dim:
self.embed_proj = nn.Linear(self._token_dim, output_dim, bias=False)
else:
self._token_dim = 0
self._n_tokens = imgs_per_cam * n_cams * self._token_dim + n_conditions * self._token_dim
self.traj_n_token = self._token_dim
self.feat_norm = nn.LayerNorm(output_dim, elementwise_affine=False, eps=1e-6) # feat_norm = None
self.feat_traj_norm = nn.LayerNorm(output_dim, elementwise_affine=False, eps=1e-6) # feat_norm = None
# self.traj2d_proj = nn.ModuleList([
# nn.Linear(output_dim, output_dim) for _ in range(2)
# ])
# self.traj2d_proj = nn.Conv2d(
# 16, output_dim, kernel_size=(2, 2), stride=2, bias=True
# )
if self.token_fusion == "cross":
self.attn = Attention(
query_dim=self._token_dim,
cross_attention_dim=1920,
dim_head=64,
out_dim=output_dim,
heads=output_dim//64,
bias=True,
# pre_only=True,
eps=1e-6
)
elif self.token_fusion == "perceiver":
num_queries = 256
self.latents1 = nn.Parameter(torch.randn(1, num_queries, hidden_dim) / hidden_dim**0.5)
# self.proj_in1 = nn.Linear(1920, hidden_dim) -> views_proj (see above)
self.cross_attn1 = Attention(
query_dim=hidden_dim,
cross_attention_dim=hidden_dim,
dim_head=64,
out_dim=output_dim,
heads=output_dim//64,
bias=True,
# pre_only=True,
eps=1e-6
)
self.latents2 = nn.Parameter(torch.randn(1, num_queries, hidden_dim) / hidden_dim**0.5)
self.proj_in2 = nn.Linear(self._token_dim, hidden_dim)
self.cross_attn2 = Attention(
query_dim=hidden_dim,
cross_attention_dim=hidden_dim,
dim_head=64,
out_dim=output_dim,
heads=output_dim//64,
bias=True,
# pre_only=True,
eps=1e-6
)
self.film = FilmConditioning(2048, output_dim) if use_film else None
self._obs_strat = "add_token"
self._n_tokens += 1
self._obs_proc = nn.Sequential(
nn.Dropout(p=0.2), nn.Linear(odim, output_dim)
)
dropout = 0.1 # @see model_cfg.dropout
linear_proj = nn.Identity() # build (optional) token feature projection layer
norm = nn.Identity() # feat_norm = None
self.post_proc = nn.Sequential(linear_proj, norm, nn.Dropout(dropout))
def forward(self,
# noisy_actions: torch.Tensor,
# timestep: torch.Tensor,
# temp
views,
obs,
traj2ds=None,
# ac_flat = None,
# mask_flat = None,
text_embeddings = None,
vlm_attn_mask=None
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Forward pass.
Args:
noisy_actions (`torch.Tensor`):
Image features.
timestep (`torch.Tensor`):
Timestep in denoising process.
Returns:
`Tuple`[`torch.Tensor`, `torch.Tensor`]: The pair (latents, timestep_emb).
"""
# DiffusionTransformerAgent.forward
s_t, vlm_attn_mask = self.tokenize_obs(views, obs, traj2ds, text_embeddings=text_embeddings, vlm_attn_mask=vlm_attn_mask) # (B, S, d_model)
# _DiTNoiseNet.forward_enc
s_t = s_t.transpose(0, 1) # (S, B, d_model)
pos = self.enc_pos(
s_t, vlm_attn_mask if self.mask_aware_positions else None
) # (S, B, d_model)
return (s_t + pos).transpose(0,1), vlm_attn_mask # (B,S,d_model)
def tokenize_obs(self, views, obs, traj2ds = None, flatten=False, text_embeddings=None, vlm_attn_mask=None):
view_tokens = self.views_proj(views)
# V:views,相机、视角数量
# S:每个view的token数(VLM长度)
# D:token维度
B,V,S,D = view_tokens.shape
if traj2ds is not None:
traj_tokens = self.embed({}, {f"cond{i}": traj2ds[:, i] for i in range(traj2ds.shape[1])})
if self.token_fusion == "concat":
traj_tokens = self.embed_proj(traj_tokens) #(B,S,d_model)
tokens = torch.cat([view_tokens.view(B,V*S,D), traj_tokens], dim=1) # (B, tok1+tok2, 1536)
elif self.token_fusion == "cross":
traj_tokens_list, view_tokens_list = [], []
for v_idx in range(traj2ds.shape[1]):
# traj2d --> cross attend --> view features
view_tokens = self.attn(
hidden_states=traj_tokens[:, v_idx*self.traj_n_token, (v_idx+1)*self.traj_n_token],
encoder_hidden_states=views[:, v_idx],
)
traj_tokens_list.append(traj_tokens)
view_tokens_list.append(view_tokens)
traj_tokens = torch.cat(traj_tokens_list, dim=1)
traj_tokens = self.embed_proj(traj_tokens)
view_tokens = torch.cat(view_tokens_list, dim=1)
tokens = torch.cat([view_tokens, traj_tokens], dim=1) # (B, tok1+tok2, 1536)
elif self.token_fusion == "perceiver":
latents1 = self.latents1.repeat(B, 1, 1)
# views = self.proj_in1(views)
# views = views + text_embeddings[:, None]
tokens1 = self.cross_attn1(
hidden_states=latents1,
encoder_hidden_states=view_tokens.view(B, V*S, D),
)
latents2 = self.latents2.repeat(B, 1, 1)
traj_tokens = self.proj_in2(traj_tokens)
# traj2ds = traj2ds + text_embeddings[:, None]
tokens2 = self.cross_attn2(
hidden_states=latents2,
encoder_hidden_states=traj_tokens,
)
tokens = torch.cat([tokens1, tokens2], dim=1) # (B, tok1+tok2, 1536)
else:
raise ValueError
else:
assert self._n_conditions == 0, "inconsistent confg"
tokens = view_tokens.view(B,V*S,D)
# 用FiLM把text_embeddings融进token
if (self.film is not None) and (text_embeddings is not None):
tokens = self.film(tokens, text_embeddings)
if self._obs_strat == "add_token":
# <modify_states> 多帧state时,每一帧都作为一个proprio token参与注意力。
obs_token = self._obs_proc(obs)#[:, None]
tokens = torch.cat((tokens, obs_token), 1)
# 原逻辑:
# vlm_attn_mask = torch.cat([vlm_attn_mask, torch.ones((vlm_attn_mask.shape[0], 1), device=vlm_attn_mask.device)], 1) if vlm_attn_mask is not None else None
if vlm_attn_mask is not None:
obs_token_count = obs_token.shape[1] if len(obs_token.shape) == 3 else 1
# <modify_states> 保持与原始训练代码一致:新增obs token mask继续使用默认float32,
# 这样拼接后整体attention mask仍会提升成float32,满足后续断言。
obs_token_mask = torch.ones(
(vlm_attn_mask.shape[0], obs_token_count),
device=vlm_attn_mask.device,
)
vlm_attn_mask = torch.cat([vlm_attn_mask, obs_token_mask], 1)
elif self._obs_strat == "pad_img_tokens":
obs = self._obs_proc(obs)
obs = obs[:, None].repeat((1, tokens.shape[1], 1))
tokens = torch.cat((obs, tokens), 2)
vlm_attn_mask = None
else:
assert self._obs_strat is None
tokens = self.post_proc(tokens)
if flatten:
return tokens.reshape((tokens.shape[0], -1)), vlm_attn_mask
return tokens, vlm_attn_mask
def embed(self, imgs, conditions=None):
""" SONGLIN: DOWNSCALE == 2**5 = 32, total 5 conv2d with stride(2) """
def embed_helper(net, im):
# with torch.autocast(device_type='cuda', dtype=torch.bfloat16): # TODO check if this is needed
if self.early_fusion and len(im.shape) == 5:
T = im.shape[1]
im = torch.cat([im[:, t] for t in range(T)], 1)
embeds = net(im)
elif len(im.shape) == 5:
B, T, C, H, W = im.shape
embeds = net(im.reshape((B * T, C, H, W)))
embeds = embeds.reshape((B, -1, net.embed_dim))
else:
assert len(im.shape) == 4
embeds = net(im)
return embeds
if self._share_cam_features:
embeds = [
embed_helper(self.visual_features, imgs[f"cam{i}"])
for i in self.camera_indices
]
if conditions is not None:
assert self._n_conditions > 0
embeds += [
embed_helper(self.visual_features, conditions[f"cond{i}"])
for i in range(self._n_conditions)
]
else:
assert conditions is not None
# img_encoders = self.visual_features[:self._n_cams] # sum([v.sum().item() for k, v in img_encoders.named_parameters()])
# cond_encoders = self.visual_features[self._n_cams:]
cond_encoders = self.visual_features
# [
# embed_helper(net, imgs[f"cam{i}"])
# for i, net in zip(self.camera_indices, img_encoders)
# ] +
embeds = [
embed_helper(net, conditions[f"cond{0}"])
for i, net in enumerate(cond_encoders)
]
""" if self.training: # FIXME better handle this thing
return torch.cat(embeds, dim=1) """
assert conditions is not None
in_dtype = next(iter(imgs.values())).dtype if len(imgs) > 0 else next(iter(conditions.values())).dtype
embeds = torch.cat(embeds, dim=1).to(dtype=in_dtype) # (B, S, d_model)
return embeds
class ActionProjectionIn(nn.Module):
def __init__(
self,
action_pred_horizon,
action_dim,
output_dim,
):
super().__init__()
self.action_pred_horizon = action_pred_horizon
self.action_dim = action_dim
# Opt-in numerical policy. Keep legacy checkpoints bitwise unchanged.
self.force_fp32 = False
self.ac_proj = nn.Sequential(
nn.Linear(action_dim, action_dim),
nn.GELU(approximate="tanh"),
nn.Linear(action_dim, output_dim),
)
# self.time_net = _TimeNetwork(time_dim=256, out_dim=output_dim)
# self.register_parameter(
# "dec_pos",
# nn.Parameter(torch.empty(action_pred_horizon, output_dim), requires_grad=True),
# ) # (1, Tp, hidden_dim)
self.dec_pos = nn.Parameter(torch.empty(action_pred_horizon, output_dim), requires_grad=True)
nn.init.xavier_uniform_(self.dec_pos.data)
def forward(
self,
noisy_actions: torch.Tensor,
position_offset: int = 0,
) -> torch.Tensor:
B = noisy_actions.shape[0]
# action_dim = self._dit_obs_proj.action_dim
# time_enc = self.time_net(timestep)
noise_acs = noisy_actions.reshape(B, -1, self.action_dim) # DiffusionTransformerAgent.forward
if self.force_fp32:
if any(p.dtype != torch.float32 for p in self.ac_proj.parameters()):
raise RuntimeError("FP32 action input projection requires FP32 master weights")
# BF16 quantization here can erase small inter-frame angle changes
# before denoising. Only this small input MLP leaves mixed precision;
# attention, output head and the rest of the model are unchanged.
with torch.autocast(device_type=noise_acs.device.type, enabled=False):
ac_tokens = self.ac_proj(noise_acs.float())
else:
ac_tokens = self.ac_proj(noise_acs)
# ac_tokens = ac_tokens.transpose(0, 1) # (B, L, d_model) _DiTNoiseNet.forward_dec
token_count = ac_tokens.shape[1]
position_end = int(position_offset) + token_count
if position_offset < 0 or position_end > self.action_pred_horizon:
raise ValueError(
"Action token positions exceed the configured horizon: "
f"offset={position_offset}, tokens={token_count}, "
f"horizon={self.action_pred_horizon}"
)
action_hidden_states = (
ac_tokens
+ self.dec_pos[int(position_offset):position_end].unsqueeze(0)
)
return action_hidden_states#, time_enc
class ActionProjectionOut(nn.Module): # _FinalLayer
def __init__(self, hidden_size, action_dim):
super().__init__()
self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6)
self.linear = nn.Linear(hidden_size, action_dim, bias=True)
self.adaLN_modulation = nn.Sequential(
nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True)
)
self.reset_parameters()
def reset_parameters(self):
# Standard DiT final-layer initialization: start with zero velocity
# prediction and let the output head open gradually. This keeps the
# first updates from backpropagating an untrained random output through
# the complete transformer stack.
modulation = self.adaLN_modulation[-1]
assert isinstance(modulation, nn.Linear)
nn.init.zeros_(modulation.weight)
nn.init.zeros_(modulation.bias)
nn.init.zeros_(self.linear.weight)
nn.init.zeros_(self.linear.bias)
def forward(self, x, t, cond=None):
# process the conditioning vector first
# cond = torch.mean(cond, dim=1)
# cond = cond + t
if len(t.shape) == 2: # (B, D)
t = t.unsqueeze(1) # (B, 1, D)
elif len(t.shape) == 3: # (B, T, D)
pass
else:
raise ValueError(f"Invalid shape of t: {t.shape}")
shift, scale = self.adaLN_modulation(t).chunk(2, dim=-1) # cond -> t
# x = self.norm_final(x) * (1+scale[None]) + shift[None] # TODO be careful!
# x = x * scale + shift
x = self.norm_final(x) * (1 + scale) + shift
x = self.linear(x)
# return x.transpose(0, 1)
return x
# @maybe_allow_in_graph
class VLATransformerBlock(nn.Module):
def __init__(
self,
dim: int,
num_attention_heads: int,
attention_head_dim: int,
context_pre_only: bool = False,
qk_norm: Optional[str] = "fp32_layer_norm",
use_dual_attention: bool = False,
training_phase: str = "joint", # "joint", "action", "traj2d", "action_mini"
):
super().__init__()
# self.training_phase = training_phase
# from models.dit_policy.data4robotics.models.diffusion import _SelfAttnEncoder #_TransformerEncoder,
# self.encoder_module = _SelfAttnEncoder(
# d_model=dim,
# nhead=num_attention_heads,
# dim_feedforward=2048,
# dropout=0.1,
# activation="gelu",
# )
# self.encoder_module.reset_parameters()
# # decoder blocks
# from models.dit_policy.data4robotics.models.diffusion import _DiTDecoder #_TransformerDecoder,
# self.decoder_module = _DiTDecoder(
# d_model=dim,
# nhead=num_attention_heads,
# dim_feedforward=2048,
# dropout=0.1,
# activation="gelu",
# )
# self.decoder_module.reset_parameters() # _TransformerDecoder.__init__
self.context_pre_only = context_pre_only
context_norm_type = "ada_norm_continous" if context_pre_only else "ada_norm_zero"
# self.norm1 = AdaLayerNormZero(dim)
self.norm1_act = AdaLayerNormZero(dim)
logger.debug(f"AdaLN act parameters: {count_parameters(self.norm1_act):,}")
if context_norm_type == "ada_norm_continous":
""" self.norm1_lang = AdaLayerNormContinuous(dim, dim, elementwise_affine=False, eps=1e-6, bias=True, norm_type="layer_norm") """
self.norm1_obs = AdaLayerNormContinuous(dim, dim, elementwise_affine=False, eps=1e-6, bias=True, norm_type="layer_norm")
logger.debug(f"AdaLN obs parameters: {count_parameters(self.norm1_obs):,}")
elif context_norm_type == "ada_norm_zero":
""" self.norm1_lang = AdaLayerNormZero(dim) """
self.norm1_obs = AdaLayerNormZero(dim)
logger.debug(f"AdaLN obs parameters: {count_parameters(self.norm1_obs):,}")
else:
raise ValueError(
f"Unknown context_norm_type: {context_norm_type}, currently only support `ada_norm_continous`, `ada_norm_zero`"
)
if hasattr(F, "scaled_dot_product_attention"):
processor = JointVLAAttnProcessor(
query_dim=dim,
dim_head=attention_head_dim,
heads=num_attention_heads,
out_dim=dim,
bias=True,
context_pre_only=context_pre_only,
)
else:
raise ValueError(
"The current PyTorch version does not support the `scaled_dot_product_attention` function."
)
self.attn = Attention(
query_dim=dim,
cross_attention_dim=None,
added_kv_proj_dim=dim,
dim_head=attention_head_dim,
heads=num_attention_heads,
out_dim=dim,
context_pre_only=context_pre_only,
bias=True,
processor=processor, # type: ignore
qk_norm=qk_norm,
eps=1e-6,
)
logger.debug(f"Joint Attention parameters: {count_parameters(self.attn):,}")
self.norm2_act = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
self.ff_act = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate")
logger.debug(f"FF act parameters: {count_parameters(self.ff_act):,}")
if not context_pre_only:
""" self.norm2_lang = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
self.ff_lang = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate") """
self.norm2_obs = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
self.ff_obs = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate")
logger.debug(f"FF obs parameters: {count_parameters(self.ff_obs):,}")
else:
""" self.norm2_lang = None
self.ff_lang = None """
self.norm2_obs= None
self.ff_obs = None
# let chunk size default to None
self._chunk_size = None
self._chunk_dim = 0
def forward(
self,
# hidden_states: torch.FloatTensor, # V
# lang_hidden_states: torch.FloatTensor, # L
action_hidden_states: torch.Tensor, # A
obs_hidden_states: torch.Tensor, # O
temb: torch.Tensor,
obs_token_mask: Optional[torch.Tensor] = None,
# obs_pos_emb: Optional[torch.FloatTensor] = None,
# time_enc: Optional[torch.FloatTensor] = None,
joint_attention_kwargs=None,
):
# obs_hidden_states = self.encoder_module(src=obs_hidden_states, pos=obs_pos_emb)
# action_hidden_states = self.decoder_module(x=action_hidden_states, t=time_enc, cond=obs_hidden_states) # recursive: _TransformerDecoder.forward
norm_action_hidden_states, gate_msa_act, shift_mlp_act, scale_mlp_act, gate_mlp_act = self.norm1_act(action_hidden_states, emb=temb)
if self.context_pre_only:
""" norm_lang_hidden_states = self.norm1_lang(lang_hidden_states, temb) """
norm_obs_hidden_states = self.norm1_obs(obs_hidden_states, temb[:,-1] if len(temb.shape) > 2 else temb)
gate_msa_obs, shift_mlp_obs, scale_mlp_obs, gate_mlp_obs = None, None, None, None
else:
""" norm_lang_hidden_states, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.norm1_lang(
lang_hidden_states, emb=temb
) """
norm_obs_hidden_states, gate_msa_obs, shift_mlp_obs, scale_mlp_obs, gate_mlp_obs = self.norm1_obs(obs_hidden_states, emb=temb[:,-1] if len(temb.shape) > 2 else temb)
act_attn_output, obs_attn_output = self.attn( ## indirectly calls JointAttnProcessor2_0.__call__, VLAAttnProcessor.__call__
# hidden_states=None,
# lang_hidden_states=norm_lang_hidden_states,
hidden_states=norm_action_hidden_states,
encoder_hidden_states=norm_obs_hidden_states,
attention_mask=obs_token_mask
# **joint_attention_kwargs,
)
# Action feed forward
act_attn_output = gate_msa_act * act_attn_output
action_hidden_states = action_hidden_states + act_attn_output
norm_action_hidden_states = self.norm2_act(action_hidden_states)
norm_action_hidden_states = norm_action_hidden_states * (1 + scale_mlp_act) + shift_mlp_act
ff_action_output = self.ff_act(norm_action_hidden_states)
ff_action_output = gate_mlp_act * ff_action_output
action_hidden_states = action_hidden_states + ff_action_output
# attention outputs for the `obs_hidden_states`.
if self.context_pre_only:
# encoder_hidden_states = None
obs_hidden_states = None # type:ignore
else:
assert gate_msa_obs is not None and \
shift_mlp_obs is not None and \
scale_mlp_obs is not None and \
gate_mlp_obs is not None
""" # lang forward
lang_attn_output = c_gate_msa.unsqueeze(1) * lang_attn_output
lang_hidden_states = lang_hidden_states + lang_attn_output
norm_lang_hidden_states = self.norm2_lang(lang_hidden_states)
norm_lang_hidden_states = norm_lang_hidden_states * (1 + c_scale_mlp[:, None]) + c_shift_mlp[:, None]
if self._chunk_size is not None:
# "feed_forward_chunk_size" can be used to save memory
context_ff_output = _chunked_feed_forward(
self.ff_lang, norm_lang_hidden_states, self._chunk_dim, self._chunk_size
)
else:
context_ff_output = self.ff_lang(norm_lang_hidden_states)
lang_hidden_states = lang_hidden_states + c_gate_mlp.unsqueeze(1) * context_ff_output """
# obs feedforward
assert self.norm2_obs is not None
obs_attn_output = gate_msa_obs * obs_attn_output
obs_hidden_states = obs_hidden_states + obs_attn_output
norm_obs_hidden_states = self.norm2_obs(obs_hidden_states)
norm_obs_hidden_states = norm_obs_hidden_states * (1 + scale_mlp_obs) + shift_mlp_obs
assert self.ff_obs is not None
ff_obs_output = self.ff_obs(norm_obs_hidden_states)
ff_obs_output = gate_mlp_obs * ff_obs_output
obs_hidden_states = obs_hidden_states + ff_obs_output
return action_hidden_states, obs_hidden_states
class ActionTransformerModel(
ModelMixin, ConfigMixin #, VLTTransformerActionMixin
# PeftAdapterMixin, FromOriginalModelMixin, ReprodMixin, SD3Transformer2DLoadersMixin,
):
""" An action transformer model """
_supports_gradient_checkpointing = True
_no_split_modules = ["ActionTransformerBlock"]
# _skip_layerwise_casting_patterns = ["pos_embed", "norm"]
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs):
# model = cls.from_config(config=None, **kwargs)
model = ActionTransformerModel()
torch_dtype = kwargs.pop("torch_dtype", torch.float32)
if pretrained_model_name_or_path is not None and "checkpoints" in str(pretrained_model_name_or_path):
if os.path.exists(pretrained_model_name_or_path):
# NOTE use accelerator.checkpointing
from safetensors.torch import load_file
file_path = f"{pretrained_model_name_or_path}/model.safetensors"
assert os.path.exists(file_path), f"Checkpoint {file_path} does not exist."
model_state_dict = load_file(file_path)
missing_keys, unexpected_keys = model.load_state_dict(model_state_dict, strict=False)
""" # load all checkpoint data, including weights, optimizer, etc.
load_dict = torch.load(pretrained_model_name_or_path, weights_only=False)
from torch.nn.parallel import DistributedDataParallel as DDP
model = model.module if isinstance(model, DDP) else model
missing_keys, unexpected_keys = model.load_state_dict(load_dict["model"], strict=False) """
if len(unexpected_keys) > 0:
logger.info(f"Unexpected keys in the checkpoint: {unexpected_keys}. ")
if len(missing_keys) > 0:
logger.info(f"Missing keys in the checkpoint: {missing_keys}.")
logger.info(f"Loaded model from {pretrained_model_name_or_path}")
else:
raise ValueError(f"Checkpoint {pretrained_model_name_or_path} does not exist. ")
model.to(torch_dtype)
return model
@register_to_config
def __init__(
self,
# sample_size: int = 128,
# patch_size: int = 2,
# in_channels: int = 16,
# num_layers: int = 24,
attention_head_dim: int = 64,
num_attention_heads: int = 24,
# joint_attention_dim: int = 4096,
caption_projection_dim: int = 1536,
pooled_projection_dim: int = 2048,
# out_channels: int = 16,
pos_embed_max_size: int = 192,
dual_attention_layers: Tuple[
int, ...
] = (), # () for sd3.0; (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12) for sd3.5
qk_norm: Optional[str] = "fp32_layer_norm",
# VLA parameters
training_phase: str = "joint", # "joint", "action", "traj2d", "action_mini"
# action head parameters
action_num_blocks:int = 6,
action_pred_horizon: int = 6,
action_dim: int = 7,
action_hidden_dim: int = 1536,
action_nheads: int = 24,
# action_attention_head_dim: Optional[int] = 64,
n_conditions: int = 0,
token_fusion: str = "concat", # "concat", "cross"
resnet_store_path: str = "cache/visual_features/resnet18/IN_1M_resnet18.pth",
odim: int = 32,
view_feature_dim: int = 1920,
use_film: bool = False,
combined_temb: bool = False,
use_future_progress_embedding: bool = False,
future_progress_dim: int = 5,
):
super().__init__()
# self.inner_dim = num_attention_heads * attention_head_dim
self.inner_dim = num_attention_heads * attention_head_dim
# TODO-yth: 扩散时间步embedding是否融合文本语义
self.combined_temb = combined_temb
if self.combined_temb:
self.time_ins_embed = CombinedTimestepTextProjEmbeddings(
embedding_dim=self.inner_dim, pooled_projection_dim=pooled_projection_dim
) # used
else:
self.time_ins_embed = _TimeNetwork(time_dim=256, out_dim=action_hidden_dim)
""" self.lang_embedder = nn.Linear(joint_attention_dim, caption_projection_dim) """
# Set the observation projection
self.obs_proj = ObservationProjection(
action_pred_horizon=action_pred_horizon,
action_dim=action_dim,
hidden_dim=self.inner_dim, # not used
output_dim=action_hidden_dim,
n_conditions=n_conditions,
token_fusion=token_fusion,
resnet_store_path=resnet_store_path,
odim=odim,
view_feature_dim=view_feature_dim,
use_film=use_film
)
total_params = sum(p.numel() for p in self.obs_proj.parameters() if p.requires_grad)
logger.debug(f"ObservationEncoder parameters: {total_params:,}")
# Set the action projection
self.action_proj_in = ActionProjectionIn(
action_pred_horizon=action_pred_horizon,
action_dim=action_dim,
output_dim=action_hidden_dim,
)
self.use_future_progress_embedding = bool(use_future_progress_embedding)
self.future_progress_dim = int(future_progress_dim)
if self.use_future_progress_embedding:
self.future_progress_mlp = nn.Sequential(
nn.Linear(self.future_progress_dim, action_hidden_dim),
nn.SiLU(),
nn.Linear(action_hidden_dim, action_hidden_dim),
)
# future_progress is an additive residual condition. Start it as
# an exact no-op for the same reason as adaLN-Zero and learn the
# conditioning path gradually.
nn.init.zeros_(self.future_progress_mlp[-1].weight)
nn.init.zeros_(self.future_progress_mlp[-1].bias)
else:
self.future_progress_mlp = None
total_params = sum(p.numel() for p in self.action_proj_in.parameters() if p.requires_grad)
logger.debug(f"ActionProjectionIn parameters: {total_params:,}")
self.transformer_blocks = nn.ModuleList(
[
# IdentityBlock()
VLATransformerBlock( # JointTransformerBlock
dim=action_hidden_dim,
num_attention_heads=action_nheads,
attention_head_dim=attention_head_dim,
context_pre_only=i == action_num_blocks - 1,
qk_norm=qk_norm,
# use_dual_attention=True if i in dual_attention_layers else False,
training_phase=training_phase, # "joint", "action", "traj2d", "action_mini"
)
for i in range(action_num_blocks)
]
)
total_params = sum(p.numel() for p in self.transformer_blocks[0].parameters() if p.requires_grad)
logger.debug(f"ActionTransformerBlock parameters: {total_params:,}")
self.action_proj_out = ActionProjectionOut(
hidden_size=action_hidden_dim,
action_dim=action_dim
)
total_params = sum(p.numel() for p in self.action_proj_out.parameters())
logger.debug(f"ActionProjectionOut parameters: {total_params:,}")
def forward(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: Optional[torch.Tensor] = None,
pooled_projections: Optional[torch.Tensor] = None,
timestep: Optional[torch.LongTensor] = None,
block_controlnet_hidden_states: Optional[List] = None,
joint_attention_kwargs: Optional[Dict[str, Any]] = None,
vlm_attn_mask: Optional[torch.Tensor] = None,
return_dict: bool = True,
skip_layers: Optional[List[int]] = None,
) -> Union[List[torch.Tensor], ActionTransformerModelOutput]:
assert joint_attention_kwargs is not None
if self.config.n_conditions > 0: # type: ignore
assert joint_attention_kwargs["traj2ds"].shape[1] == self.config.n_conditions, "model n_conditions does not match" # type: ignore
# L
if self.combined_temb:
temb = self.time_ins_embed(timestep, pooled_projections)
else:
temb = self.time_ins_embed(timestep)
""" lang_hidden_states = self.lang_embedder(encoder_hidden_states) """
# A. For RTC, only the unknown postfix lives on the action/AdaLN
# branch. The known overlap is encoded with the existing action
# projection (no new parameters), then appended to observation context.
noisy_action = joint_attention_kwargs.pop("action_hidden_embeds")
known_action_prefix = joint_attention_kwargs.pop("known_action_prefix", None)
prefix_len = 0
if known_action_prefix is not None:
if known_action_prefix.ndim != noisy_action.ndim:
raise ValueError(
"known_action_prefix and action_hidden_embeds must have "
f"the same rank, got {known_action_prefix.ndim} and {noisy_action.ndim}"
)
if (
known_action_prefix.shape[0] != noisy_action.shape[0]
or known_action_prefix.shape[-1] != noisy_action.shape[-1]
):
raise ValueError(
"known_action_prefix batch/action dimensions must match "
"action_hidden_embeds"
)
prefix_len = int(known_action_prefix.shape[1])
if prefix_len + noisy_action.shape[1] != self.config.action_pred_horizon:
raise ValueError(
"RTC prefix plus postfix must fill the configured action horizon: "
f"prefix={prefix_len}, postfix={noisy_action.shape[1]}, "
f"horizon={self.config.action_pred_horizon}"
)
action_hidden_states = self.action_proj_in(
noisy_action, position_offset=prefix_len
)
prefix_hidden_states = None
if known_action_prefix is not None:
prefix_hidden_states = self.action_proj_in(
known_action_prefix, position_offset=0
)
if self.use_future_progress_embedding:
future_progress = joint_attention_kwargs.get("future_progress", None)
if future_progress is None:
raise ValueError("use_future_progress_embedding=True requires future_progress tensor")
assert self.future_progress_mlp is not None
future_progress = future_progress.to(
device=action_hidden_states.device,
dtype=action_hidden_states.dtype,
)
expected_progress_shape = (
action_hidden_states.shape[0],
prefix_len + action_hidden_states.shape[1],
)
if future_progress.shape[:2] != expected_progress_shape:
raise ValueError(
f"future_progress shape {future_progress.shape} does not match full action "
f"timeline {expected_progress_shape}"
)
if prefix_hidden_states is not None:
prefix_hidden_states = prefix_hidden_states + self.future_progress_mlp(
future_progress[:, :prefix_len]
)
action_hidden_states = action_hidden_states + self.future_progress_mlp(
future_progress[:, prefix_len:]
)
# Obs: V+Proprio
obs_hidden_states, obs_token_mask = self.obs_proj(
views=joint_attention_kwargs["views"],
obs=joint_attention_kwargs["obs"],
traj2ds=joint_attention_kwargs["traj2ds"],
text_embeddings=pooled_projections,
vlm_attn_mask=vlm_attn_mask
) # S, B, d_model
if prefix_hidden_states is not None:
# Prefix is context-only: it is a K/V condition alongside VLM and
# proprio tokens. It never enters action AdaLN or the output head,
# while postfix queries can still attend to it in every block.
obs_hidden_states = torch.cat(
[prefix_hidden_states, obs_hidden_states], dim=1
)
if obs_token_mask is not None:
prefix_token_mask = torch.ones(
(obs_token_mask.shape[0], prefix_len),
device=obs_token_mask.device,
dtype=obs_token_mask.dtype,
)
obs_token_mask = torch.cat(
[prefix_token_mask, obs_token_mask], dim=1
)
for index_block, block in enumerate(self.transformer_blocks):
is_skip = True if skip_layers is not None and index_block in skip_layers else False
if is_skip:
continue
action_hidden_states, obs_hidden_states = block(
# hidden_states=hidden_states, # V
# lang_hidden_states=lang_hidden_states, # L
action_hidden_states=action_hidden_states, # A
obs_hidden_states=obs_hidden_states, # O
temb=temb,
obs_token_mask=obs_token_mask
# time_enc=time_enc,
# obs_pos_emb=obs_pos_emb,
# joint_attention_kwargs=joint_attention_kwargs,
)
action_output = self.action_proj_out(
x=action_hidden_states,
t=temb,
# cond=obs_hidden_states,
)
return ActionTransformerModelOutput(action=action_output)
class DiTActionTransformerModel(
ModelMixin, ConfigMixin #, VLTTransformerActionMixin
# PeftAdapterMixin, FromOriginalModelMixin, ReprodMixin, SD3Transformer2DLoadersMixin,
):
""" An action transformer model """
_supports_gradient_checkpointing = True
# _no_split_modules = ["ActionTransformerBlock"]
# _skip_layerwise_casting_patterns = ["pos_embed", "norm"]
@register_to_config
def __init__(
self,
# sample_size: int = 128,
# patch_size: int = 2,
# in_channels: int = 16,
# num_layers: int = 24,
attention_head_dim: int = 64,
num_attention_heads: int = 24,
# joint_attention_dim: int = 4096,
caption_projection_dim: int = 1536,
pooled_projection_dim: int = 2048,
# out_channels: int = 16,
pos_embed_max_size: int = 192,
dual_attention_layers: Tuple[
int, ...
] = (), # () for sd3.0; (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12) for sd3.5
qk_norm: Optional[str] = None,
# VLA parameters
training_phase: str = "joint", # "joint", "action", "traj2d", "action_mini"
# action head parameters
action_num_blocks:int = 6,
action_pred_horizon: int = 6,
action_dim: int = 7,
action_hidden_dim: int = 1536, #this
action_nheads: int = 24,
# action_attention_head_dim: Optional[int] = 64,
n_conditions: int = 0,
token_fusion: str = "concat", # "concat", "cross"
resnet_store_path: str = "cache/visual_features/resnet18/IN_1M_resnet18.pth",
odim: int = 32,
view_feature_dim: int = 1920,
use_film: bool = False,
combined_temb: bool = False,
use_future_progress_embedding: bool = False,
future_progress_dim: int = 5,
):
super().__init__()
# self.inner_dim = num_attention_heads * attention_head_dim
self.inner_dim = num_attention_heads * attention_head_dim
if use_future_progress_embedding:
raise ValueError("DiTActionTransformerModel does not support future progress embedding")
# Set the observation projection
self.obs_proj = ObservationProjection(
action_pred_horizon=action_pred_horizon,
action_dim=action_dim,
hidden_dim=self.inner_dim, # not used
output_dim=action_hidden_dim,
n_conditions=n_conditions,
token_fusion="concat",
resnet_store_path=resnet_store_path,
odim=odim,
view_feature_dim=view_feature_dim,
use_film=use_film
)
total_params = sum(p.numel() for p in self.obs_proj.parameters() if p.requires_grad)
logger.debug(f"ObservationEncoder parameters: {total_params:,}")
# q_former:
# TODO: boqian: hardcode here
num_query_tokens = 256
num_heads_ = 16
self.query_tokens = nn.Parameter(torch.randn(num_query_tokens, action_hidden_dim))
self.cross_attention = CrossAttentionBlock(action_hidden_dim, num_heads_)
# assert action_hidden_dim == 1024
self.DiT = DiT(
in_channels=action_dim,
depth=24,
num_heads=16,
token_size=action_hidden_dim,
learn_sigma=False,
class_dropout_prob=0,
future_action_window_size=action_pred_horizon-1,
past_action_window_size=0,
n_conditon_token=num_query_tokens
)
total_params = sum(p.numel() for p in self.DiT.parameters() if p.requires_grad)
logger.debug(f"DiT parameters: {total_params:,}")
def forward(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: Optional[torch.Tensor] = None,
pooled_projections: Optional[torch.Tensor] = None,
timestep: Optional[torch.LongTensor] = None,
block_controlnet_hidden_states: Optional[List] = None,
joint_attention_kwargs: Optional[Dict[str, Any]] = None,
vlm_attn_mask: Optional[torch.Tensor] = None,
return_dict: bool = True,
skip_layers: Optional[List[int]] = None,
) -> Union[List[torch.Tensor], ActionTransformerModelOutput]:
assert joint_attention_kwargs is not None
if joint_attention_kwargs.get("known_action_prefix", None) is not None:
raise ValueError(
"Context-only RTC is implemented for ActionTransformerModel; "
"DiTActionTransformerModel cannot consume known_action_prefix."
)
if self.config.n_conditions > 0: # type: ignore
assert joint_attention_kwargs["traj2ds"].shape[1] == self.config.n_conditions, "model n_conditions does not match" # type: ignore
""" lang_hidden_states = self.lang_embedder(encoder_hidden_states) """
# A
noisy_action = joint_attention_kwargs.pop("action_hidden_embeds")
# Obs: V+Proprio
condition_tokens, vlm_attn_mask = self.obs_proj(
views=joint_attention_kwargs["views"],
obs=joint_attention_kwargs["obs"],
traj2ds=joint_attention_kwargs["traj2ds"],
text_embeddings=pooled_projections,
vlm_attn_mask=vlm_attn_mask
) # B, S, d_model
# print("condition_tokens.shape:", condition_tokens.shape) # [B, 106, 1024])
# q_former
# TODO: forward vlm_attn_mask here
query = self.query_tokens.unsqueeze(0).expand(condition_tokens.shape[0], -1, -1) # (B, Q, D)
queried = self.cross_attention(query, condition_tokens) # (B, Q, D)
# DiT
action_output = self.DiT(noisy_action, timestep, queried)
return ActionTransformerModelOutput(action=action_output)
class HumanFoundationModel(nn.Module):
def __init__(self, model_cfg, feature_extractors: dict[str, str]):
super().__init__()
# self.qwen = _QWen_VL_Interface(config=model_cfg)
# self.layer_qformer = get_layerwise_qformer(config=self.config)
# self.action_model = get_action_model(config=self.config)
action_nheads = int(model_cfg.nhead)
if model_cfg.hidden_dim % action_nheads != 0:
raise ValueError(
f"Action hidden dim {model_cfg.hidden_dim} must be divisible by nhead {action_nheads}"
)
action_head_dim = model_cfg.hidden_dim // action_nheads
self.action_header = ActionTransformerModel(
attention_head_dim=action_head_dim,
num_attention_heads=action_nheads,
action_num_blocks=model_cfg.num_blocks,
action_nheads=action_nheads,
resnet_store_path=model_cfg.resnet_store_path,
odim=model_cfg.odim,
action_dim=model_cfg.action_dim,
action_pred_horizon=model_cfg.action_chunk_size,
view_feature_dim=model_cfg.view_feature_dim,
use_film=model_cfg.use_film,
combined_temb=model_cfg.combined_temb,
action_hidden_dim=model_cfg.hidden_dim,
qk_norm=getattr(model_cfg, "qk_norm", "fp32_layer_norm"),
use_future_progress_embedding=getattr(model_cfg, "use_future_progress_embedding", False),
future_progress_dim=getattr(model_cfg, "future_progress_dim", 5),
)
total_params = sum(p.numel() for p in self.action_header.parameters())
overwatch.info(f"Total ActionTransformerModel parameters: {total_params:,}")
vision_backbones = {}
for backbone, model_path in feature_extractors.items():
if backbone == "siglip":
siglip = SiglipModel.from_pretrained(
model_path
) # , resume_download=None, local_files_only=True
vision_backbones[backbone] = siglip.vision_model
vision_backbones[backbone].head = nn.Identity() # remove classification head
del siglip
elif backbone == "dinov2":
vision_backbones[backbone] = Dinov2Model.from_pretrained(
model_path
) # , resume_download=None, local_files_only=True
vision_backbones[backbone].embeddings.mask_token = None
else:
raise ValueError(f"Unsupported vision backbone: {backbone}")
self.vision_backbones = nn.ModuleDict(vision_backbones)
for name, backbone in self.vision_backbones.items():
num_parameters = sum(p.numel() for p in backbone.parameters() if p.requires_grad)
overwatch.info(f"Vision backbone {name} has {num_parameters*1e-6:,}M parameters")
def tune_vision_backbones(self, backbone_names: List[str]):
for name, backbone in self.vision_backbones.items():
if name in backbone_names:
overwatch.info(f"Tuning vision backbone: {name}")
for param in backbone.parameters():
param.requires_grad = True
else:
overwatch.info(f"Freezing vision backbone: {name}")
for param in backbone.parameters():
param.requires_grad = False
def extract_vision_features(self, batch):
features = []
for vision_backbone, model in self.vision_backbones.items():
assert vision_backbone in batch
B, V, C, H, W = batch[vision_backbone].shape
if vision_backbone == "siglip":
vision_outputs = model(
pixel_values=batch[vision_backbone].reshape(B * V, C, H, W),
return_dict=True,
)
siglip_features = vision_outputs.last_hidden_state
features.append(siglip_features.view(B, V, *siglip_features.shape[1:]))
elif vision_backbone == "dinov2":
dinov2_outputs = model(
pixel_values=batch[vision_backbone].view(B * V, C, H, W)
)
dinov2_features = dinov2_outputs.last_hidden_state[:, 1:, :]
features.append(dinov2_features.view(B, V, *dinov2_features.shape[1:]))
else:
raise ValueError(f"Unsupported vision backbone: {vision_backbone}")
return torch.cat(features, dim=3) # (B,V,N,D)
def forward(
self,
action_samples: torch.Tensor,
views: dict[str, torch.Tensor],
states: torch.Tensor,
# hidden_states: torch.Tensor,
lang_hidden_states: torch.Tensor,
pooled_projections: torch.Tensor,
timestep: torch.LongTensor,
# block_controlnet_hidden_states: List,
# joint_attention_kwargs: Optional[Dict[str, Any]],
traj2ds: Optional[torch.Tensor] = None,
return_dict: bool = True,
skip_layers: Optional[List[int]] = None,
) -> Union[List[torch.Tensor], HumanFoundationModelOutput]:
view_features = self.extract_vision_features(views)
model_output = self.action_header(
hidden_states=None,
encoder_hidden_states=lang_hidden_states, # (B, 333, 4096)
timestep=timestep,
pooled_projections=pooled_projections, # (B, 2048)
joint_attention_kwargs=dict(
action_hidden_embeds=action_samples, # (B,Tp,Da)
views=view_features, # (B,V,N,D)
obs=states, # (B,1,M)
traj2ds=traj2ds, # (B, C, 3, H, W)
# text_embeddings=text_embeddings
),
return_dict=return_dict,
)
return HumanFoundationModelOutput(action=model_output.action)
@torch.inference_mode()
def predict_action(
self,
observations: List[List[Image.Image]], # B * List of PIL Image as [view1, view2]
states: List[Any],
instructions: List[str],
**kwargs: str
) -> np.ndarray:
# with torch.autocast("cuda", dtype=torch.float32):
...
@torch.inference_mode()
def chat(self):
...
class Psi0Model(nn.Module):
# 这里改不改2.5无所谓,外部会传参(来自posttrain.py的init_model)
def __init__(self, model_cfg, vlm_model: Optional[nn.Module]):
super().__init__()
self.action_horizon = int(model_cfg.action_chunk_size)
self.action_dim = int(model_cfg.action_dim)
self.action_exec_horizon = int(model_cfg.action_exec_horizon)
self.rtc_max_delay = int(model_cfg.max_delay)
self.rtc_expected_overlap = (
self.action_horizon - self.action_exec_horizon
)
if bool(getattr(model_cfg, "rtc", False)):
_validate_rtc_inference_contract(
action_horizon=self.action_horizon,
action_exec_horizon=self.action_exec_horizon,
inference_delay=self.rtc_expected_overlap,
max_delay=self.rtc_max_delay,
)
if model_cfg.use_dit:
self.action_header = DiTActionTransformerModel(
resnet_store_path=model_cfg.resnet_store_path,
odim=model_cfg.odim,
action_dim=model_cfg.action_dim,
action_pred_horizon=model_cfg.action_chunk_size,
view_feature_dim=model_cfg.view_feature_dim,
use_film=model_cfg.use_film,
combined_temb=model_cfg.combined_temb,
action_hidden_dim=model_cfg.hidden_dim,
qk_norm=getattr(model_cfg, "qk_norm", "fp32_layer_norm"),
use_future_progress_embedding=getattr(model_cfg, "use_future_progress_embedding", False),
future_progress_dim=getattr(model_cfg, "future_progress_dim", 5),
)
else:
action_nheads = int(model_cfg.nhead)
if model_cfg.hidden_dim % action_nheads != 0:
raise ValueError(
f"Action hidden dim {model_cfg.hidden_dim} must be divisible by nhead {action_nheads}"
)
action_head_dim = model_cfg.hidden_dim // action_nheads
self.action_header = ActionTransformerModel(
attention_head_dim=action_head_dim,
num_attention_heads=action_nheads,
action_num_blocks=model_cfg.num_blocks,
action_nheads=action_nheads,
resnet_store_path=model_cfg.resnet_store_path,
odim=model_cfg.odim,
action_dim=model_cfg.action_dim,
action_pred_horizon=model_cfg.action_chunk_size,
view_feature_dim=model_cfg.view_feature_dim,
use_film=model_cfg.use_film,
combined_temb=model_cfg.combined_temb,
action_hidden_dim=model_cfg.hidden_dim,
qk_norm=getattr(model_cfg, "qk_norm", "fp32_layer_norm"),
use_future_progress_embedding=getattr(model_cfg, "use_future_progress_embedding", False),
future_progress_dim=getattr(model_cfg, "future_progress_dim", 5),
)
self.action_header.obs_proj.mask_aware_positions = bool(
getattr(model_cfg, "action_mask_aware_positions", False)
)
if bool(getattr(model_cfg, "action_input_projection_fp32", False)):
if not hasattr(self.action_header, "action_proj_in"):
raise ValueError("FP32 action input projection requires the masked action transformer")
self.action_header.action_proj_in.force_fp32 = True
total_params = sum(p.numel() for p in self.action_header.parameters())
overwatch.info(f"Total ActionTransformerModel parameters: {total_params:,}")
self.vlm_model = vlm_model
if self.vlm_model is None:
overwatch.info("Total VLM Backbone parameters: 0 (cached hidden states only)")
else:
total_params = sum(p.numel() for p in self.vlm_model.parameters())
overwatch.info(f"Total VLM Backbone parameters: {total_params:,}")
# load pretrained vlm+action head, and all the modules needed for predict_action
# TODO-yth: 暂时没看怎么改加载预训练
@classmethod
def from_pretrained(cls, run_dir, ckpt_step, launch_config:LaunchConfig, device):
if not os.path.exists(run_dir):
raise ValueError(f"Pretrained model path {run_dir} does not exist.")
from safetensors.torch import load_file
# <modify> 支持infer脚本传入best_ckpt目录名,用于读取checkpoints/best_ckpt。
# 原逻辑:
# ckpt_dir = f"{run_dir}/checkpoints/ckpt_{ckpt_step}"
if str(ckpt_step) in {"best", "best_ckpt"}:
ckpt_dir = f"{run_dir}/checkpoints/best_ckpt"
else:
ckpt_dir = f"{run_dir}/checkpoints/ckpt_{ckpt_step}"
# <modify> 兼容训练中间保存到run根目录的ckpt_xxx,例如用户当前的.../2604192034/ckpt_8000。
# 原逻辑只读取run_dir/checkpoints/ckpt_xxx。
if not os.path.exists(ckpt_dir):
root_ckpt_dir = f"{run_dir}/ckpt_{ckpt_step}"
if os.path.exists(root_ckpt_dir):
ckpt_dir = root_ckpt_dir
# ckpt_dir = f"{run_dir}/checkpoints/ckpt_800"
file_path = f"{ckpt_dir}/model.safetensors"
index_path = f"{ckpt_dir}/model.safetensors.index.json"
if os.path.exists(file_path):
state_dict = load_file(file_path, device="cpu")
elif os.path.exists(index_path):
with open(index_path, "r") as f:
index_obj = json.load(f)
weight_map = index_obj.get("weight_map", {})
shard_files = sorted(set(weight_map.values()))
state_dict = {}
for shard_file in shard_files:
shard_path = os.path.join(ckpt_dir, shard_file)
if not os.path.exists(shard_path):
raise ValueError(f"Checkpoint shard {shard_path} does not exist.")
state_dict.update(load_file(shard_path, device="cpu"))
else:
raise ValueError(f"Checkpoint file {file_path} does not exist.")
configured_model_name_or_path = str(launch_config.model.model_name_or_path)
model_path_override = os.getenv("PSI_PRETRAIN_BACKBONE_PATH", "").strip()
model_name_or_path = os.path.expanduser(
model_path_override or configured_model_name_or_path
)
if model_path_override:
overwatch.info(
"Using PSI_PRETRAIN_BACKBONE_PATH override for inference: "
f"{model_name_or_path} (run_config={configured_model_name_or_path})"
)
# Transformers treats a missing local absolute path as a Hub repo id,
# which otherwise produces a misleading HFValidationError. Fail early
# with the actual configuration problem instead.
if os.path.isabs(model_name_or_path) and not os.path.isfile(
os.path.join(model_name_or_path, "config.json")
):
source = "PSI_PRETRAIN_BACKBONE_PATH" if model_path_override else "run_config.json"
raise FileNotFoundError(
f"VLM backbone from {source} is not a valid local model directory: "
f"{model_name_or_path} (missing config.json). "
"Set PSI_PRETRAIN_BACKBONE_PATH to a complete local Qwen checkpoint."
)
is_omni = "omni" in model_name_or_path.lower()
# <modify_float> 增加float32推理开关;开启后避免FlashAttention2(fp16/bf16-only)报错。
# 原逻辑:
# attn_impl = "flash_attention_2" if str(device).startswith("cuda") else "sdpa"
force_float32_infer = os.getenv("PSI_INFER_FLOAT32", "0") == "1"
if force_float32_infer:
attn_impl = "sdpa"
vlm_dtype = torch.float32
else:
attn_impl = "flash_attention_2" if str(device).startswith("cuda") else "sdpa"
vlm_dtype = torch.bfloat16
if is_omni:
vlm_model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained(
model_name_or_path,
attn_implementation=attn_impl,
dtype=vlm_dtype,
)
else:
# init empty vlm backbone from config only (skip loading base pretrained weights)
vlm_config = AutoConfig.from_pretrained(model_name_or_path)
vlm_config._attn_implementation = attn_impl
vlm_model = Qwen3VLForConditionalGeneration(vlm_config)
# <modify_float> float32开关打开时,VLM保持float32;默认保持bf16。
# 原逻辑:
# vlm_model = vlm_model.to(dtype=torch.bfloat16) # type: ignore
vlm_model = vlm_model.to(dtype=vlm_dtype) # type: ignore
vlm_state_dict = {}
for k, v in state_dict.items():
if k.startswith("action_header."):
continue
elif k.startswith('vlm_model.'):
vlm_state_dict[k.replace("vlm_model.", "")] = v
elif k.startswith("motion_encoder."):
continue
else:
assert False, "check here"
if vlm_state_dict:
if "lm_head.weight" not in vlm_state_dict:
if "model.language_model.embed_tokens.weight" in vlm_state_dict:
vlm_state_dict["lm_head.weight"] = vlm_state_dict["model.language_model.embed_tokens.weight"]
elif "model.embed_tokens.weight" in vlm_state_dict:
vlm_state_dict["lm_head.weight"] = vlm_state_dict["model.embed_tokens.weight"]
if "lm_head.weight" not in vlm_state_dict:
raise KeyError(
"Checkpoint contains VLM weights but neither lm_head.weight nor a supported "
"embed_tokens.weight key was found."
)
if vlm_state_dict["lm_head.weight"].shape[0] != vlm_model.lm_head.weight.shape[0]:
vlm_model.resize_token_embeddings(
vlm_state_dict["lm_head.weight"].shape[0],
pad_to_multiple_of=192,
mean_resizing=True,
)
overwatch.info(f"Resized model token embeddings to {vlm_model.lm_head.weight.shape[0]}")
vlm_model.load_state_dict(vlm_state_dict, strict=True)
overwatch.info("loaded vlm_backbone checkpoint successfully.")
else:
if not is_omni:
raise ValueError(
"Checkpoint contains no VLM weights, but this non-Omni inference path only "
"initialized the VLM from config. A pretrained base VLM is required."
)
# Cached-hidden-state training with PSI_SKIP_VLM_MODEL=1 saves an
# action-only checkpoint. The Omni backbone above was already loaded
# from model_name_or_path, so keep those frozen base Qwen weights.
overwatch.info(
f"Checkpoint {ckpt_dir} contains no VLM weights; retaining pretrained "
f"VLM backbone from {model_name_or_path}."
)
# init hfm-together model with vlm backbone
model = cls(
model_cfg=launch_config.model,
vlm_model = vlm_model,
)
# load action head
action_head_state_dict = {}
for k, v in state_dict.items():
if k.startswith("vlm_model."):
continue
elif k.startswith('action_header.'):
action_head_state_dict[k.replace("action_header.", "")] = v
elif k.startswith("motion_encoder."):
continue
else:
assert False, "check here"
model.action_header.load_state_dict(action_head_state_dict, strict=True)
overwatch.info("loaded action head checkpoint successfully.")
if hasattr(model, "motion_encoder"):
motion_encoder_state_dict = {
k.replace("motion_encoder.", ""): v
for k, v in state_dict.items()
if k.startswith("motion_encoder.")
}
if motion_encoder_state_dict:
model.motion_encoder.load_state_dict(motion_encoder_state_dict, strict=True) # type: ignore[attr-defined]
overwatch.info("loaded motion encoder checkpoint successfully.")
# load necessary modules
if is_omni:
model.vlm_processor = Qwen2_5OmniProcessor.from_pretrained(model_name_or_path)
else:
model.vlm_processor = AutoProcessor.from_pretrained(model_name_or_path)
# model.tokenizer = model.vlm_processor.tokenizer
if launch_config.model.noise_scheduler == "ddpm":
from diffusers.schedulers.scheduling_ddim import DDIMScheduler
scheduler = DDIMScheduler(
num_train_timesteps=launch_config.model.train_diffusion_steps,
beta_start=0.0001,
beta_end=0.02,
beta_schedule="squaredcos_cap_v2",
clip_sample=True,
set_alpha_to_one=True,
steps_offset=0,
prediction_type="epsilon",
)
elif launch_config.model.noise_scheduler == "flow":
from diffusers.schedulers.scheduling_flow_match_euler_discrete import (FlowMatchEulerDiscreteScheduler)
scheduler= FlowMatchEulerDiscreteScheduler(
num_train_timesteps=launch_config.model.train_diffusion_steps, # MUST be 1000 as per pretrained SD3
)
else:
raise ValueError(f"Unsupported noise scheduler: {launch_config.model.noise_scheduler}")
model.noise_scheduler = scheduler
model.action_horizon = launch_config.model.action_chunk_size
model.action_dim = launch_config.model.action_dim
model.text_only = bool(getattr(launch_config.data.transform.model, "text_only", False))
model.audio_only = bool(getattr(launch_config.data.transform.model, "audio_only", False))
model.video_only = bool(getattr(launch_config.data.transform.model, "video_only", False))
model.video_size_pixels = int(getattr(launch_config.data.transform.model, "video_size_pixels", 50176))
model.use_delta_actions = bool(getattr(launch_config.data.transform.repack, "use_delta_actions", False))
# <modify_jerk> 记录hybrid action模式,方便infer/调试区分root-delta checkpoint。
model.root_delta_actions = bool(getattr(launch_config.data.transform.repack, "root_delta_actions", False))
# 记录TextOp动作checkpoint,推理脚本据此走一致的decode路径。
model.textop_actions = bool(getattr(launch_config.data.transform.repack, "textop_actions", False))
# <modify_float> 记录推理精度配置,供predict_action构造autocast上下文。
model.infer_autocast_dtype = vlm_dtype
model.device = device
return model
def _action_condition_hidden_states(
self,
vlm_hidden_states: torch.Tensor,
input_ids: Optional[torch.Tensor] = None,
attention_mask: Optional[torch.Tensor] = None,
) -> torch.Tensor:
del input_ids, attention_mask
return vlm_hidden_states.unsqueeze(1)
def _omni_hidden_states_without_lm_head(self, vlm_kwargs: dict[str, Any]) -> torch.Tensor:
"""Run Qwen2.5-Omni thinker up to hidden states without materializing logits."""
input_ids = vlm_kwargs.get("input_ids", None)
attention_mask = vlm_kwargs.get("attention_mask", None)
input_features = vlm_kwargs.get("input_features", None)
pixel_values = vlm_kwargs.get("pixel_values", None)
pixel_values_videos = vlm_kwargs.get("pixel_values_videos", None)
image_grid_thw = vlm_kwargs.get("image_grid_thw", None)
video_grid_thw = vlm_kwargs.get("video_grid_thw", None)
feature_attention_mask = vlm_kwargs.get("feature_attention_mask", None)
audio_feature_lengths = vlm_kwargs.get("audio_feature_lengths", None)
video_second_per_grid = vlm_kwargs.get("video_second_per_grid", None)
inputs_embeds = vlm_kwargs.get("inputs_embeds", None)
position_ids = vlm_kwargs.get("position_ids", None)
cache_position = vlm_kwargs.get("cache_position", None)
use_audio_in_video = vlm_kwargs.get("use_audio_in_video", None)
if inputs_embeds is None:
inputs_embeds = self.vlm_model.get_input_embeddings()(input_ids)
if input_features is not None:
audio_features = self.vlm_model.get_audio_features(
input_features,
feature_attention_mask=feature_attention_mask,
audio_feature_lengths=audio_feature_lengths,
)
audio_features = audio_features.to(inputs_embeds.device, inputs_embeds.dtype)
_, _, audio_mask = self.vlm_model.get_placeholder_mask(input_ids, inputs_embeds=inputs_embeds)
inputs_embeds = inputs_embeds.masked_scatter(audio_mask, audio_features)
if pixel_values is not None:
image_embeds = self.vlm_model.get_image_features(pixel_values, image_grid_thw)
image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
image_mask, _, _ = self.vlm_model.get_placeholder_mask(
input_ids, inputs_embeds=inputs_embeds, image_features=image_embeds
)
inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)
if pixel_values_videos is not None:
video_embeds = self.vlm_model.get_video_features(pixel_values_videos, video_grid_thw)
video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
_, video_mask, _ = self.vlm_model.get_placeholder_mask(
input_ids, inputs_embeds=inputs_embeds, video_features=video_embeds
)
inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds)
if feature_attention_mask is not None:
audio_feature_lengths = torch.sum(feature_attention_mask, dim=1)
else:
audio_feature_lengths = None
if attention_mask is not None and position_ids is None:
if cache_position is None or (cache_position is not None and cache_position[0] == 0) or self.vlm_model.rope_deltas is None:
delta0 = (1 - attention_mask).sum(dim=-1).unsqueeze(1)
position_ids, rope_deltas = self.vlm_model.get_rope_index(
input_ids,
image_grid_thw,
video_grid_thw,
attention_mask,
use_audio_in_video,
audio_feature_lengths,
video_second_per_grid,
)
self.vlm_model.rope_deltas = rope_deltas - delta0
else:
batch_size, seq_length = input_ids.shape
delta = cache_position[0] + self.vlm_model.rope_deltas if cache_position is not None else 0
position_ids = torch.arange(seq_length, device=input_ids.device)
position_ids = position_ids.view(1, -1).expand(batch_size, -1)
position_ids = position_ids.add(delta)
position_ids = position_ids.unsqueeze(0).expand(3, -1, -1)
outputs = self.vlm_model.model(
attention_mask=attention_mask,
position_ids=position_ids,
past_key_values=vlm_kwargs.get("past_key_values", None),
inputs_embeds=inputs_embeds,
use_cache=False,
output_attentions=vlm_kwargs.get("output_attentions", None),
output_hidden_states=True,
return_dict=True,
cache_position=cache_position,
)
return outputs.hidden_states[-1]
def _extract_vlm_hidden_states(self, vlm_kwargs: dict[str, Any]) -> torch.Tensor:
if self.vlm_model.__class__.__name__ == "Qwen2_5OmniThinkerForConditionalGeneration":
return self._omni_hidden_states_without_lm_head(vlm_kwargs)
return self.vlm_model(**vlm_kwargs).hidden_states[-1]
# TODO-yth: forward入口,vlm(Qwen3VLForConditionalGeneration)输出hidden最后一层到action_head,得到action_pred和目标噪声算MSE
def forward(
self,
input_ids,
attention_mask,
pixel_values,
image_grid_thw,
action_samples: torch.Tensor,
states: torch.Tensor,
timestep: torch.LongTensor,
# <modify_audio> 支持Qwen2.5-Omni audio-only processor输出的音频特征。
input_features=None,
feature_attention_mask=None,
audio_feature_lengths=None,
vlm_hidden_states=None,
pixel_values_videos=None,
video_grid_thw=None,
video_second_per_grid=None,
traj2ds: Optional[torch.Tensor] = None,
progress: Optional[torch.Tensor] = None,
future_progress: Optional[torch.Tensor] = None,
return_dict: bool = True,
skip_layers: Optional[List[int]] = None,
known_action_prefix: Optional[torch.Tensor] = None,
) -> Union[List[torch.Tensor], HumanFoundationModelOutput]:
# <modify>
# qwen2.5里面有一行delta0 = (1 - attention_mask),而qwen3这边接收的mask是bool,所以这里要转成long
vlm_attention_mask = attention_mask
if vlm_attention_mask is not None and vlm_attention_mask.dtype == torch.bool:
vlm_attention_mask = vlm_attention_mask.to(dtype=torch.long)
# <modify_audio> 原逻辑固定传pixel_values/image_grid_thw;现在按实际batch动态拼VLM输入,避免Qwen3VL收到音频空参数。
# 原逻辑:
# vlm_hidden_states = self.vlm_model(
# input_ids=input_ids,
# attention_mask=vlm_attention_mask,
# pixel_values=pixel_values,
# image_grid_thw=image_grid_thw,
# output_hidden_states=True,
# return_dict=True
# ).hidden_states[-1]
vlm_kwargs = {
"input_ids": input_ids,
"attention_mask": vlm_attention_mask,
"output_hidden_states": True,
"return_dict": True,
"use_cache": False,
}
if pixel_values is not None:
vlm_kwargs["pixel_values"] = pixel_values
if image_grid_thw is not None:
vlm_kwargs["image_grid_thw"] = image_grid_thw
if input_features is not None:
vlm_kwargs["input_features"] = input_features
if feature_attention_mask is not None:
vlm_kwargs["feature_attention_mask"] = feature_attention_mask
if audio_feature_lengths is not None:
vlm_kwargs["audio_feature_lengths"] = audio_feature_lengths
if pixel_values_videos is not None:
vlm_kwargs["pixel_values_videos"] = pixel_values_videos
if video_grid_thw is not None:
vlm_kwargs["video_grid_thw"] = video_grid_thw
if video_second_per_grid is not None:
vlm_kwargs["video_second_per_grid"] = video_second_per_grid
# extract vision/audio + language features, or reuse precomputed Qwen-Omni hidden states.
if vlm_hidden_states is None:
if self.vlm_model is None:
raise RuntimeError(
"VLM backbone is not loaded because PSI_SKIP_VLM_MODEL=1; "
"this training run requires every sample to provide cached vlm_hidden_states."
)
vlm_hidden_states = self._extract_vlm_hidden_states(vlm_kwargs)
else:
vlm_hidden_states = vlm_hidden_states.to(device=states.device, dtype=states.dtype)
# print(vlm_hidden_states.shape)
model_output = self.action_header(
hidden_states=None,
timestep=timestep,
joint_attention_kwargs=dict(
action_hidden_embeds=action_samples,
known_action_prefix=known_action_prefix,
views=vlm_hidden_states.unsqueeze(1), # (B,V,N,D)
obs=states, # (B,1,M)
traj2ds=traj2ds, # (B, C, 3, H, W)
progress=progress,
future_progress=future_progress,
),
vlm_attn_mask=attention_mask, # (B, seq_len)
return_dict=return_dict,
)
return HumanFoundationModelOutput(action=model_output.action)
def _build_video_only_inputs(
self,
texts,
video=None,
video_embed=None,
video_size_pixels: int = 50176,
video_sample_fps: float = 1.0,
video_pixel_cache_dtype: str = "none",
):
if video_embed is not None:
pixel_values_videos = torch.as_tensor(
video_embed["pixel_values_videos"],
dtype=torch.float32,
device=self.device,
)
video_grid_thw = torch.as_tensor(
video_embed["video_grid_thw"],
dtype=torch.long,
device=self.device,
)
if video_grid_thw.dim() == 1:
video_grid_thw = video_grid_thw.unsqueeze(0)
if "video_second_per_grid" in video_embed:
video_second_per_grid = torch.as_tensor(
video_embed["video_second_per_grid"],
dtype=torch.float32,
device=self.device,
).reshape(-1)
else:
video_second_per_grid = torch.tensor(
[float(self.vlm_processor.video_processor.temporal_patch_size) / float(video_sample_fps)],
dtype=torch.float32,
device=self.device,
)
texts = self.vlm_processor.replace_multimodal_special_tokens(
texts,
iter([]),
iter([]),
iter([video_grid_thw[0].detach().cpu()]),
iter([float(video_second_per_grid.reshape(-1)[0].item())]),
False,
25,
2.0,
)
text_inputs = self.vlm_processor.tokenizer(
texts,
padding=True,
return_tensors="pt",
)
inputs = {key: value.to(self.device) for key, value in text_inputs.items()}
inputs["pixel_values_videos"] = pixel_values_videos
inputs["video_grid_thw"] = video_grid_thw
inputs["video_second_per_grid"] = video_second_per_grid
return inputs
inputs = self.vlm_processor(
text=texts,
videos=[np.asarray(video, dtype=np.uint8)],
padding=True,
return_tensors="pt",
use_audio_in_video=False,
videos_kwargs={
"size": {
"shortest_edge": int(video_size_pixels),
"longest_edge": int(video_size_pixels),
},
"fps": float(video_sample_fps),
},
).to(self.device)
# The v23 video-clip training cache stores Qwen pixel_values_videos as
# FP16 before exporting BF16 hidden states. Reproduce that quantization
# for raw-video inference so the frozen Qwen input matches training.
cache_dtype = str(video_pixel_cache_dtype).strip().lower()
if cache_dtype not in {"", "none"}:
quant_dtype = {
"fp16": torch.float16,
"float16": torch.float16,
"bf16": torch.bfloat16,
"bfloat16": torch.bfloat16,
}.get(cache_dtype)
if quant_dtype is None:
raise ValueError(f"Unsupported video_pixel_cache_dtype={video_pixel_cache_dtype!r}")
inputs["pixel_values_videos"] = (
inputs["pixel_values_videos"].to(dtype=quant_dtype).to(dtype=torch.float32)
)
return inputs
@torch.inference_mode()
def predict_action(
self,
observations: Optional[List[List[Image.Image]]], # B * List of PIL Image as [view1, view2]
states: torch.Tensor, # (B, Ts, Ds)
instructions: List[str], # (B,)
num_inference_steps: int,
traj2ds,
**kwargs: str
) -> torch.Tensor:
bsz = states.shape[0]
progress = kwargs.get("progress", None)
future_progress = kwargs.get("future_progress", None)
text_only = bool(getattr(self, "text_only", False))
audio_only = bool(getattr(self, "audio_only", False))
video_only = bool(getattr(self, "video_only", False))
batch_input_ids = []
batch_attention_mask = []
batch_pixel_values = [] if (not text_only and not audio_only and not video_only) else None
batch_image_grid_thw = [] if (not text_only and not audio_only and not video_only) else None
# <modify_audio> audio推理时从kwargs接收每个batch样本的waveform片段,可同时使用文本指令。
batch_input_features = [] if audio_only else None
batch_feature_attention_mask = [] if audio_only else None
batch_pixel_values_videos = [] if video_only else None
batch_video_grid_thw = [] if video_only else None
batch_video_second_per_grid = [] if video_only else None
if audio_only:
audios = kwargs.get("audios", None)
assert audios is not None, "audios must be provided for audio-only inference"
audio_items = list(audios)
audio_instructions = list(instructions or [""] * len(audio_items))
assert len(audio_instructions) == len(audio_items), "instructions must match audios for audio-only inference"
items = list(zip(audio_items, audio_instructions))
elif video_only:
video_embeds = kwargs.get("video_embeds", None)
if video_embeds is not None:
video_paths = kwargs.get("video_paths", ["video.mp4"] * len(video_embeds))
items = list(zip(video_embeds, video_paths, instructions))
else:
videos = kwargs.get("videos", None)
assert videos is not None, "videos must be provided for video-only inference"
video_paths = kwargs.get("video_paths", ["video.mp4"] * len(videos))
items = list(zip(videos, video_paths, instructions))
elif text_only:
items = [(None, instruction) for instruction in instructions]
else:
assert observations is not None, "observations must be provided for non-text-only inference"
items = list(zip(observations, instructions))
for item in items:
if audio_only:
audio, instruction = item
content = [{"type": "audio", "audio": "audio.wav"}]
if str(instruction).strip():
content.append({"type": "text", "text": str(instruction).strip()})
messages = [[
{"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]},
{"role": "user", "content": content},
]]
elif video_only:
video_item, video_path, instruction = item
content = [{"type": "video", "video": str(video_path or "video.mp4")}]
if str(instruction).strip():
content.append({"type": "text", "text": str(instruction).strip()})
messages = [[
{"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]},
{"role": "user", "content": content},
]]
elif text_only:
_, instruction = item
messages = [[
{"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]},
{"role": "user", "content": [{"type": "text", "text": instruction}]},
]]
else:
observation, instruction = item
assert observation is not None
messages = []
content = [{"type": "image", "image": img} for img in observation]
content.append({"type": "text", "text": instruction})
user_msg = {"role": "user", "content": content}
messages.append([user_msg])
texts = [
self.vlm_processor.apply_chat_template(
m, tokenize=False, add_generation_prompt=True
)
for m in messages
]
if audio_only:
inputs = self.vlm_processor(
text=texts,
audio=[np.asarray(audio, dtype=np.float32)],
padding=True,
return_tensors="pt",
).to(self.device)
elif video_only:
video_size_pixels = int(kwargs.get("video_size_pixels", getattr(self, "video_size_pixels", 50176)))
video_sample_fps = float(kwargs.get("video_sample_fps", 1.0))
inputs = self._build_video_only_inputs(
texts,
video=None if isinstance(video_item, dict) else video_item,
video_embed=video_item if isinstance(video_item, dict) else None,
video_size_pixels=video_size_pixels,
video_sample_fps=video_sample_fps,
video_pixel_cache_dtype=str(kwargs.get("video_pixel_cache_dtype", "none")),
)
elif text_only:
inputs = self.vlm_processor(
text=texts,
padding=True,
return_tensors="pt",
).to(self.device)
else:
image_inputs, video_inputs = process_vision_info(messages, image_patch_size=16)
inputs = self.vlm_processor(
text=texts,
images=image_inputs,
videos=video_inputs,
padding=True,
return_tensors="pt",
).to(self.device)
batch_input_ids.append(inputs['input_ids'].squeeze(0))
batch_attention_mask.append(inputs['attention_mask'].squeeze(0))
if audio_only:
assert batch_input_features is not None and batch_feature_attention_mask is not None
batch_input_features.append(inputs["input_features"].squeeze(0))
batch_feature_attention_mask.append(inputs["feature_attention_mask"].squeeze(0))
if video_only:
assert batch_pixel_values_videos is not None and batch_video_grid_thw is not None
assert batch_video_second_per_grid is not None
batch_pixel_values_videos.append(inputs["pixel_values_videos"].squeeze(0))
batch_video_grid_thw.append(inputs["video_grid_thw"].squeeze(0))
batch_video_second_per_grid.append(inputs["video_second_per_grid"].reshape(-1))
if not text_only and not audio_only and not video_only:
assert batch_pixel_values is not None and batch_image_grid_thw is not None
batch_pixel_values.append(inputs['pixel_values'])
batch_image_grid_thw.append(inputs['image_grid_thw'].squeeze(0))
pad_token_id = int(getattr(self.vlm_processor.tokenizer, "pad_token_id", 0))
batch_input_ids = torch.nn.utils.rnn.pad_sequence(
batch_input_ids, batch_first=True, padding_value=pad_token_id
)
batch_attention_mask = torch.nn.utils.rnn.pad_sequence(
batch_attention_mask, batch_first=True, padding_value=0
)
if audio_only:
assert batch_input_features is not None and batch_feature_attention_mask is not None
# <modify_full_audio> 整段音频推理时不同样本audio feature长度可能不同,按batch最长补零。
max_audio_len = max(int(feature.shape[-1]) for feature in batch_input_features)
padded_input_features = []
padded_feature_attention_mask = []
for feature, feature_mask in zip(batch_input_features, batch_feature_attention_mask):
pad_len = max_audio_len - int(feature.shape[-1])
if pad_len > 0:
feature_pad_shape = list(feature.shape)
feature_pad_shape[-1] = pad_len
feature = torch.cat(
[feature, torch.zeros(feature_pad_shape, dtype=feature.dtype, device=feature.device)],
dim=-1,
)
mask_pad_shape = list(feature_mask.shape)
mask_pad_shape[-1] = pad_len
feature_mask = torch.cat(
[feature_mask, torch.zeros(mask_pad_shape, dtype=feature_mask.dtype, device=feature_mask.device)],
dim=-1,
)
padded_input_features.append(feature)
padded_feature_attention_mask.append(feature_mask)
batch_input_features = torch.stack(padded_input_features)
batch_feature_attention_mask = torch.stack(padded_feature_attention_mask)
if not text_only and not audio_only and not video_only:
assert batch_pixel_values is not None and batch_image_grid_thw is not None
batch_pixel_values = torch.stack(batch_pixel_values) # (B, 256, 1536)
batch_image_grid_thw = torch.stack(batch_image_grid_thw) # (B, 3)
if video_only:
assert batch_pixel_values_videos is not None and batch_video_grid_thw is not None
assert batch_video_second_per_grid is not None
batch_pixel_values_videos = torch.cat(batch_pixel_values_videos, dim=0)
batch_video_grid_thw = torch.stack(batch_video_grid_thw)
batch_video_second_per_grid = torch.cat(batch_video_second_per_grid, dim=0)
# <modify_float> 推理时按infer_autocast_dtype选择精度;float32时不启用autocast。
# 原逻辑:
# autocast_ctx = (
# torch.autocast("cuda", dtype=torch.bfloat16)
# if str(self.device).startswith("cuda")
# else contextlib.nullcontext()
# )
infer_dtype = getattr(self, "infer_autocast_dtype", torch.bfloat16)
if str(self.device).startswith("cuda") and infer_dtype in (torch.float16, torch.bfloat16):
autocast_ctx = torch.autocast("cuda", dtype=infer_dtype)
else:
autocast_ctx = contextlib.nullcontext()
with autocast_ctx:
# extract vision + language features
if audio_only:
output = self.vlm_model(
input_ids=batch_input_ids,
attention_mask=batch_attention_mask,
input_features=batch_input_features, # type: ignore
feature_attention_mask=batch_feature_attention_mask, # type: ignore
output_hidden_states=True,
return_dict=True
)
elif video_only:
output = self.vlm_model(
input_ids=batch_input_ids,
attention_mask=batch_attention_mask,
pixel_values_videos=batch_pixel_values_videos, # type: ignore
video_grid_thw=batch_video_grid_thw, # type: ignore
video_second_per_grid=batch_video_second_per_grid, # type: ignore
output_hidden_states=True,
return_dict=True,
)
elif text_only:
output = self.vlm_model(
input_ids=batch_input_ids,
attention_mask=batch_attention_mask,
output_hidden_states=True,
return_dict=True
)
else:
output = self.vlm_model(
input_ids=batch_input_ids,
attention_mask=batch_attention_mask,
pixel_values=batch_pixel_values, # type: ignore
image_grid_thw=batch_image_grid_thw, # type: ignore
output_hidden_states=True,
return_dict=True
)
vlm_hidden_states_ = output.hidden_states # len(vlm_hidden_states_) == 29
# use hidden states from the last layer
vlm_hidden_states = vlm_hidden_states_[-1] # shape (B, seq_len, D_h) shape(16, 80, 2048)
vlm_hidden_states = self._action_condition_hidden_states(
vlm_hidden_states.to(dtype=states.dtype),
input_ids=batch_input_ids,
attention_mask=batch_attention_mask,
) # shape (B, 1, seq_len, D_h) or subclass-specific latent tokens
# generate action from noise
action_samples = torch.randn(
bsz, self.action_horizon, self.action_dim, device=self.device
)
self.noise_scheduler.set_timesteps(num_inference_steps)
for timestep in self.noise_scheduler.timesteps:
batched_timestep = timestep.expand(bsz).to(self.device)
model_pred = self.action_header(
hidden_states=None,
timestep=batched_timestep,
joint_attention_kwargs=dict(
action_hidden_embeds=action_samples, # (B,Tp,Da)
views=vlm_hidden_states, # (B,V,N,D)
obs=states, # (B,1,M)
traj2ds=traj2ds, # (B, C, 3, H, W)
progress=progress,
future_progress=future_progress,
),
return_dict=True,
).action
action_samples = self.noise_scheduler.step(
model_output=model_pred, timestep=timestep, sample=action_samples # type: ignore
).prev_sample
return action_samples.float()
@torch.inference_mode()
def predict_action_with_training_rtc_flow(
self,
observations: Optional[List[List[Image.Image]]], # B * List of PIL Image as [view1, view2]
states: torch.Tensor, # (B, Ts, Ds)
instructions: List[str], # (B,)
num_inference_steps: int,
traj2ds,
prev_actions: torch.Tensor = None, # (1, H, D)
inference_delay: int = 0,
max_delay: int = 0,
**kwargs: str
) -> torch.Tensor:
"""Predict a full RTC chunk with v18-style clean overlap positions.
Callers shift the unexecuted tail of the previous chunk into the first
``inference_delay`` positions. All 30 positions use the same action
forward; the overlap positions are reset to those clean actions before
every denoising step and in the returned chunk.
"""
# RTC overlap: [known tail from previous chunk | newly generated tail]
H = self.action_horizon
bsz = states.shape[0]
if prev_actions is None or inference_delay <= 0 or max_delay <= 0:
raise ValueError(
"prev_actions, a positive inference_delay, and max_delay "
"must be provided for RTC inference"
)
if prev_actions.ndim != 3 or prev_actions.shape != (bsz, H, self.action_dim):
raise ValueError(
"prev_actions must have shape "
f"({bsz}, {H}, {self.action_dim}), got {tuple(prev_actions.shape)}"
)
prev_actions = prev_actions.to(device=self.device, dtype=torch.float32)
if not hasattr(self, "action_exec_horizon") or not hasattr(self, "rtc_max_delay"):
raise RuntimeError(
"RTC model is missing action_exec_horizon/max_delay deployment metadata"
)
if int(max_delay) != int(self.rtc_max_delay):
raise ValueError(
"RTC caller max_delay does not match the loaded model config: "
f"caller={max_delay}, model={self.rtc_max_delay}"
)
d = _validate_rtc_inference_contract(
action_horizon=H,
action_exec_horizon=int(self.action_exec_horizon),
inference_delay=int(inference_delay),
max_delay=int(self.rtc_max_delay),
)
progress = kwargs.get("progress", None)
future_progress = kwargs.get("future_progress", None)
text_only = bool(getattr(self, "text_only", False))
audio_only = bool(getattr(self, "audio_only", False))
video_only = bool(getattr(self, "video_only", False))
batch_input_ids = []
batch_attention_mask = []
batch_pixel_values = [] if (not text_only and not audio_only and not video_only) else None
batch_image_grid_thw = [] if (not text_only and not audio_only and not video_only) else None
# <modify_audio> audio RTC推理同样从kwargs接收waveform片段,可同时使用文本指令。
batch_input_features = [] if audio_only else None
batch_feature_attention_mask = [] if audio_only else None
batch_pixel_values_videos = [] if video_only else None
batch_video_grid_thw = [] if video_only else None
batch_video_second_per_grid = [] if video_only else None
if audio_only:
audios = kwargs.get("audios", None)
assert audios is not None, "audios must be provided for audio-only inference"
audio_items = list(audios)
audio_instructions = list(instructions or [""] * len(audio_items))
assert len(audio_instructions) == len(audio_items), "instructions must match audios for audio-only inference"
items = list(zip(audio_items, audio_instructions))
elif video_only:
video_embeds = kwargs.get("video_embeds", None)
if video_embeds is not None:
video_paths = kwargs.get("video_paths", ["video.mp4"] * len(video_embeds))
items = list(zip(video_embeds, video_paths, instructions))
else:
videos = kwargs.get("videos", None)
assert videos is not None, "videos must be provided for video-only inference"
video_paths = kwargs.get("video_paths", ["video.mp4"] * len(videos))
items = list(zip(videos, video_paths, instructions))
elif text_only:
items = [(None, instruction) for instruction in instructions]
else:
assert observations is not None, "observations must be provided for non-text-only inference"
items = list(zip(observations, instructions))
for item in items:
if audio_only:
audio, instruction = item
content = [{"type": "audio", "audio": "audio.wav"}]
if str(instruction).strip():
content.append({"type": "text", "text": str(instruction).strip()})
messages = [[
{"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]},
{"role": "user", "content": content},
]]
elif video_only:
video_item, video_path, instruction = item
content = [{"type": "video", "video": str(video_path or "video.mp4")}]
if str(instruction).strip():
content.append({"type": "text", "text": str(instruction).strip()})
messages = [[
{"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]},
{"role": "user", "content": content},
]]
elif text_only:
_, instruction = item
messages = [[
{"role": "system", "content": [{"type": "text", "text": OMNI_DEFAULT_SYSTEM_PROMPT}]},
{"role": "user", "content": [{"type": "text", "text": instruction}]},
]]
else:
observation, instruction = item
assert observation is not None
messages = []
content = [{"type": "image", "image": img} for img in observation]
content.append({"type": "text", "text": instruction})
user_msg = {"role": "user", "content": content}
messages.append([user_msg])
texts = [
self.vlm_processor.apply_chat_template(
m, tokenize=False, add_generation_prompt=True
)
for m in messages
]
if audio_only:
inputs = self.vlm_processor(
text=texts,
audio=[np.asarray(audio, dtype=np.float32)],
padding=True,
return_tensors="pt",
).to(self.device)
elif video_only:
video_size_pixels = int(kwargs.get("video_size_pixels", getattr(self, "video_size_pixels", 50176)))
video_sample_fps = float(kwargs.get("video_sample_fps", 1.0))
inputs = self._build_video_only_inputs(
texts,
video=None if isinstance(video_item, dict) else video_item,
video_embed=video_item if isinstance(video_item, dict) else None,
video_size_pixels=video_size_pixels,
video_sample_fps=video_sample_fps,
video_pixel_cache_dtype=str(kwargs.get("video_pixel_cache_dtype", "none")),
)
elif text_only:
inputs = self.vlm_processor(
text=texts,
padding=True,
return_tensors="pt",
).to(self.device)
else:
image_inputs, video_inputs = process_vision_info(messages, image_patch_size=16)
inputs = self.vlm_processor(
text=texts,
images=image_inputs,
videos=video_inputs,
padding=True,
return_tensors="pt",
).to(self.device)
batch_input_ids.append(inputs['input_ids'].squeeze(0))
batch_attention_mask.append(inputs['attention_mask'].squeeze(0))
if audio_only:
assert batch_input_features is not None and batch_feature_attention_mask is not None
batch_input_features.append(inputs["input_features"].squeeze(0))
batch_feature_attention_mask.append(inputs["feature_attention_mask"].squeeze(0))
if video_only:
assert batch_pixel_values_videos is not None and batch_video_grid_thw is not None
assert batch_video_second_per_grid is not None
batch_pixel_values_videos.append(inputs["pixel_values_videos"].squeeze(0))
batch_video_grid_thw.append(inputs["video_grid_thw"].squeeze(0))
batch_video_second_per_grid.append(inputs["video_second_per_grid"].reshape(-1))
if not text_only and not audio_only and not video_only:
assert batch_pixel_values is not None and batch_image_grid_thw is not None
batch_pixel_values.append(inputs['pixel_values'])
batch_image_grid_thw.append(inputs['image_grid_thw'].squeeze(0))
pad_token_id = int(getattr(self.vlm_processor.tokenizer, "pad_token_id", 0))
batch_input_ids = torch.nn.utils.rnn.pad_sequence(
batch_input_ids, batch_first=True, padding_value=pad_token_id
)
batch_attention_mask = torch.nn.utils.rnn.pad_sequence(
batch_attention_mask, batch_first=True, padding_value=0
)
if audio_only:
assert batch_input_features is not None and batch_feature_attention_mask is not None
# <modify_full_audio> 整段音频RTC推理同样补齐batch内audio feature长度。
max_audio_len = max(int(feature.shape[-1]) for feature in batch_input_features)
padded_input_features = []
padded_feature_attention_mask = []
for feature, feature_mask in zip(batch_input_features, batch_feature_attention_mask):
pad_len = max_audio_len - int(feature.shape[-1])
if pad_len > 0:
feature_pad_shape = list(feature.shape)
feature_pad_shape[-1] = pad_len
feature = torch.cat(
[feature, torch.zeros(feature_pad_shape, dtype=feature.dtype, device=feature.device)],
dim=-1,
)
mask_pad_shape = list(feature_mask.shape)
mask_pad_shape[-1] = pad_len
feature_mask = torch.cat(
[feature_mask, torch.zeros(mask_pad_shape, dtype=feature_mask.dtype, device=feature_mask.device)],
dim=-1,
)
padded_input_features.append(feature)
padded_feature_attention_mask.append(feature_mask)
batch_input_features = torch.stack(padded_input_features)
batch_feature_attention_mask = torch.stack(padded_feature_attention_mask)
if not text_only and not audio_only and not video_only:
assert batch_pixel_values is not None and batch_image_grid_thw is not None
batch_pixel_values = torch.stack(batch_pixel_values) # (B, 256, 1536)
batch_image_grid_thw = torch.stack(batch_image_grid_thw) # (B, 3)
if video_only:
assert batch_pixel_values_videos is not None and batch_video_grid_thw is not None
assert batch_video_second_per_grid is not None
batch_pixel_values_videos = torch.cat(batch_pixel_values_videos, dim=0)
batch_video_grid_thw = torch.stack(batch_video_grid_thw)
batch_video_second_per_grid = torch.cat(batch_video_second_per_grid, dim=0)
# <modify_float> RTC推理同样按infer_autocast_dtype选择精度;float32时不启用autocast。
# 原逻辑:
# autocast_ctx = (
# torch.autocast("cuda", dtype=torch.bfloat16)
# if str(self.device).startswith("cuda")
# else contextlib.nullcontext()
# )
infer_dtype = getattr(self, "infer_autocast_dtype", torch.bfloat16)
if str(self.device).startswith("cuda") and infer_dtype in (torch.float16, torch.bfloat16):
autocast_ctx = torch.autocast("cuda", dtype=infer_dtype)
else:
autocast_ctx = contextlib.nullcontext()
with autocast_ctx:
# extract vision + language features
if audio_only:
output = self.vlm_model(
input_ids=batch_input_ids,
attention_mask=batch_attention_mask,
input_features=batch_input_features, # type: ignore
feature_attention_mask=batch_feature_attention_mask, # type: ignore
output_hidden_states=True,
return_dict=True
)
elif video_only:
output = self.vlm_model(
input_ids=batch_input_ids,
attention_mask=batch_attention_mask,
pixel_values_videos=batch_pixel_values_videos, # type: ignore
video_grid_thw=batch_video_grid_thw, # type: ignore
video_second_per_grid=batch_video_second_per_grid, # type: ignore
output_hidden_states=True,
return_dict=True,
)
elif text_only:
output = self.vlm_model(
input_ids=batch_input_ids,
attention_mask=batch_attention_mask,
output_hidden_states=True,
return_dict=True
)
else:
output = self.vlm_model(
input_ids=batch_input_ids,
attention_mask=batch_attention_mask,
pixel_values=batch_pixel_values, # type: ignore
image_grid_thw=batch_image_grid_thw, # type: ignore
output_hidden_states=True,
return_dict=True
)
vlm_hidden_states_ = output.hidden_states # len(vlm_hidden_states_) == 29
# use hidden states from the last layer
vlm_hidden_states = vlm_hidden_states_[-1] # shape (B, seq_len, D_h) shape(16, 80, 2048)
vlm_hidden_states = self._action_condition_hidden_states(
vlm_hidden_states.to(dtype=states.dtype),
input_ids=batch_input_ids,
attention_mask=batch_attention_mask,
) # shape (B, 1, seq_len, D_h) or subclass-specific latent tokens
# v18-style RTC: keep the complete 30-frame action path. The
# known prefix is overwritten with clean previous actions before
# every denoising step and receives timestep zero.
action_samples = torch.randn(
bsz, self.action_horizon, self.action_dim, device=self.device
)
prefix_mask = (
torch.arange(H, device=self.device)[None, :] < d
).expand(bsz, -1)
self.noise_scheduler.set_timesteps(num_inference_steps)
# self.noise_scheduler.timesteps: tensor([1000., 889., 778., 667., 556., 445., 334., 223., 112., 1.])
# self.noise_scheduler.sigmas: tensor([1.0000, 0.8890, 0.7780, 0.6670, 0.5560, 0.4450, 0.3340, 0.2230, 0.1120, 0.0010, 0.0000])
for timestep in self.noise_scheduler.timesteps:
batched_timestep = torch.where(
prefix_mask,
torch.zeros((), device=self.device, dtype=timestep.dtype),
timestep,
)
action_samples = torch.where(
prefix_mask[:, :, None], prev_actions, action_samples
)
model_pred = self.action_header(
hidden_states=None,
timestep=batched_timestep,
joint_attention_kwargs=dict(
action_hidden_embeds=action_samples,
known_action_prefix=None,
views=vlm_hidden_states, # (B,V,N,D)
obs=states, # (B,1,M)
traj2ds=traj2ds, # (B, C, 3, H, W)
progress=progress,
future_progress=future_progress,
),
return_dict=True,
).action
action_samples = self.noise_scheduler.step(
model_output=model_pred,
timestep=timestep,
sample=action_samples,
).prev_sample
# The old implementation overwrote the prefix at the start of each
# solver step. Re-apply it once after the final step as well so the
# returned overlap is exact rather than solver-perturbed.
action_samples = torch.where(
prefix_mask[:, :, None], prev_actions, action_samples
)
return action_samples.float()
def predict_action_with_rtc_flow(
self,
observations: List[List[Image.Image]], # B * List of PIL Image as [view1, view2]
states: torch.Tensor, # (B, Ts, Ds)
instructions: List[str], # (B,)
num_inference_steps: int,
traj2ds,
prev_actions: torch.Tensor = None, # (1, H, D)
inference_delay: int = 0,
execution_horizon: int = 0,
mask_schedule: str = "exponential",
guidance_weight: float = 5.0,
**kwargs: str
) -> np.ndarray:
with torch.no_grad():
## RTC related ##
H = self.action_horizon
assert prev_actions is not None and inference_delay > 0 and execution_horizon > 0, "prev_actions, inference_delay and execution_horizon must be provided"
assert prev_actions.shape[0] == 1
prev_actions = prev_actions.to(device=self.device, dtype=torch.float32)
# Create soft mask for inpainting
d = inference_delay
s = execution_horizon
# Validate constraint from paper: d ≤ s ≤ H - d
assert d <= s and s <= H - d, f"Constraint violated: d={d}, s={s}, H={H}. Need d ≤ s ≤ H-d"
# Create soft mask [H] according to Equation 5
mask = self._create_soft_mask(
H, d, s, schedule=mask_schedule, device=self.device
)
mask_expanded = mask.view(1, -1, 1).expand(1, self.action_horizon, self.action_dim)
## ##
bsz = states.shape[0]
progress = kwargs.get("progress", None)
future_progress = kwargs.get("future_progress", None)
batch_input_ids = []
batch_attention_mask = []
batch_pixel_values = []
batch_image_grid_thw = []
for observation, instruction in zip(observations, instructions):
messages = []
content = [{"type": "image", "image": img} for img in observation]
content.append({"type": "text", "text": instruction})
user_msg = {"role": "user", "content": content}
messages.append([user_msg])
texts = [
self.vlm_processor.apply_chat_template(
m, tokenize=False, add_generation_prompt=True
)
for m in messages
]
image_inputs, video_inputs = process_vision_info(messages, image_patch_size=16)
inputs = self.vlm_processor(
text=texts,
images=image_inputs,
videos=video_inputs,
padding=True,
return_tensors="pt",
).to(self.device)
# input_ids torch.Size([1, 80])
# attention_mask torch.Size([1, 80])
# pixel_values torch.Size([256, 1536])
# image_grid_thw torch.Size([1, 3])
batch_input_ids.append(inputs['input_ids'].squeeze(0))
batch_attention_mask.append(inputs['attention_mask'].squeeze(0))
batch_pixel_values.append(inputs['pixel_values'])
batch_image_grid_thw.append(inputs['image_grid_thw'].squeeze(0))
batch_input_ids = torch.stack(batch_input_ids) # (B, 80)
batch_attention_mask = torch.stack(batch_attention_mask) # (B, 80)
batch_pixel_values = torch.stack(batch_pixel_values) # (B, 256, 1536)
batch_image_grid_thw = torch.stack(batch_image_grid_thw) # (B, 3)
with torch.autocast("cuda", dtype=torch.bfloat16):
# extract vision + language features
output = self.vlm_model(
input_ids=batch_input_ids,
attention_mask=batch_attention_mask,
pixel_values=batch_pixel_values,
image_grid_thw=batch_image_grid_thw,
output_hidden_states=True,
return_dict=True
)
vlm_hidden_states_ = output.hidden_states # len(vlm_hidden_states_) == 29
# use hidden states from the last layer
vlm_hidden_states = vlm_hidden_states_[-1] # shape (B, seq_len, D_h) shape(16, 80, 2048)
vlm_hidden_states = vlm_hidden_states.unsqueeze(1) # shape (B, 1, seq_len, D_h) (16, 1, 80, 2048)
# generate action from noise
action_samples = torch.randn(
bsz, self.action_horizon, self.action_dim, device=self.device
)
# target_noise = action_samples.clone().detach()
self.noise_scheduler.set_timesteps(num_inference_steps)
# self.noise_scheduler.timesteps: tensor([1000., 889., 778., 667., 556., 445., 334., 223., 112., 1.])
# self.noise_scheduler.sigmas: tensor([1.0000, 0.8890, 0.7780, 0.6670, 0.5560, 0.4450, 0.3340, 0.2230, 0.1120, 0.0010, 0.0000])
# self.noise_scheduler.timesteps: tensor([1000., 889., 778., 667., 556., 445., 334., 223., 112., 1.])
# self.noise_scheduler.sigmas: tensor([1.0000, 0.8890, 0.7780, 0.6670, 0.5560, 0.4450, 0.3340, 0.2230, 0.1120, 0.0010, 0.0000])
for timestep in self.noise_scheduler.timesteps:
batched_timestep = timestep.expand(bsz).to(self.device).detach()
### pseudo inverse (with gradient):
# Enable gradient only for this step
with torch.enable_grad():
# Detach and enable gradient only for sample_actions in this step
action_samples_grad = action_samples.detach().requires_grad_(True)
model_pred = self.action_header(
hidden_states=None,
timestep=batched_timestep,
joint_attention_kwargs=dict(
action_hidden_embeds=action_samples_grad, # (B,Tp,Da)
views=vlm_hidden_states, # (B,V,N,D)
obs=states, # (B,1,M)
traj2ds=traj2ds, # (B, C, 3, H, W)
progress=progress,
future_progress=future_progress,
),
return_dict=True,
).action
# 1. 外推干净data X_0
tau = self.noise_scheduler.sigmas[self.noise_scheduler.index_for_timestep(timestep)].detach()
# DEBUG test inpainting
# noisy_prev_actions = (1 - tau) * prev_actions + tau * target_noise
# action_samples[:, :d, 14:28] = noisy_prev_actions[:, :d, 14:28]
# print("tau:", tau.item())
# print("timestep:", timestep)
# print("self.scheduler.sigmas:", self.scheduler.sigmas)
# print("self.scheduler.index_for_timestep(timestep):", self.scheduler.index_for_timestep(timestep))
pred_x0 = action_samples_grad - tau * model_pred
# pred_x0 = epsilon - model_pred
# print("pred_x0:", pred_x0)
# 2. 计算(Y - pred_X_0) * weights
error = (prev_actions - pred_x0.detach()) * mask_expanded
# print("error:", error)
# print("error.abs().mean():", error.abs().mean())
# 3. vjp
pinv_correction = torch.autograd.grad(
outputs=pred_x0, # [1,T,D]
inputs=action_samples_grad, # [1,T,D]
grad_outputs=error, # 匹配形状 [1,T,D]
retain_graph=False,
create_graph=False
)[0] # 输出 [1,T,D]
# print("pinv_correction:", pinv_correction)
# 4. 计算tau相关参数,得到修正项,修正v
inv_r2 = (tau**2 + (1 - tau) ** 2) / (tau**2)
c = torch.nan_to_num(tau / (1 - tau), posinf=guidance_weight)
g = torch.clamp(c * inv_r2, max=guidance_weight)
# print("c * inv_r2:", c * inv_r2)
# print("model_pred_before_correction:", model_pred)
model_pred = model_pred - g * pinv_correction
# print("pinv_correction.abs().mean():", pinv_correction.abs().mean())
###
action_samples = self.noise_scheduler.step(
model_output=model_pred, timestep=timestep, sample=action_samples # type: ignore
).prev_sample
# DEBUG test inpainting
# if i == len(self.noise_scheduler.timesteps) - 1:
# action_samples[:, :d, 14:28] = prev_actions[:, :d, 14:28]
return action_samples
def predict_action_with_rtc_flow_naive_inpaint(
self,
observations: List[List[Image.Image]], # B * List of PIL Image as [view1, view2]
states: torch.Tensor, # (B, Ts, Ds)
instructions: List[str], # (B,)
num_inference_steps: int,
traj2ds,
prev_actions: torch.Tensor = None, # (1, H, D)
inference_delay: int = 0,
execution_horizon: int = 0,
mask_schedule: str = "exponential",
guidance_weight: float = 5.0,
**kwargs: str
) -> np.ndarray:
with torch.no_grad():
## RTC related ##
H = self.action_horizon
assert prev_actions is not None and inference_delay > 0 and execution_horizon > 0, "prev_actions, inference_delay and execution_horizon must be provided"
assert prev_actions.shape[0] == 1
prev_actions = prev_actions.to(device=self.device, dtype=torch.float32)
# Create soft mask for inpainting
d = inference_delay
s = execution_horizon
# Validate constraint from paper: d ≤ s ≤ H - d
assert d <= s and s <= H - d, f"Constraint violated: d={d}, s={s}, H={H}. Need d ≤ s ≤ H-d"
# Create soft mask [H] according to Equation 5
mask = self._create_soft_mask(
H, d, s, schedule=mask_schedule, device=self.device
)
mask_expanded = mask.view(1, -1, 1).expand(1, self.action_horizon, self.action_dim)
## ##
bsz = states.shape[0]
progress = kwargs.get("progress", None)
future_progress = kwargs.get("future_progress", None)
batch_input_ids = []
batch_attention_mask = []
batch_pixel_values = []
batch_image_grid_thw = []
for observation, instruction in zip(observations, instructions):
messages = []
content = [{"type": "image", "image": img} for img in observation]
content.append({"type": "text", "text": instruction})
user_msg = {"role": "user", "content": content}
messages.append([user_msg])
texts = [
self.vlm_processor.apply_chat_template(
m, tokenize=False, add_generation_prompt=True
)
for m in messages
]
image_inputs, video_inputs = process_vision_info(messages, image_patch_size=16)
inputs = self.vlm_processor(
text=texts,
images=image_inputs,
videos=video_inputs,
padding=True,
return_tensors="pt",
).to(self.device)
# input_ids torch.Size([1, 80])
# attention_mask torch.Size([1, 80])
# pixel_values torch.Size([256, 1536])
# image_grid_thw torch.Size([1, 3])
batch_input_ids.append(inputs['input_ids'].squeeze(0))
batch_attention_mask.append(inputs['attention_mask'].squeeze(0))
batch_pixel_values.append(inputs['pixel_values'])
batch_image_grid_thw.append(inputs['image_grid_thw'].squeeze(0))
batch_input_ids = torch.stack(batch_input_ids) # (B, 80)
batch_attention_mask = torch.stack(batch_attention_mask) # (B, 80)
batch_pixel_values = torch.stack(batch_pixel_values) # (B, 256, 1536)
batch_image_grid_thw = torch.stack(batch_image_grid_thw) # (B, 3)
with torch.autocast("cuda", dtype=torch.bfloat16):
# extract vision + language features
output = self.vlm_model(
input_ids=batch_input_ids,
attention_mask=batch_attention_mask,
pixel_values=batch_pixel_values,
image_grid_thw=batch_image_grid_thw,
output_hidden_states=True,
return_dict=True
)
vlm_hidden_states_ = output.hidden_states # len(vlm_hidden_states_) == 29
# use hidden states from the last layer
vlm_hidden_states = vlm_hidden_states_[-1] # shape (B, seq_len, D_h) shape(16, 80, 2048)
vlm_hidden_states = vlm_hidden_states.unsqueeze(1) # shape (B, 1, seq_len, D_h) (16, 1, 80, 2048)
# generate action from noise
action_samples = torch.randn(
bsz, self.action_horizon, self.action_dim, device=self.device
)
self.noise_scheduler.set_timesteps(num_inference_steps)
# Fixed noise for consistency in inpainting
# Use the same noise as action_samples to ensure the whole sequence starts from the same latent x_1
target_noise = action_samples.clone()
for i, timestep in enumerate(self.noise_scheduler.timesteps):
batched_timestep = timestep.expand(bsz).to(self.device).detach()
### Naive inpainting (Repaint method):
# Step 1: Add noise to prev_actions matching current timestep's noise level
# This ensures consistency in noise levels across the entire action sequence
tau = self.noise_scheduler.sigmas[self.noise_scheduler.index_for_timestep(timestep)]
noisy_prev_actions = (1 - tau) * prev_actions + tau * target_noise
# Step 2: Replace first d positions with noisy prev_actions (hard mask)
action_samples[:, :d, :] = noisy_prev_actions[:, :d, :]
# Step 3: Model prediction
model_pred = self.action_header(
hidden_states=None,
timestep=batched_timestep,
joint_attention_kwargs=dict(
action_hidden_embeds=action_samples, # (B,Tp,Da)
views=vlm_hidden_states, # (B,V,N,D)
obs=states, # (B,1,M)
traj2ds=traj2ds, # (B, C, 3, H, W)
progress=progress,
future_progress=future_progress,
),
return_dict=True,
).action
# Step 4: Denoise one step (updates all positions including first d)
action_samples = self.noise_scheduler.step(
model_output=model_pred, timestep=timestep, sample=action_samples # type: ignore
).prev_sample
# Step 5: Re-inpaint with noise level matching the denoised state
# TODO: do this or not??? boqian
if i == len(self.noise_scheduler.timesteps) - 1:
# Last step: use clean prev_actions (tau = 0)
action_samples[:, :d, :] = prev_actions[:, :d, :]
return action_samples
# TODO-yth: 重要
def predict_action_with_rtc_flow_based_gradient(
self,
observations: List[List[Image.Image]], # B * List of PIL Image as [view1, view2]
states: torch.Tensor, # (B, Ts, Ds)
instructions: List[str], # (B,)
num_inference_steps: int,
traj2ds,
prev_actions: torch.Tensor | None = None, # (1, H, D)
inference_delay: int = 0,
execution_horizon: int = 0,
mask_schedule: str = "exponential",
guidance_weight: float = 5.0,
**kwargs: str
) -> np.ndarray:
with torch.no_grad():
## RTC related ##
H = self.action_horizon
assert prev_actions is not None and inference_delay > 0 and execution_horizon > 0, "prev_actions, inference_delay and execution_horizon must be provided"
assert prev_actions.shape[0] == 1
prev_actions = prev_actions.to(device=self.device, dtype=torch.float32)
# Create soft mask for inpainting
d = inference_delay
s = execution_horizon
# Validate constraint from paper: d ≤ s ≤ H - d
assert d <= s and s <= H - d, f"Constraint violated: d={d}, s={s}, H={H}. Need d ≤ s ≤ H-d"
# Create soft mask [H] according to Equation 5
mask = self._create_soft_mask(
H, d, s, schedule=mask_schedule, device=self.device
)
mask_expanded = mask.view(1, -1, 1).expand(1, self.action_horizon, self.action_dim)
## ##
bsz = states.shape[0]
progress = kwargs.get("progress", None)
future_progress = kwargs.get("future_progress", None)
batch_input_ids = []
batch_attention_mask = []
batch_pixel_values = []
batch_image_grid_thw = []
for observation, instruction in zip(observations, instructions):
messages = []
content = [{"type": "image", "image": img} for img in observation]
content.append({"type": "text", "text": instruction})
user_msg = {"role": "user", "content": content}
messages.append([user_msg])
texts = [
self.vlm_processor.apply_chat_template(
m, tokenize=False, add_generation_prompt=True
)
for m in messages
]
image_inputs, video_inputs = process_vision_info(messages, image_patch_size=16)
inputs = self.vlm_processor(
text=texts,
images=image_inputs,
videos=video_inputs,
padding=True,
return_tensors="pt",
).to(self.device)
# input_ids torch.Size([1, 80])
# attention_mask torch.Size([1, 80])
# pixel_values torch.Size([256, 1536])
# image_grid_thw torch.Size([1, 3])
batch_input_ids.append(inputs['input_ids'].squeeze(0))
batch_attention_mask.append(inputs['attention_mask'].squeeze(0))
batch_pixel_values.append(inputs['pixel_values'])
batch_image_grid_thw.append(inputs['image_grid_thw'].squeeze(0))
batch_input_ids = torch.stack(batch_input_ids) # (B, 80)
batch_attention_mask = torch.stack(batch_attention_mask) # (B, 80)
batch_pixel_values = torch.stack(batch_pixel_values) # (B, 256, 1536)
batch_image_grid_thw = torch.stack(batch_image_grid_thw) # (B, 3)
with torch.autocast("cuda", dtype=torch.bfloat16):
# extract vision + language features
output = self.vlm_model(
input_ids=batch_input_ids,
attention_mask=batch_attention_mask,
pixel_values=batch_pixel_values,
image_grid_thw=batch_image_grid_thw,
output_hidden_states=True,
return_dict=True
)
vlm_hidden_states_ = output.hidden_states # len(vlm_hidden_states_) == 29
# use hidden states from the last layer
vlm_hidden_states = vlm_hidden_states_[-1] # shape (B, seq_len, D_h) shape(16, 80, 2048)
vlm_hidden_states = vlm_hidden_states.unsqueeze(1) # shape (B, 1, seq_len, D_h) (16, 1, 80, 2048)
# 初始化 action_samples
action_samples = torch.randn(
bsz, self.action_horizon, self.action_dim, device=self.device
)
# 使用同样的 noise 构建目标,保证一致性
target_noise = action_samples.clone()
self.noise_scheduler.set_timesteps(num_inference_steps)
for i, timestep in enumerate(self.noise_scheduler.timesteps):
# 1. 计算当前步的目标 (Noisy Target)
# ---------------------------------------------------------
tau = self.noise_scheduler.sigmas[self.noise_scheduler.index_for_timestep(timestep)]
# 这是一个移动的目标:随着 tau 变小,它越来越接近真实的 prev_actions
noisy_prev_actions = (1 - tau) * prev_actions + tau * target_noise
# 2. 基于梯度的 Guidance (核心部分)
# ---------------------------------------------------------
# 我们希望 action_samples 在 mask 区域尽可能接近 noisy_prev_actions
# 通过梯度下降来修改 action_samples
# 定义 Guidance 的强度衰减:
# 早期(噪声大)需要强引导来定型,晚期(噪声小)减弱引导以允许模型平滑过渡
current_scale = guidance_weight * (tau ** 0.5) # 这里的衰减策略可以调整,例如用 linear
# 临时开启梯度计算
with torch.enable_grad():
# Detach 并开启梯度,创建一个新的叶子节点
x_in = action_samples.detach().requires_grad_(True)
# 计算 Loss: 只在 mask 有值的地方计算 MSE
# 这里的 mask_expanded 充当了权重的角色
# 重点:只计算前 s 步的差异 (inference_delay + execution_horizon)
diff = (x_in - noisy_prev_actions)
# 使用 mask 加权平方误差
# 形状: (B, H, D) * (1, H, D)
loss = (diff ** 2 * mask_expanded).sum()
# 计算梯度
grad = torch.autograd.grad(loss, x_in)[0]
# 更新 action_samples (朝着 loss 减小的方向)
# 注意:这里我们是在 Latent 空间进行微调
print("grad[:, :, 0:3].abs().mean():", grad[:, :, 0:3].abs().mean())
print("grad[:, :, 15:17].abs().mean():", grad[:, :, 15:17].abs().mean())
action_samples = action_samples - current_scale * grad
# 3. 常规的 Flow Matching / Diffusion Step
# ---------------------------------------------------------
batched_timestep = timestep.expand(bsz).to(self.device).detach()
model_pred = self.action_header(
hidden_states=None,
timestep=batched_timestep,
joint_attention_kwargs=dict(
action_hidden_embeds=action_samples,
views=vlm_hidden_states,
obs=states,
traj2ds=traj2ds,
progress=progress,
future_progress=future_progress,
),
return_dict=True,
).action
# 4. Denoise one step
action_samples = self.noise_scheduler.step(
model_output=model_pred, timestep=timestep, sample=action_samples
).prev_sample
# 5. (可选) Soft Blending 作为最后的保险
# 如果单纯靠梯度还是压不住,可以在最后一步稍微融合一下
# 但如果 guidance_weight 足够,通常不需要这步
# if i == len(self.noise_scheduler.timesteps) - 1:
# # 最后一步不再有噪声,直接做一次软混合确保开头完全吻合
# action_samples = prev_actions * mask_expanded + action_samples * (1 - mask_expanded)
return action_samples
def _create_soft_mask(self, H, d, s, schedule="exponential", device="cpu"):
"""
Create soft mask for RTC inpainting (Equation 5 in paper).
Paper: "Real-Time Execution of Action Chunking Flow Policies"
Figure 3 and Equation 5:
W_i = { 1 if i < d (frozen region)
{ c_i * (e^(c_i) - 1)/(e - 1) if d ≤ i < H - s (intermediate region)
{ 0 if i ≥ H - s (free region)
where c_i = (H - s - i)/(H - s - d + 1)
Args:
H: Prediction horizon (total sequence length)
d: Inference delay (number of frozen steps, already executed)
s: Execution horizon (non-overlapping steps at end).
schedule: 'exponential' (paper default), 'linear', 'hard', or 'simple'
Returns:
mask: [H] tensor with values in [0, 1]
Guidance weights for each timestep
"""
mask = torch.zeros(H, device=device)
if schedule == "hard":
# Hard mask: 1 for frozen, 0 for rest
mask[:d] = 1.0
elif schedule == "linear":
# Linear decay
mask[:d] = 1.0
# Three-region version (full paper)
overlap_end = H - s
if d < overlap_end:
# Linear decay from 1 to 0 in intermediate region
indices = torch.arange(d, overlap_end, device=device).float()
mask[d:overlap_end] = 1.0 - (indices - d) / (overlap_end - d)
# mask[overlap_end:] remains 0
elif schedule == "exponential":
# Exponential decay (paper default, Equation 5)
mask[:d] = 1.0
# Three-region version (full paper implementation)
overlap_end = H - s
if d < overlap_end:
indices = torch.arange(d, overlap_end, device=device).float()
# c_i = (H - s - i) / (H - s - d + 1)
c_i = (overlap_end - indices) / (overlap_end - d + 1)
# W_i = c_i * (e^(c_i) - 1) / (e - 1)
e = torch.tensor(torch.e, device=device)
mask[d:overlap_end] = c_i * (torch.exp(c_i) - 1) / (e - 1)
# mask[overlap_end:] remains 0
elif schedule == "simple":
# Simplified exponential (for backward compatibility)
mask[:d] = 1.0
if d < H:
indices = torch.arange(d, H, device=device).float()
mask[d:] = torch.exp(-5.0 * (indices - d) / (H - d))
else:
raise ValueError(f"Unknown mask schedule: {schedule}")
return mask
@torch.inference_mode()
def chat(self):
...
import os
import json
import copy
import contextlib
from tqdm import tqdm
from typing import List, Optional, Tuple, Union, Dict, Any
from typing import Any, Dict, List, Optional, Tuple, Union
from typing_extensions import Self
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from PIL import Image
from dataclasses import dataclass
from diffusers.utils.outputs import BaseOutput
from diffusers.configuration_utils import ConfigMixin, FrozenDict, register_to_config
# from diffusers.utils.torch_utils import maybe_allow_in_graph
from diffusers.models.modeling_utils import ModelMixin
# from diffusers.models.transformers.transformer_sd3 import SD3Transformer2DModel
from diffusers.models.embeddings import CombinedTimestepTextProjEmbeddings
# from diffusers.models.normalization import AdaLayerNorm, AdaLayerNormZero, AdaLayerNormContinuous
from diffusers.models.embeddings import CombinedTimestepLabelEmbeddings
from diffusers.models.normalization import FP32LayerNorm
from diffusers.models.attention_processor import Attention, JointAttnProcessor2_0
from diffusers.models.attention import FeedForward, _chunked_feed_forward
# from dit_policy.data4robotics.models.diffusion import _TimeNetwork
from psi.config.config import LaunchConfig
from transformers import (
Qwen3VLForConditionalGeneration,
Qwen2_5OmniThinkerForConditionalGeneration,
Qwen2_5OmniProcessor,
AutoConfig,
AutoProcessor,
Qwen2TokenizerFast,
Qwen3VLProcessor,
)
from qwen_vl_utils import process_vision_info
from psi.utils import initialize_overwatch, count_parameters
# from InternVLA.model.modules.action_model.DiT_modules.models import DiT
# from InternVLA.model.modules.projector.QFormer import CrossAttentionBlock
overwatch = initialize_overwatch(__name__)
# from psi.config.model import ModelConfig, InternVLA_M1_ModelConfig
# from psi.learn.vlms import _QWen_VL_Interface
from transformers.models.siglip import SiglipModel
from transformers.models.dinov2 import Dinov2Model
# from vlt.transformers.vla import ActionTransformerModel
from diffusers.utils import logging
logger = logging.get_logger(__name__) # pylint: disable=invalid-name FIXME why this not works any more?
QWEN3VL_VARIANT = "Qwen/Qwen3-VL-2B-Instruct"
OMNI_DEFAULT_SYSTEM_PROMPT = (
"You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, "
"capable of perceiving auditory and visual inputs, as well as generating text and speech."
)
def _validate_rtc_inference_contract(
*,
action_horizon: int,
action_exec_horizon: int,
inference_delay: int,
max_delay: int,
) -> int:
"""Return the configured RTC overlap or fail on train/deploy mismatch."""
horizon = int(action_horizon)
execute = int(action_exec_horizon)
delay = int(inference_delay)
delay_limit = int(max_delay)
if not 0 < execute < horizon:
raise ValueError(
"RTC requires 0 < action_exec_horizon < action_horizon, "
f"got execute={execute}, horizon={horizon}"
)
expected_overlap = horizon - execute
if delay != expected_overlap:
raise ValueError(
"RTC inference_delay must equal the overlap used by training and "
"deployment: "
f"got inference_delay={delay}, expected={expected_overlap} "
f"(action_horizon={horizon}, action_exec_horizon={execute})"
)
if expected_overlap >= delay_limit:
raise ValueError(
"RTC overlap must be smaller than max_delay, "
f"got overlap={expected_overlap}, max_delay={delay_limit}"
)
return expected_overlap
class HumanFoundationModelOutput(BaseOutput):
action: "torch.Tensor" # noqa: F821
def to_tuple(self):
return (None, self.action)