Tinghong-Ye22's picture
Add files using upload-large-folder tool
c4f8e4a verified
Raw History Blame Contribute Delete
22.9 kB
"""
InternVLA M1 framework:
Vision-Language-Action diffusion model integrating:
- Qwen2.5 vision-language backbone
- Layer-wise QFormer aggregation
- DINO multi-view visual encoder
- DiT diffusion head for future action sequence prediction
Primary goal: predict continuous future actions conditioned on multi-view images + instruction.
"""
from typing import List
from tqdm import tqdm
from typing import List, Optional, Tuple
import torch
import torch.nn as nn
import numpy as np
from PIL import Image
from qwen_vl_utils import process_vision_info
from InternVLA.training.trainer_utils import initialize_overwatch
logger = initialize_overwatch(__name__)
# HuggingFace Default / LLaMa-2 IGNORE_INDEX (for labels)
IGNORE_INDEX = -100
from InternVLA.model.framework.base_framework import baseframework
from InternVLA.model.modules.vlm.QWen2_5 import get_qwen2_5_interface
from InternVLA.model.modules.projector.QFormer import get_layerwise_qformer
from InternVLA.model.modules.action_model.DiTActionHeader import get_action_model
from InternVLA.model.modules.dino_model.dino import get_dino_model
from InternVLA.training.trainer_utils.metrics import resize_images
class InternVLA_M1(baseframework):
"""
Multimodal vision-language-action model.
Components:
- Qwen2.5 VL interface for fused language/vision token embeddings
- Layer-wise QFormer for multi-layer feature aggregation
- DINO encoder for dense multi-view spatial tokens
- DiT diffusion head for future action sequence modeling
Focus: Predict future continuous actions conditioned on images + instruction.
"""
def __init__(
self,
config: Optional[dict] = None,
**kwargs,
) -> None:
"""
Construct all submodules and cache key configuration values.
Args:
config: Hierarchical configuration (OmegaConf/dict) containing framework + trainer sections.
**kwargs: Reserved for future overrides (unused).
"""
super().__init__()
self.config = config
self.qwen_vl_interface = get_qwen2_5_interface(config=self.config)
self.layer_qformer = get_layerwise_qformer(config=self.config)
self.action_model = get_action_model(config=self.config)
self.dino_encoder = get_dino_model(backone_name=getattr(self.config.framework.dino, "dino_backbone", "dinov2_vits14"))
self.dino_pro = nn.Linear(
in_features=self.dino_encoder.num_channels,
out_features=self.qwen_vl_interface.model.config.hidden_size
)
self.future_action_window_size = config.framework.action_model.future_action_window_size
self.past_action_window_size = config.framework.action_model.past_action_window_size
def forward(
self,
examples: List[dict] = None,
**kwargs,
) -> Tuple:
"""
Forward pass for training (diffusion objective).
Flow:
1. Build QwenVL inputs (images + instruction tokens)
2. Extract hidden states from configured layer range
3. Encode images with DINO, flatten multi-view tokens and project
4. Concatenate per-layer language tokens with visual tokens
5. Fuse via layer-wise QFormer -> action condition embeddings
6. Prepare repeated future action windows (for diffusion efficiency)
7. Predict noise and compute diffusion loss
Args:
examples: List[dict], each dict requires:
- image: List[PIL.Image] (multi-view)
- lang: str instruction
- action: np.ndarray or list shaped [T, action_dim]
**kwargs: Reserved.
Returns:
dict:
action_loss (torch.Tensor): Scalar diffusion noise prediction loss.
"""
batch_images = [example["image"] for example in examples] # [B,[PLT]]
instructions = [example["lang"] for example in examples] # [B, str]
actions = [example["action"] for example in examples] # label [B, len, 7]
# Step 1: QWenVL input format
qwen_inputs = self.qwen_vl_interface.build_qwenvl_inputs(images=batch_images, instructions=instructions)
with torch.autocast("cuda", dtype=torch.bfloat16):
qwenvl_outputs = self.qwen_vl_interface(
**qwen_inputs,
output_attentions=False,
output_hidden_states=True,
return_dict=True,
)
pass
# Step 2: DINO Forward
image_tensors = self.dino_encoder.prepare_dino_input(batch_images) #
B = len(batch_images)
dino_features = self.dino_encoder(image_tensors) # DINO output is [B*num_view, token, dim]
dino_encoded_features = dino_features.reshape(B, -1, dino_features.shape[-1]) # [B, num_view * token, dim]
dino_encoded_features = self.dino_pro(dino_encoded_features) # [B, num_view * token, hidden_size]
# Step 3: aggregation condition for Action expert
start_layer = self.config.framework.layer_qformer.qformer_start_layer
end_layer = self.config.framework.layer_qformer.qformer_end_layer
condition_features = qwenvl_outputs.hidden_states[start_layer:end_layer]
cat_conditions = []
for layer_index in range(len(condition_features)):
layer_features = condition_features[layer_index] # [B, n_qformer_token, D]
layer_features = torch.cat(
[layer_features, dino_encoded_features], dim=1
) # [B, n_qformer_token + num_view * token, D]
cat_conditions.append(layer_features)
action_condition = self.layer_qformer(cat_conditions) # [B, 64, D_action]
# Step 4: Action Expert Forward and Loss
with torch.autocast("cuda", dtype=torch.float32):
# here is a tips to accelerate training speed, by repeating each sample for several times @ref to CogACT
actions = torch.tensor(np.array(actions), device=action_condition.device) # [B, chunk, 7]
actions_future = actions[:, -(self.future_action_window_size + 1) :, :]
# tips: Repeat 'actions' 'repeated_diffusion_steps' times, resulting in [repeated_diffusion_steps*B, T, D]
repeated_diffusion_steps = (
self.config.trainer.get("repeated_diffusion_steps", 4) if self.config and self.config.trainer else 4
)
actions_repeated = actions_future.repeat(repeated_diffusion_steps, 1, 1)
action_condition = action_condition.repeat(
repeated_diffusion_steps, 1, 1
) # [repeated_diffusion_steps*B, T, D_action]
# DiT noise add and predict
noise_pred, noise, timestep = self.action_model(actions_repeated, action_condition)
# perdition loss
action_loss = self.action_model.loss(noise_pred, noise)
return {"action_loss": action_loss}
@torch.inference_mode()
def predict_action(
self,
batch_images: List[List[Image.Image]], # B * List of PIL Image as [view1, view2]
instructions: List[str],
cfg_scale: float = 1.5,
use_ddim: bool = True,
num_ddim_steps: int = 5,
resize_image = [224, 224],
**kwargs: str,
) -> np.ndarray:
"""
Inference: generate future normalized action sequence via diffusion sampling.
Steps:
1. Resize images to training resolution (if specified)
2. Encode with QwenVL (hidden states retained)
3. Extract DINO tokens and project to vlm hidden size
4. Build multi-layer fused QwenVL and DINO features via QFormer
5. Run diffusion sampling (DDIM optional, CFG optional)
6. Return normalized action trajectory
Args:
batch_images: List of samples; each sample is List[PIL.Image] (multi-view).
instructions: List[str] natural language task instructions.
cfg_scale: >1 enables classifier-free guidance (scales conditional vs unconditional).
use_ddim: Whether to use DDIM deterministic sampling.
num_ddim_steps: Number of DDIM steps if enabled.
**kwargs: Reserved.
Returns:
dict:
normalized_actions (np.ndarray): Shape [B, T, action_dim], diffusion-sampled normalized actions.
"""
# align obs and lang # is policy's duty to make sure the image size?
train_obs_image_size = getattr(self.config.datasets.vla_data, "image_size", None)
print("resize image from:", batch_images[0][0].size)
if train_obs_image_size:
batch_images = resize_images(batch_images, target_size=train_obs_image_size)
print("resize image to:", batch_images[0][0].size)
instructions = [instruction.lower() for instruction in instructions]
inferface_inputs = self.qwen_vl_interface.build_qwenvl_inputs(images=batch_images, instructions=instructions)
qwen_inputs = inferface_inputs
with torch.autocast("cuda", dtype=torch.bfloat16):
qwenvl_outputs = self.qwen_vl_interface(
**qwen_inputs,
output_hidden_states=True,
return_dict=True,
)
B = len(batch_images) # dino don't have smart resize in processing
image_tensors = self.dino_encoder.prepare_dino_input(batch_images)
dino_features = self.dino_encoder(image_tensors)
B = len(batch_images)
dino_encoded_features = dino_features.reshape(B, -1, dino_features.shape[-1]) # [B, num_view * token, dim]
dino_encoded_features = self.dino_pro(dino_encoded_features) # [B, 256, D]
with torch.autocast("cuda", dtype=torch.bfloat16):
start_layer = self.config.framework.layer_qformer.qformer_start_layer
end_layer = self.config.framework.layer_qformer.qformer_end_layer
condition_features = qwenvl_outputs.hidden_states[start_layer:end_layer] # final transformer block
cat_conditions = []
for layer_index in range(len(condition_features)):
layer_features = condition_features[layer_index] # [B, n_qformer_token, D]
layer_features = torch.cat(
[layer_features, dino_encoded_features], dim=1
) # [B, n_qformer_token + num_view * token, D]
cat_conditions.append(layer_features)
action_condition_feature = self.layer_qformer(cat_conditions) # [B, 64, D_action]
using_cfg = cfg_scale > 1.0
model_dtype = next(self.action_model.net.parameters()).dtype
B = action_condition_feature.shape[0]
# Sample random noise
noise = torch.randn(
B,
self.future_action_window_size + 1,
self.action_model.in_channels,
device=action_condition_feature.device,
).to(
model_dtype
) # [B, T, D]
# Setup classifier-free guidance:
if using_cfg:
noise = torch.cat([noise, noise], 0) # [2,16,7]
uncondition = self.action_model.net.z_embedder.uncondition # [64, 768]
uncondition_shape = uncondition.shape
uncondition = uncondition.unsqueeze(0) # [1, 64, D]
uncondition = uncondition.expand(
B, uncondition_shape[0], uncondition_shape[1]
) # [B, n_qformer_token, D]
z = torch.cat([action_condition_feature, uncondition], 0) # [2, 64, 768]
cfg_scale = cfg_scale
model_kwargs = dict(z=z, cfg_scale=cfg_scale)
sample_fn = self.action_model.net.forward_with_cfg
else:
model_kwargs = dict(z=action_condition_feature)
sample_fn = self.action_model.net.forward
# DDIM Sampling
if use_ddim and num_ddim_steps is not None:
if self.action_model.ddim_diffusion is None:
self.action_model.create_ddim(ddim_step=num_ddim_steps)
samples = self.action_model.ddim_diffusion.ddim_sample_loop(
sample_fn,
noise.shape,
noise,
clip_denoised=False,
model_kwargs=model_kwargs,
progress=False,
device=action_condition_feature.device,
eta=0.0,
)
if using_cfg:
samples, _ = samples.chunk(2, dim=0) # Remove null class samples
normalized_actions = samples.cpu().numpy()
raw_actions = None
return {"normalized_actions": normalized_actions} # [B, T, action_dim]
# @torch.inference_mode()
def predict_action_rtc(
self,
batch_images: List[List[Image.Image]], # B * List of PIL Image as [view1, view2]
instructions: List[str],
cfg_scale: float = 1.5,
use_ddim: bool = True,
num_ddim_steps: int = 5,
resize_image = [224, 224],
prev_actions: np.ndarray = None,
inference_delay: int = 0,
execution_horizon: int = 0,
mask_schedule: str = "exponential",
guidance_weight: float = 5.0,
**kwargs: str,
) -> np.ndarray:
"""
Inference: generate future normalized action sequence via diffusion sampling.
Steps:
1. Resize images to training resolution (if specified)
2. Encode with QwenVL (hidden states retained)
3. Extract DINO tokens and project to vlm hidden size
4. Build multi-layer fused QwenVL and DINO features via QFormer
5. Run diffusion sampling (DDIM optional, CFG optional)
6. Return normalized action trajectory
Args:
batch_images: List of samples; each sample is List[PIL.Image] (multi-view).
instructions: List[str] natural language task instructions.
cfg_scale: >1 enables classifier-free guidance (scales conditional vs unconditional).
use_ddim: Whether to use DDIM deterministic sampling.
num_ddim_steps: Number of DDIM steps if enabled.
prev_actions: Previous action chunk [B, H, D] (from last inference)
inference_delay: Number of frozen steps d (already executed during inference)
execution_horizon: Execution horizon s (non-overlapping steps at end).
mask_schedule: Soft mask schedule - 'exponential' (default), 'linear', or 'hard'
guidance_weight: Max guidance weight β (default=5.0, as in paper)
**kwargs: Reserved.
Returns:
dict:
normalized_actions (np.ndarray): Shape [B, T, action_dim], diffusion-sampled normalized actions.
"""
# align obs and lang # is policy's duty to make sure the image size?
train_obs_image_size = getattr(self.config.datasets.vla_data, "image_size", None)
print("resize image from:", batch_images[0][0].size)
if train_obs_image_size:
batch_images = resize_images(batch_images, target_size=train_obs_image_size)
print("resize image to:", batch_images[0][0].size)
instructions = [instruction.lower() for instruction in instructions]
inferface_inputs = self.qwen_vl_interface.build_qwenvl_inputs(images=batch_images, instructions=instructions)
qwen_inputs = inferface_inputs
with torch.autocast("cuda", dtype=torch.bfloat16):
qwenvl_outputs = self.qwen_vl_interface(
**qwen_inputs,
output_hidden_states=True,
return_dict=True,
)
B = len(batch_images) # dino don't have smart resize in processing
image_tensors = self.dino_encoder.prepare_dino_input(batch_images)
dino_features = self.dino_encoder(image_tensors)
B = len(batch_images)
dino_encoded_features = dino_features.reshape(B, -1, dino_features.shape[-1]) # [B, num_view * token, dim]
dino_encoded_features = self.dino_pro(dino_encoded_features) # [B, 256, D]
with torch.autocast("cuda", dtype=torch.bfloat16):
start_layer = self.config.framework.layer_qformer.qformer_start_layer
end_layer = self.config.framework.layer_qformer.qformer_end_layer
condition_features = qwenvl_outputs.hidden_states[start_layer:end_layer]
cat_conditions = []
for layer_index in range(len(condition_features)):
layer_features = condition_features[layer_index] # [B, n_qformer_token, D]
layer_features = torch.cat(
[layer_features, dino_encoded_features], dim=1
) # [B, n_qformer_token + num_view * token, D]
cat_conditions.append(layer_features)
action_condition_feature = self.layer_qformer(cat_conditions) # [B, 64, D_action]
using_cfg = cfg_scale > 1.0
model_dtype = next(self.action_model.net.parameters()).dtype
B = action_condition_feature.shape[0]
# Sample random noise
noise = torch.randn(
B,
self.future_action_window_size + 1,
self.action_model.in_channels,
device=action_condition_feature.device,
).to(
model_dtype
) # [B, T, D]
# Setup classifier-free guidance:
if using_cfg:
noise = torch.cat([noise, noise], 0) # [2,16,7]
uncondition = self.action_model.net.z_embedder.uncondition # [64, 768]
uncondition_shape = uncondition.shape
uncondition = uncondition.unsqueeze(0) # [1, 64, D]
uncondition = uncondition.expand(
B, uncondition_shape[0], uncondition_shape[1]
) # [B, n_qformer_token, D]
z = torch.cat([action_condition_feature, uncondition], 0) # [2, 64, 768]
cfg_scale = cfg_scale
model_kwargs = dict(z=z, cfg_scale=cfg_scale)
sample_fn = self.action_model.net.forward_with_cfg
else:
model_kwargs = dict(z=action_condition_feature)
sample_fn = self.action_model.net.forward
# DDIM Sampling
if use_ddim and num_ddim_steps is not None:
if self.action_model.ddim_diffusion is None:
self.action_model.create_ddim(ddim_step=num_ddim_steps)
samples = self.action_model.ddim_diffusion.ddim_sample_loop_rtc(
sample_fn,
noise.shape,
noise,
prev_actions=torch.tensor(prev_actions, device=noise.device),
inference_delay=inference_delay,
execution_horizon=execution_horizon,
mask_schedule=mask_schedule,
guidance_weight=guidance_weight,
clip_denoised=False,
model_kwargs=model_kwargs,
progress=False,
device=action_condition_feature.device,
eta=0.0,
)
if using_cfg:
samples, _ = samples.chunk(2, dim=0) # Remove null class samples
normalized_actions = samples.cpu().numpy()
raw_actions = None
return {"normalized_actions": normalized_actions} # [B, T, action_dim]
@torch.inference_mode()
def chat_with_M1(
self,
image: Image.Image,
text: str,
max_new_tokens: int = 128,
device: Optional[str] = "cuda",
) -> List[str]:
processor = getattr(self.qwen_vl_interface, "processor", None)
model = getattr(self.qwen_vl_interface, "model", None)
# if processor is None or model is None:
# raise RuntimeError("qwen_vl_interface 缺少 processor 或 model。")
messages0 = [
{
"role": "user",
"content": [
{
"type": "image",
"image": image,
},
{"type": "text", "text": text},
],
}
]
messages = [messages0]
# text info
texts = [processor.apply_chat_template(msg, tokenize=False, add_generation_prompt=True) for msg in messages]
# visual info
image_inputs, video_inputs = process_vision_info(messages)
# tokenizer
inputs = processor(
text=texts,
images=image_inputs,
videos=video_inputs,
padding=True,
return_tensors="pt",
).to(device)
model.eval()
generated_ids = model.generate(**inputs, max_new_tokens=max_new_tokens)
generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]
outputs = processor.batch_decode(
generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
)
return outputs
def build_model_framework(config: dict = {}) -> InternVLA_M1:
"""
Factory helper to build InternVLA_M1 with provided config.
Args:
config: Dict or OmegaConf containing framework/component settings.
Returns:
InternVLA_M1: Initialized model instance.
"""
model = InternVLA_M1(config=config)
return model
if __name__ == "__main__":
from omegaconf import OmegaConf
# model parameters
import debugpy
debugpy.listen(("0.0.0.0", 10092))
print("🔍 Rank 0 waiting for debugger attach on port 10092...")
debugpy.wait_for_client()
config_yaml = "InternVLA/config/training/internvla_cotrain_oxe.yaml"
cfg = OmegaConf.load(config_yaml)
# try get model
model = build_model_framework(cfg)
print(model)
# try forward model
# can be fake sample, but here get from dataloader for simpler
from InternVLA.dataloader.lerobot_datasets import get_vla_dataset, collate_fn
vla_dataset_cfg = cfg.datasets.vla_data
dataset = get_vla_dataset(data_cfg=vla_dataset_cfg)
from torch.utils.data import DataLoader
train_dataloader = DataLoader(
dataset,
batch_size=2,
num_workers=1, # For Debug
collate_fn=collate_fn,
)
for batch in tqdm(train_dataloader, desc="Processing Batches"):
batch
break
# try get model
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = model.to(device)
model(batch)