Download src/psi/trainers/posttrain_v1.py from Tinghong-Ye22/code-Psi0-yth-v21: direct link, hf CLI and curl.
- Browser
- Download file 50.8 kB
-
https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/src/psi/trainers/posttrain_v1.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Psi0-yth-v21/src/psi/trainers/posttrain_v1.py
-
curl -L -o posttrain_v1.py https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/src/psi/trainers/posttrain_v1.py
50.8 kB
from __future__ import annotations
import os
import json
import torch
import wandb
import pickle
import numpy as np
from tqdm import tqdm
from copy import copy, deepcopy
import contextlib
import torch.nn.functional as F
from typing import Dict, Optional, List, Union, Any, TYPE_CHECKING
from torch.utils.data import DataLoader
# <modify> 支持按 checkpoint 类型加载 Qwen3-VL 或 Qwen2.5-Omni
# 原代码:
# from transformers import Qwen2_5OmniThinkerForConditionalGeneration, Qwen2_5OmniProcessor #, Qwen2TokenizerFast, Qwen3VLProcessor
from transformers import (
AutoConfig,
AutoProcessor,
Qwen2_5OmniProcessor,
Qwen2_5OmniThinkerForConditionalGeneration,
Qwen3VLForConditionalGeneration,
)
from psi.trainers.qwen2_5omni_mixin import PaddedCollatorForTogether # <modify>
if TYPE_CHECKING:
from psi.config.config import TrainConfig #, LaunchConfig
from psi.config.model_psi0 import Psi0ModelConfig
from psi.config.transform import Psi0ModelTransform
from psi.config.data_seed import SeedDataConfig
from psi.config.data_mix import MixedDataConfig
from accelerate import Accelerator
# from utils.overwatch import initialize_overwatch
from psi.utils import (
initialize_overwatch,
shorten,
move_to_device,
batch_str_to_tensor
)
overwatch = initialize_overwatch(__name__)
# from .base import Trainer
from .trainer import Trainer, worker_init_fn
from psi.utils import flatten, shorten, move_to_device, rmse, seed_everything
from psi.models.psi0 import Psi0Model
class PosttrainV1Trainer(Trainer):
TASK_FAMILY_TO_ID = {"text": 0, "audio": 1, "video": 2}
TEMPLATE_TO_ID = {
"text": 0,
"audio_text": 1,
"audio_only": 2,
"audio_command_text": 3,
"audio_command_only": 4,
"video_text": 5,
"video_only": 6,
"video_clip_text": 7,
"video_clip_only": 8,
}
def __init__(self, cfg, device: torch.device):
super().__init__(cfg, device)
self.noise_scheduler_name = self.model_cfg.noise_scheduler
self.train_diffusion_steps = self.model_cfg.train_diffusion_steps
self.eval_diffusion_steps = self.model_cfg.eval_diffusion_steps
self.ac_chunk = self.model_cfg.action_chunk_size
self.ac_dim = self.model_cfg.action_dim
self.maxmin = self.data_cfg.transform.field
# 机械臂构型,按初始输入loss权重分配各关节
if self.model_cfg.action_dim == 7:
w_xyz, w_rpy, w_gripper = self.model_cfg.loss_w
self.loss_w = (
torch.from_numpy(np.array([w_xyz] * 3 + [w_rpy] * 3 + [w_gripper]))
.float()
.to(self.device)
) # (7,)
else:
# 全身构型,loss权重均分
self.loss_w = torch.tensor([1.0 / self.model_cfg.action_dim] * self.model_cfg.action_dim, dtype=torch.float32).to(self.device)
assert self.loss_w.sum() == 1.0, "Weights better sum to 1.0 to keep loss range consistent"
# <modify>+70D分组loss权重:在原有逐维均权之外,再叠加一层按动作组的可配权重。
self.loss_group_w = self._build_group_loss_weight()
# diffusion
if self.noise_scheduler_name == "ddpm":
from diffusers.schedulers.scheduling_ddim import DDIMScheduler
self.noise_scheduler = DDIMScheduler( # FIXME configure args
num_train_timesteps=self.train_diffusion_steps,
beta_start=0.0001,
beta_end=0.02,
beta_schedule="squaredcos_cap_v2",
clip_sample=True,
set_alpha_to_one=True,
steps_offset=0,
prediction_type="epsilon",
)
# flow-matching
elif self.noise_scheduler_name == "flow":
from diffusers.schedulers.scheduling_flow_match_euler_discrete import (
FlowMatchEulerDiscreteScheduler,
)
self.noise_scheduler = FlowMatchEulerDiscreteScheduler(
num_train_timesteps=self.train_diffusion_steps, # MUST be 1000 as per pretrained SD3
)
assert self.train_cfg.mixed_precision == "bf16", "keep qwen3vl in bf16 while training action header in fp32"
# assert self.model_cfg.n_conditions == len(self.data_cfg.transform.repack.conditions), "inconsistent confs" # type: ignore
# <modify>
@property
def data_cfg(self) -> SeedDataConfig | MixedDataConfig:
return self.cfg.data # type: ignore
@property
def model_cfg(self) -> Psi0ModelConfig:
return self.cfg.model # type: ignore
# <modify>+梯度保护:先清理 nan/inf,再做逐元素梯度裁剪,返回本步是否见到过非有限梯度。
def _protect_gradients(self) -> bool:
max_grad_value = self.train_cfg.max_grad_value
had_nonfinite_grad = False
if hasattr(self.accelerator, "unscale_gradients"):
self.accelerator.unscale_gradients()
for param in self.model.parameters():
if param.grad is None:
continue
if not torch.isfinite(param.grad).all():
had_nonfinite_grad = True
if max_grad_value is None:
param.grad.data.nan_to_num_(nan=0.0, posinf=0.0, neginf=0.0)
else:
param.grad.data.nan_to_num_(
nan=0.0,
posinf=float(max_grad_value),
neginf=-float(max_grad_value),
)
if max_grad_value is not None:
param.grad.data.clamp_(-float(max_grad_value), float(max_grad_value))
return had_nonfinite_grad
# TextOp分组loss日志:39D新布局,70D旧布局保留兜底。
def _action_group_losses(self, loss_by_dim: torch.Tensor) -> dict[str, torch.Tensor]:
if loss_by_dim.shape[-1] == 39:
group_slices = {
"joint_abs": slice(0, 29),
"root_delta_xy": slice(29, 31),
"root_rot5_delta": slice(31, 36),
"root_height": slice(36, 37),
"foot_contact": slice(37, 39),
}
elif loss_by_dim.shape[-1] == 70:
group_slices = {
"joint_abs": slice(0, 29),
"joint_delta": slice(29, 58),
"root_delta_local": slice(58, 61),
"root_rot6d_delta": slice(61, 67),
"root_height": slice(67, 68),
"foot_contact": slice(68, 70),
}
else:
return {}
return {
f"loss_group/{name}": loss_by_dim[group_slice].mean()
for name, group_slice in group_slices.items()
}
# TextOp分组loss权重:把配置里的组权重展开成逐维权重向量。
def _build_group_loss_weight(self) -> torch.Tensor:
loss_group_w = torch.ones(self.model_cfg.action_dim, dtype=torch.float32, device=self.device)
group_cfg = self.train_cfg.loss_group_weight
if self.model_cfg.action_dim == 39:
group_slices = {
"joint_abs": slice(0, 29),
"root_delta_xy": slice(29, 31),
"root_rot5_delta": slice(31, 36),
"root_height": slice(36, 37),
"foot_contact": slice(37, 39),
}
elif self.model_cfg.action_dim == 70:
group_slices = {
"joint_abs": slice(0, 29),
"joint_delta": slice(29, 58),
"root_delta_local": slice(58, 61),
"root_rot6d_delta": slice(61, 67),
"root_height": slice(67, 68),
"foot_contact": slice(68, 70),
}
else:
return loss_group_w
for group_name, group_slice in group_slices.items():
loss_group_w[group_slice] = float(getattr(group_cfg, group_name))
return loss_group_w
def _batch_task_logs(self, losses: dict[str, torch.Tensor]) -> dict[str, float]:
task_logs: dict[str, float] = {}
task_specs = (
("_task_loss_sums", "_task_loss_counts", "loss_task", self.TASK_FAMILY_TO_ID),
("_template_loss_sums", "_template_loss_counts", "loss_template", self.TEMPLATE_TO_ID),
)
for sum_key, count_key, prefix, mapping in task_specs:
if sum_key not in losses or count_key not in losses:
continue
gathered_sums = self.accelerator.gather(losses[sum_key].detach().float()).reshape(-1, len(mapping)).sum(dim=0)
gathered_counts = self.accelerator.gather(losses[count_key].detach().float()).reshape(-1, len(mapping)).sum(dim=0)
for task_name, task_id in mapping.items():
if gathered_counts[task_id].item() > 0:
task_logs[f"{prefix}/{task_name}"] = (gathered_sums[task_id] / gathered_counts[task_id]).item()
return task_logs
def _batch_label(self, batch: dict[str, Any], key: str) -> str:
value = batch.get(key, "")
if isinstance(value, list) and value:
return str(value[0])
if isinstance(value, str):
return value
return ""
def _init_task_metric_tensors(self) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
task_loss_sums = torch.zeros(len(self.TASK_FAMILY_TO_ID), device=self.device, dtype=torch.float32)
task_loss_counts = torch.zeros(len(self.TASK_FAMILY_TO_ID), device=self.device, dtype=torch.float32)
template_loss_sums = torch.zeros(len(self.TEMPLATE_TO_ID), device=self.device, dtype=torch.float32)
template_loss_counts = torch.zeros(len(self.TEMPLATE_TO_ID), device=self.device, dtype=torch.float32)
return task_loss_sums, task_loss_counts, template_loss_sums, template_loss_counts
def _accumulate_task_metrics(
self,
batch: dict[str, Any],
loss_value: torch.Tensor,
batch_size: int,
task_loss_sums: torch.Tensor,
task_loss_counts: torch.Tensor,
template_loss_sums: torch.Tensor,
template_loss_counts: torch.Tensor,
) -> None:
task_family = self._batch_label(batch, "task_family")
if task_family in self.TASK_FAMILY_TO_ID:
task_id = self.TASK_FAMILY_TO_ID[task_family]
task_loss_sums[task_id] += loss_value.detach().float() * float(batch_size)
task_loss_counts[task_id] += float(batch_size)
task_name = self._batch_label(batch, "task_name")
if task_name in self.TEMPLATE_TO_ID:
template_id = self.TEMPLATE_TO_ID[task_name]
template_loss_sums[template_id] += loss_value.detach().float() * float(batch_size)
template_loss_counts[template_id] += float(batch_size)
# <modify> 根据 model_type 自动加载 Ψ₀/Qwen3-VL 预训练模型或 Qwen2.5-Omni
def init_vlm_models(self):
model_path = self.model_cfg.model_name_or_path
overwatch.info(f"Loading VLM backbone from {model_path}")
model_config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
model_type = str(getattr(model_config, "model_type", "")).lower()
# 原 Qwen2.5-Omni 专用加载逻辑:
# vlm_model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained(
# self.model_cfg.model_name_or_path,
# attn_implementation="flash_attention_2",
# dtype=torch.bfloat16
# )
# self.vlm_processor = Qwen2_5OmniProcessor.from_pretrained(self.model_cfg.model_name_or_path)
if "omni" in model_type:
vlm_model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained(
model_path,
attn_implementation="flash_attention_2",
dtype=torch.bfloat16,
)
self.vlm_processor = Qwen2_5OmniProcessor.from_pretrained(model_path)
elif model_type == "qwen3_vl":
vlm_model = Qwen3VLForConditionalGeneration.from_pretrained(
model_path,
attn_implementation="flash_attention_2",
dtype=torch.bfloat16,
)
self.vlm_processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
else:
raise ValueError(f"Unsupported VLM model_type={model_type!r} from {model_path}")
if not hasattr(vlm_model.config, "hidden_size"):
vlm_model.config.hidden_size = vlm_model.config.text_config.hidden_size
self.tokenizer = self.vlm_processor.tokenizer
assert self.cfg.train.lora == False
overwatch.info(f"Loaded VLM model_type={model_type}, hidden_size={vlm_model.config.hidden_size}")
return vlm_model
# <modify> 保留旧函数名,避免其它地方仍调用旧入口
def init_qwen2_5omni_models(self):
return self.init_vlm_models()
def init_models(self):
# init pretrained vlm
# <modify> 原逻辑只加载 Qwen2.5-Omni;现在按 checkpoint 类型自动加载
# 原代码:vlm_model = self.init_qwen2_5omni_models()
vlm_model = self.init_vlm_models()
self.model = Psi0Model(
model_cfg=self.model_cfg,
vlm_model=vlm_model,
# dtype=self.dtype
)
# print("L129", self.model.action_header.time_ins_embed.out_net[0].weight.dtype) # debug
if self.train_cfg.data_parallel == "deepspeed":
# HACK to set correct config for deepspeed
self.model.config = vlm_model.config
self.model.config.hidden_size = vlm_model.config.text_config.hidden_size
# <modify_tune_vlm> 支持按 tune_vlm 开关冻结/解冻 VLM,默认仍保持冻结行为。
# 原逻辑:
# assert self.model_cfg.tune_vlm == False
# for p in self.model.vlm_model.parameters():
# p.requires_grad = False
if not self.model_cfg.tune_vlm:
for p in self.model.vlm_model.parameters():
p.requires_grad = False
overwatch.info("VLM parameters are frozen (tune_vlm=False)")
else:
# <modify_tune_vlm_text_only> text-only训练时只解冻实际参与action loss的语言模型主干,避免DDP检测到视觉/audio等未用参数。
# 原逻辑:
# for p in self.model.vlm_model.parameters():
# p.requires_grad = True
# if hasattr(self.model.vlm_model, "model") and \
# hasattr(self.model.vlm_model.model, "language_model") and \
# hasattr(self.model.vlm_model.model.language_model, "norm"):
# for p in self.model.vlm_model.model.language_model.norm.parameters():
# p.requires_grad = False
# overwatch.info("VLM final norm layer frozen (not used in forward pass)")
for p in self.model.vlm_model.parameters():
p.requires_grad = False
language_model = None
if hasattr(self.model.vlm_model, "model") and hasattr(self.model.vlm_model.model, "language_model"):
language_model = self.model.vlm_model.model.language_model
elif hasattr(self.model.vlm_model, "model"):
# Qwen2.5-Omni thinker uses vlm_model.model as the text backbone directly.
language_model = self.model.vlm_model.model
elif hasattr(self.model.vlm_model, "language_model"):
language_model = self.model.vlm_model.language_model
if language_model is None:
raise ValueError("tune_vlm=True but cannot find VLM language_model to tune.")
for p in language_model.parameters():
p.requires_grad = True
if hasattr(language_model, "norm"):
for p in language_model.norm.parameters():
p.requires_grad = False
overwatch.info("VLM final norm layer frozen (not used in forward pass)")
if hasattr(self.model.vlm_model, "lm_head"):
for p in self.model.vlm_model.lm_head.parameters():
p.requires_grad = False
overwatch.info("VLM lm_head frozen (action loss uses hidden states only)")
overwatch.info("Only VLM language_model parameters are trainable (tune_vlm=True)")
num_parameters = sum(p.numel() for p in self.model.parameters() if p.requires_grad)
overwatch.info(f"Model has {num_parameters:,} trainable parameters")
def create_optimizers(self):
# Separate parameters into groups
action_header_params = []
vlm_model_params = []
other_params = []
for name, param in self.model.named_parameters():
# <modify_tune_vlm_text_only> optimizer只接收真正可训练参数,避免冻结模块被放进参数组。
if not param.requires_grad:
continue
if name.startswith("action_header."):
action_header_params.append(param)
elif name.startswith("vlm_model."):
if self.model_cfg.tune_vlm:
vlm_model_params.append(param)
else:
other_params.append(param)
param_groups = []
if action_header_params:
param_groups.append({
'params': action_header_params,
'lr': self.cfg.train.learning_rate,
'group_name': 'action_header'
})
if vlm_model_params:
param_groups.append({
'params': vlm_model_params,
'lr': self.model_cfg.lang_backbone_lr,
'group_name': 'vlm_model'
})
if other_params:
param_groups.append({
'params': other_params,
'lr': self.cfg.train.learning_rate, # Default group
'group_name': 'other'
})
optimizer_kwargs = dict(self.cfg.train.lr_scheduler_kwargs)
if self.cfg.train.optimizer_foreach is not None:
optimizer_kwargs["foreach"] = self.cfg.train.optimizer_foreach
self.optimizer = torch.optim.AdamW(
param_groups,
**optimizer_kwargs, # type: ignore
)
def create_lr_schedulers(
self,
num_training_steps: int | None = None,
optimizer: torch.optim.Optimizer | None = None,
):
from transformers.optimization import get_scheduler
assert num_training_steps is not None
self.lr_scheduler = get_scheduler(
self.cfg.train.lr_scheduler_type,
num_training_steps=num_training_steps * self.world_size,
optimizer=optimizer if optimizer is not None else self.optimizer,
num_warmup_steps=self.num_warmup_steps * self.world_size,
scheduler_specific_kwargs=self.cfg.train.scheduler_specific_kwargs,
)
def create_optimizer_and_scheduler(self, num_training_steps: int | None = None):
optimizer = self.create_optimizers()
self.create_lr_schedulers(
num_training_steps=num_training_steps, optimizer=optimizer
)
def create_datasets(self): # TODO use parent impl.
transform_kwargs=dict(
vlm_processor=self.vlm_processor,
)
self.train_dataset = self.data_cfg(split="train", transform_kwargs=transform_kwargs)
self.val_dataset = self.data_cfg(split="val", transform_kwargs=transform_kwargs)
return self.train_dataset, self.val_dataset
def create_dataloaders(self, train_dataset, val_dataset):
from psi.data.dataset import MixtureDataset
if isinstance(self.data_cfg, MixedDataConfig) or isinstance(self.train_dataset, MixtureDataset):
assert isinstance(self.train_dataset, MixtureDataset)
sampler = getattr(self.data_cfg, "sampler", "batch_mixture")
if sampler == "batch_mixture":
from psi.data.sampler import BatchMixtureSampler
batch_sampler = BatchMixtureSampler(
dataset_lens=[d.dataset_length for d in self.train_dataset.datasets],
mixture_ratios=self.train_dataset.ratios,
num_samples_per_epoch=len(self.train_dataset),
batch_size=self.train_cfg.train_batch_size,
)
elif sampler == "task_mixture":
from psi.data.sampler import TaskMixtureSampler
batch_sampler = TaskMixtureSampler(
dataset_lens=[d.dataset_length for d in self.train_dataset.datasets],
mixture_ratios=self.train_dataset.ratios,
num_samples_per_epoch=len(self.train_dataset),
batch_size=self.train_cfg.train_batch_size,
)
elif sampler == "token_mixture":
from psi.data.sampler import TokenMixtureSampler
batch_sampler = TokenMixtureSampler(
specs=self.train_dataset.specs,
tokens_per_batch=self.data_cfg.tokens_per_device,
num_batches_per_rank=self.num_steps_per_epoch,
)
else:
raise ValueError(f"Unknown sampler type: {sampler}")
g = shuffle = batch_size = drop_last = None
else:
batch_sampler = None
shuffle = True
batch_size = self.train_cfg.train_batch_size
drop_last = True
g = torch.Generator()
g.manual_seed(self.cfg.seed or 42)
collator = PaddedCollatorForTogether(
model_max_length=self.tokenizer.model_max_length,
pad_token_id=self.tokenizer.pad_token_id,
padding_side="right",
)
dataloader_num_workers = max(int(getattr(self.train_cfg, "num_workers", 1)), 0)
if batch_sampler is not None:
self.train_dataloader = DataLoader(
train_dataset,
batch_sampler=batch_sampler,
collate_fn=collator,
num_workers=dataloader_num_workers,
worker_init_fn=worker_init_fn,
persistent_workers=dataloader_num_workers > 0
)
self.train_sampler = batch_sampler # !!important
else:
self.train_dataloader = DataLoader(
train_dataset,
shuffle=shuffle, #
batch_sampler=batch_sampler,
generator=g, #
batch_size=batch_size, #
collate_fn=collator,
num_workers=dataloader_num_workers,
worker_init_fn=worker_init_fn,
persistent_workers=dataloader_num_workers > 0,
drop_last=drop_last or False
)
val_num_workers = getattr(self.train_cfg, "val_num_workers", None)
val_dataloader_num_workers = (
max(int(val_num_workers), 0)
if val_num_workers is not None
else min(dataloader_num_workers, 2)
)
if self.train_cfg.val_num_batches != -1 and val_dataloader_num_workers > 0:
overwatch.info(
"Validation uses only a prefix of the val dataloader; forcing val num_workers=0 "
"to avoid noisy worker aborts during early iterator shutdown."
)
val_dataloader_num_workers = 0
val_dataloader_kwargs = {
"num_workers": val_dataloader_num_workers,
"drop_last": False,
# "pin_memory": True,
"persistent_workers": False,
"shuffle": True,
} # 1 small enough to fit im Mem. 2. no need distributed sampler
def build_val_dataloader(dataset):
return torch.utils.data.DataLoader(
dataset,
batch_size=self.train_cfg.val_batch_size,
collate_fn=collator,
**val_dataloader_kwargs,
)
if isinstance(val_dataset, dict):
self.val_dataloader = {
task_name: build_val_dataloader(dataset)
for task_name, dataset in val_dataset.items()
}
else:
self.val_dataloader = build_val_dataloader(val_dataset)
return self.train_dataloader, self.val_dataloader
@property
def task_run_name(self):
dataset_name = shorten(self.data_cfg.transform.repack.dataset_name) # type:ignore
return (
f".{dataset_name}"
f".{self.noise_scheduler_name}{self.train_diffusion_steps}"
f".{shorten(self.train_cfg.lr_scheduler_type)}"
f".lr{self.train_cfg.learning_rate:.1e}"
)
def prepare(self, accelerator: Accelerator) -> DataLoader:
from psi.data.dataset import MixtureDataset
# NOTE: if use deepspeed, model, optimizer, lr_scheduler must be prepared together !!!
# print("L301", self.model.action_header.time_ins_embed.out_net[0].weight.dtype) # debug
self.model, self.optimizer, self.lr_scheduler = accelerator.prepare(
self.model, self.optimizer, self.lr_scheduler
)
# self.model.action_header.to(torch.float32) # HACK keep action header in fp32
# print("L307", self.model.module.action_header.time_ins_embed.out_net[0].weight.dtype) # debug
if not (
isinstance(self.train_dataset, MixtureDataset)
and getattr(self.data_cfg, "sampler", None) is not None
):
# dont prepare dataloader for mixed dataset as custom sampler is used
self.train_dataloader = accelerator.prepare(self.train_dataloader)
# Gradient Checkpoint Setup for tuned VLM backbones. DDP needs this too;
# otherwise activation memory dominates once the language backbone is unfrozen.
if self.model_cfg.gradient_checkpointing and self.model_cfg.tune_vlm:
if self.train_cfg.data_parallel == "deepspeed":
assert "DeepSpeedEngine" in self.model.__class__.__name__, "deepspeed is not properly initialized!"
model_for_gc = self.model
else:
model_for_gc = accelerator.unwrap_model(self.model)
vlm_model = model_for_gc.vlm_model
if hasattr(vlm_model, "config"):
vlm_model.config.use_cache = False
if hasattr(vlm_model, "gradient_checkpointing_enable"):
try:
vlm_model.gradient_checkpointing_enable(
gradient_checkpointing_kwargs={"use_reentrant": False}
)
except TypeError:
vlm_model.gradient_checkpointing_enable()
if hasattr(vlm_model, "enable_input_require_grads"):
vlm_model.enable_input_require_grads()
else:
input_embeddings = vlm_model.get_input_embeddings()
if not getattr(input_embeddings, "_psi_input_require_grads_hook", False):
def make_inputs_require_grad(module, input, output):
output.requires_grad_(True)
input_embeddings.register_forward_hook(make_inputs_require_grad)
input_embeddings._psi_input_require_grads_hook = True
overwatch.info("VLM gradient checkpointing enabled")
val_dataloader = getattr(self, "val_dataloader", None)
if isinstance(val_dataloader, dict):
self.val_dataloader = {
task_name: accelerator.prepare(dataloader)
for task_name, dataloader in val_dataloader.items()
}
elif val_dataloader is not None: # not using if self.val_dataloader to avoid DataLoader.__len__() being called on iterable dataset
self.val_dataloader = accelerator.prepare(self.val_dataloader)
if self.cfg.train.overfit_single_batch:
overwatch.warning("Overfitting a single batch: reusing first batch every step. set cfg.data.image_aug = False for true memorization.")
first_batch = next(iter(self.train_dataloader))
class SingleBatchLoader:
def __iter__(self):
while True:
yield first_batch
def __len__(self):
return 1
self.train_dataloader = SingleBatchLoader()
self.val_dataloader = SingleBatchLoader()
self.accelerator = accelerator
return self.train_dataloader # type: ignore
def training_step(
self,
batch: dict[str, Union[torch.Tensor, Any]],
) -> tuple[bool, dict[str, Any]]:
# <modify_audio_grad_accum> 音频整段输入训练使用梯度累积;非sync步不会进入grad_norm计算,
# 这里先给日志字段默认值,避免第一轮accumulation访问未初始化属性。
self._grad_norm_act = torch.tensor(0.0, device=self.device)
if self.train_cfg.data_parallel == "deepspeed":
gac = contextlib.nullcontext()
else:
# otherwise gradient accumulation is managed by deepspeed
gac = self.accelerator.accumulate(self.model)
with gac:
with self.accelerator.autocast():
batch = move_to_device(batch, self.device) # type: ignore
losses = self.forward_and_loss(self.model, batch)
self.accelerator.backward(losses["loss"])
# overwatch.debug(self.global_step, self.device, batch["dataset_name"])
had_nonfinite_grad = False
grad_norm_value = 0.0
if self.accelerator.sync_gradients:
# <modify>+梯度保护:在 norm clip 前先清理 nan/inf,并做逐元素 value clip。
had_nonfinite_grad = self._protect_gradients()
if self.train_cfg.max_grad_norm is not None:
self._grad_norm_act = self.accelerator.clip_grad_norm_(
self.model.parameters(), self.cfg.train.max_grad_norm
)
else:
self._grad_norm_act = self.get_total_grad_norm(self.model.parameters())
grad_norm_value = self._grad_norm_act.item() if torch.is_tensor(self._grad_norm_act) else float(self._grad_norm_act or 0.0)
if self.train_cfg.data_parallel == "ddp":
# NOTE: It is normal that param.grad is None for deepspeed and fsdp
for name, param in self.model.named_parameters():
if param.requires_grad and param.grad is None:
overwatch.critical(f"[Unused] {name} did not receive a gradient.")
# <modify>+梯度保护:如果 loss / grad_norm / grad 本身已经出现非有限值,就跳过本次参数更新。
loss_value = losses["loss"].detach().float()
should_skip_update = False
if self.train_cfg.skip_step_on_nonfinite_grad:
should_skip_update = (
(not torch.isfinite(loss_value).item())
or (not np.isfinite(grad_norm_value))
or had_nonfinite_grad
)
# 原逻辑:
# self.optimizer.step()
# self.lr_scheduler.step()
# self.optimizer.zero_grad()
if should_skip_update:
if self.accelerator.is_main_process:
overwatch.warning(
f"[modify_grad_protect] skip optimizer step, "
f"loss={loss_value.item():.6f}, grad_norm={grad_norm_value}, "
f"lr={self.lr:.6e}, had_nonfinite_grad={had_nonfinite_grad}"
)
self.optimizer.zero_grad()
else:
self.optimizer.step()
self.lr_scheduler.step()
self.optimizer.zero_grad()
# Log batch["raw_image"] to wandb every 100 steps
if (
hasattr(self, "global_step")
and self.global_step % self.cfg.log.log_freq == 0
and "raw_images" in batch
and self.accelerator.is_main_process
and self.cfg.log.report_to == "wandb"
):
raw_imgs = batch["raw_images"]
# Support both torch.Tensor and numpy
if isinstance(raw_imgs, torch.Tensor):
raw_imgs = raw_imgs.detach().cpu().numpy()
# Log up to 4 images for visualization: concat them horizontally into one image
img_arrays = []
for i in range(min(10, len(raw_imgs))):
img = raw_imgs[i][0] # t=0
img_arrays.append(img)
# Images are assumed to have same shape and 3 channels; concat directly
concat_img = np.concatenate(img_arrays, axis=1)
wandb.log({"raw_images": [wandb.Image(concat_img, caption=f"raw images {self.global_step}")]}, step=self.global_step)
self.train_loss_tracker = step_loss = self.accelerator.gather(losses["loss"]).mean().item() # type: ignore
# <modify>+70D分组loss日志:把forward里返回的loss_group/*一起写入wandb。
loss_logs = {}
for loss_name, loss_value in losses.items():
if not loss_name.startswith("loss_group/"):
continue
loss_logs[loss_name] = self.accelerator.gather(loss_value.detach()).mean().item() # type: ignore
loss_logs.update(self._batch_task_logs(losses))
return (self.accelerator.sync_gradients, {
"lr_act": self.lr,
"grad_norm_act": self._grad_norm_act, # type: ignore
"loss": step_loss,
**loss_logs,
})
@torch.no_grad()
def inference(self, eval_model, repacked_batch):
traj2ds = repacked_batch["traj2ds"] if "traj2ds" in repacked_batch else None # (B, 3, H, W)
obs = repacked_batch["states"] # (B,1,M)
# imgs_in = {"cam0": view_features} # views # TODO support multi-view
bsz = obs.shape[0]
action_samples = torch.randn(
bsz, self.ac_chunk, self.ac_dim, device=self.device
)
self.noise_scheduler.set_timesteps(self.eval_diffusion_steps)
for timestep in self.noise_scheduler.timesteps:
batched_timestep = timestep.expand(bsz).to(self.device)
model_pred = eval_model(
input_ids=repacked_batch["input_ids"],#####
attention_mask=repacked_batch["attention_mask"], # vlm related
# pixel_values=repacked_batch["pixel_values"],
# image_grid_thw=repacked_batch["image_grid_thw"], #### <modify> 改成可选读取
pixel_values=repacked_batch.get("pixel_values", None),
image_grid_thw=repacked_batch.get("image_grid_thw", None),
# <modify_audio> 纯音频Qwen-Omni输入透传;没有音频键时保持原text/image行为。
input_features=repacked_batch.get("input_features", None),
feature_attention_mask=repacked_batch.get("feature_attention_mask", None),
audio_feature_lengths=repacked_batch.get("audio_feature_lengths", None),
pixel_values_videos=repacked_batch.get("pixel_values_videos", None),
video_grid_thw=repacked_batch.get("video_grid_thw", None),
video_second_per_grid=repacked_batch.get("video_second_per_grid", None),
action_samples=action_samples, # (B,Tp,Da)
states=obs,
timestep=batched_timestep,
traj2ds=traj2ds,
progress=repacked_batch.get("progress", None),
future_progress=repacked_batch.get("future_progress", None),
return_dict=True,
).action
action_samples = self.noise_scheduler.step(
model_output=model_pred, timestep=timestep, sample=action_samples # type: ignore
).prev_sample # type: ignore
return action_samples
def save_checkpoint(self, global_step: int) -> str | None:
save_dir = os.path.join(self.project_dir, "checkpoints")
os.makedirs(save_dir, exist_ok=True)
ckpt_dir = os.path.join(save_dir, f"ckpt_{global_step}")
# <modify> 不再额外保存 save_model 生成的分片权重,避免每个ckpt重复占用约10G存储
# self.accelerator.save_model(self.model, ckpt_dir)
return super().save_checkpoint(global_step)
def evaluate(self) -> dict[str, float] | None:
accelerator = self.accelerator
global_step = self.global_step
eval_model = self.unwrap_model()
val_dataloaders = self.val_dataloader
if isinstance(val_dataloaders, dict):
val_dataloader_items = list(val_dataloaders.items())
else:
val_name = str(getattr(self.data_cfg, "val_task", "val"))
val_dataloader_items = [(val_name, val_dataloaders)]
all_val_loss_list = []
val_task_logs: dict[str, float] = {}
task_loss_sums = torch.zeros(len(self.TASK_FAMILY_TO_ID), device=self.device, dtype=torch.float32)
task_loss_counts = torch.zeros(len(self.TASK_FAMILY_TO_ID), device=self.device, dtype=torch.float32)
template_loss_sums = torch.zeros(len(self.TEMPLATE_TO_ID), device=self.device, dtype=torch.float32)
template_loss_counts = torch.zeros(len(self.TEMPLATE_TO_ID), device=self.device, dtype=torch.float32)
for val_name, val_dataloader in val_dataloader_items:
total_val_batches = (
len(val_dataloader)
if self.train_cfg.val_num_batches == -1
else min(self.train_cfg.val_num_batches, len(val_dataloader))
)
if total_val_batches <= 0:
continue
val_progress_bar = tqdm(
val_dataloader,
total=total_val_batches,
disable=not accelerator.is_local_main_process,
position=1,
leave=False,
)
val_progress_bar.set_description(f"Eval {val_name} at global step {global_step}")
val_loss_list = []
for val_step, val_batch in enumerate(val_progress_bar):
val_batch: dict = move_to_device(batch_str_to_tensor(val_batch), self.device) # type: ignore
with accelerator.autocast():
val_loss = self.forward_and_loss(eval_model, val_batch)
gathered_loss = accelerator.gather(val_loss["loss"])
val_loss_list.append(gathered_loss)
all_val_loss_list.append(gathered_loss)
task_loss_sums += val_loss.get("_task_loss_sums", torch.zeros_like(task_loss_sums)).detach().float()
task_loss_counts += val_loss.get("_task_loss_counts", torch.zeros_like(task_loss_counts)).detach().float()
template_loss_sums += val_loss.get("_template_loss_sums", torch.zeros_like(template_loss_sums)).detach().float()
template_loss_counts += val_loss.get("_template_loss_counts", torch.zeros_like(template_loss_counts)).detach().float()
if val_step + 1 >= total_val_batches:
if accelerator.is_local_main_process:
val_progress_bar.close()
if hasattr(val_dataloader, "end"):
val_dataloader.end() # type: ignore
break
if val_loss_list:
val_task_logs[f"loss_val_task/{val_name}"] = torch.cat(val_loss_list).mean().item()
if not all_val_loss_list:
return None
avg_val_loss = torch.cat(all_val_loss_list).mean().item()
task_loss_sums_all = accelerator.gather(task_loss_sums).reshape(-1, len(self.TASK_FAMILY_TO_ID)).sum(dim=0)
task_loss_counts_all = accelerator.gather(task_loss_counts).reshape(-1, len(self.TASK_FAMILY_TO_ID)).sum(dim=0)
template_loss_sums_all = accelerator.gather(template_loss_sums).reshape(-1, len(self.TEMPLATE_TO_ID)).sum(dim=0)
template_loss_counts_all = accelerator.gather(template_loss_counts).reshape(-1, len(self.TEMPLATE_TO_ID)).sum(dim=0)
task_logs = {}
for task_name, task_id in self.TASK_FAMILY_TO_ID.items():
if task_loss_counts_all[task_id].item() > 0:
task_logs[f"loss_task/{task_name}"] = (
task_loss_sums_all[task_id] / task_loss_counts_all[task_id]
).item()
for template_name, template_id in self.TEMPLATE_TO_ID.items():
if template_loss_counts_all[template_id].item() > 0:
task_logs[f"loss_template/{template_name}"] = (
template_loss_sums_all[template_id] / template_loss_counts_all[template_id]
).item()
# <modify>+eval日志精简:原来的action L1指标不再返回,wandb中eval下只保留eval/loss。
# 原逻辑:
# action_l1_err_list = np.concatenate(action_l1_err_list, axis=0) # (len_val_dataset*Ta, Da)
# action_l1_err_list_denormed = self.maxmin.denormalize_L1_action_err( action_l1_err_list) # type: ignore
#
# # action L1 errors
# avg_action_errors_denormed = action_l1_err_list_denormed.mean(0) # (Da,) NOTE only if the error is L1 (linear)
# labels_denormed = [
# "hand_left_l1",
# "hand_right_l1",
# "arm_left_l1",
# "arm_right_l1",
# ]
# avg_lr_action_err_denormed = np.split(
# avg_action_errors_denormed, [7, 14, 21], axis=-1
# )
# l1_action_errs = [
# float(np.mean(x)) for x in avg_lr_action_err_denormed
# ]
# log metrics
return {
"loss": avg_val_loss,
**val_task_logs,
**task_logs,
}
def _forward_and_loss_single(self, model, batch) -> dict[str, torch.Tensor]:
bsz, *_, h, w = batch["actions"].shape
if self.model_cfg.rtc and self.noise_scheduler_name != "flow":
raise ValueError("RTC training in posttrain_v1 currently supports flow scheduler only")
# for debugging purposes
# gen = torch.Generator(device=self.device)
# gen.manual_seed(42) # or any fixed seed you want
gen = None
if self.noise_scheduler_name == "ddpm":
timesteps = torch.randint(
low=0, high=self.train_diffusion_steps, size=(bsz,), device=self.device
).long()
elif self.noise_scheduler_name == "flow":
# For reproducibility/debugging
sigmas = torch.rand((bsz,), device=self.device, generator=gen).float() * 0.95 + 0.05
timesteps = sigmas * self.train_diffusion_steps
# <modify> 支持RTC训练:随机选择prefix delay,并把prefix位置设成clean action条件
# 原逻辑:sigmas = sigmas.view(-1, 1, 1)
if self.model_cfg.rtc:
delay = torch.randint(
low=0,
high=self.model_cfg.max_delay,
size=(bsz,),
device=self.device,
dtype=torch.long,
generator=gen,
)
prefix_mask = torch.arange(batch["actions"].shape[1], device=self.device)[None, :] < delay[:, None]
sigmas = torch.where(
prefix_mask,
torch.tensor(0.0, device=self.device, dtype=sigmas.dtype),
sigmas[:, None],
)
timesteps = sigmas * self.train_diffusion_steps
else:
sigmas = sigmas.view(-1, 1, 1)
else:
raise ValueError(f"Unknown noise scheduler: {self.noise_scheduler_name}")
# noise_action = torch.randn_like(batch["actions"])
noise_action = torch.randn(batch["actions"].shape, device=self.device, generator=gen)
if self.noise_scheduler_name == "ddpm":
noisy_actions = self.noise_scheduler.add_noise(
batch["actions"], noise_action, timesteps # type: ignore
)
target_action = noise_action # ddpm epsilon-prediction
elif self.noise_scheduler_name == "flow":
# <modify> RTC时sigmas是(B,Tp),普通flow时是(B,1,1),这里统一扩到actions维度
# 原逻辑:sigmas_action = sigmas.view(-1, 1, 1)
sigmas_action = sigmas # type: ignore
while len(sigmas_action.shape) < len(batch["actions"].shape):
sigmas_action = sigmas_action.unsqueeze(-1)
noisy_actions = (1 - sigmas_action) * batch["actions"] + sigmas_action * noise_action
target_action = noise_action - batch["actions"] # flow-matching velocity prediction
else:
raise ValueError
model_output = model(
input_ids=batch["input_ids"],#####
attention_mask=batch["attention_mask"], # vlm related
# pixel_values=batch["pixel_values"],
# image_grid_thw=batch["image_grid_thw"], #### <modify> 改成可选输入
pixel_values=batch.get("pixel_values", None),
image_grid_thw=batch.get("image_grid_thw", None),
# <modify_audio> 纯音频Qwen-Omni输入透传;没有音频键时保持原text/image行为。
input_features=batch.get("input_features", None),
feature_attention_mask=batch.get("feature_attention_mask", None),
audio_feature_lengths=batch.get("audio_feature_lengths", None),
pixel_values_videos=batch.get("pixel_values_videos", None),
video_grid_thw=batch.get("video_grid_thw", None),
video_second_per_grid=batch.get("video_second_per_grid", None),
action_samples=noisy_actions, # (B,Tp,Da)
states=batch["states"], # (B,1,M)
timestep=timesteps,
traj2ds=batch["traj2ds"] if "traj2ds" in batch else None, # (B, C, 3, H, W)
progress=batch.get("progress", None),
future_progress=batch.get("future_progress", None),
return_dict=True,
)
action_pred = model_output.action
loss_action = F.mse_loss(
action_pred.float(), target_action.float(), reduction="none"
) # (B, Tp, Da)
# for post pre training, we only care about the overall loss
# """
# sum(1) -> to keep gradient consistent when training with different action chunks
# mean(0) -> to average over the batch
# sum() -> to sum over the weighted loss of each action dimension
# """
if "actions_mask" not in batch:
batch["actions_mask"] = torch.ones_like(batch["actions"])
# <modify> RTC训练时prefix是已知旧动作,只在postfix上计算loss
# 原逻辑:loss_action = (loss_action * batch["actions_mask"]).sum(1)
action_mask = batch["actions_mask"]
if self.model_cfg.rtc:
postfix_mask = (~prefix_mask)[:, :, None].float() # type: ignore
action_mask = action_mask * postfix_mask
loss_action = (loss_action * action_mask).sum(1) # (B, Da)
# <modify>+70D分组loss日志:先保留逐维loss,再按原逻辑计算总loss。
loss_action_by_dim = loss_action.mean(0)
# <modify>+70D分组loss权重:在原有 self.loss_w 的基础上,继续乘一层组权重。
# 原逻辑:
# loss_action = (loss_action_by_dim * self.loss_w).sum()
loss_action = (loss_action_by_dim * self.loss_w * self.loss_group_w).sum()
group_losses = self._action_group_losses(loss_action_by_dim)
return {"loss": loss_action, **group_losses}
def forward_and_loss(self, model, batch) -> dict[str, torch.Tensor]:
task_batches = batch.get("task_batches", None)
if not isinstance(task_batches, dict):
losses = self._forward_and_loss_single(model, batch)
task_loss_sums, task_loss_counts, template_loss_sums, template_loss_counts = self._init_task_metric_tensors()
self._accumulate_task_metrics(
batch,
losses["loss"],
int(batch["actions"].shape[0]),
task_loss_sums,
task_loss_counts,
template_loss_sums,
template_loss_counts,
)
return {
**losses,
"_task_loss_sums": task_loss_sums,
"_task_loss_counts": task_loss_counts,
"_template_loss_sums": template_loss_sums,
"_template_loss_counts": template_loss_counts,
}
total_batch_size = sum(int(task_batch["actions"].shape[0]) for task_batch in task_batches.values())
if total_batch_size <= 0:
raise ValueError("Empty mixed batch received in forward_and_loss")
task_loss_sums, task_loss_counts, template_loss_sums, template_loss_counts = self._init_task_metric_tensors()
total_loss = None
merged_group_losses: dict[str, torch.Tensor] = {}
for task_batch in task_batches.values():
task_batch_size = int(task_batch["actions"].shape[0])
task_weight = float(task_batch_size) / float(total_batch_size)
task_losses = self._forward_and_loss_single(model, task_batch)
weighted_loss = task_losses["loss"] * task_weight
total_loss = weighted_loss if total_loss is None else total_loss + weighted_loss
for loss_name, loss_value in task_losses.items():
if loss_name == "loss":
continue
merged_group_losses[loss_name] = (
loss_value * task_weight
if loss_name not in merged_group_losses
else merged_group_losses[loss_name] + loss_value * task_weight
)
self._accumulate_task_metrics(
task_batch,
task_losses["loss"],
task_batch_size,
task_loss_sums,
task_loss_counts,
template_loss_sums,
template_loss_counts,
)
assert total_loss is not None
return {
"loss": total_loss,
**merged_group_losses,
"_task_loss_sums": task_loss_sums,
"_task_loss_counts": task_loss_counts,
"_template_loss_sums": template_loss_sums,
"_template_loss_counts": template_loss_counts,
}
from __future__ import annotations
import os
import json
import torch
import wandb
import pickle
import numpy as np
from tqdm import tqdm
from copy import copy, deepcopy
import contextlib
import torch.nn.functional as F
from typing import Dict, Optional, List, Union, Any, TYPE_CHECKING
from torch.utils.data import DataLoader
# <modify> 支持按 checkpoint 类型加载 Qwen3-VL 或 Qwen2.5-Omni
# 原代码:
# from transformers import Qwen2_5OmniThinkerForConditionalGeneration, Qwen2_5OmniProcessor #, Qwen2TokenizerFast, Qwen3VLProcessor
from transformers import (
AutoConfig,
AutoProcessor,
Qwen2_5OmniProcessor,
Qwen2_5OmniThinkerForConditionalGeneration,
Qwen3VLForConditionalGeneration,
)
from psi.trainers.qwen2_5omni_mixin import PaddedCollatorForTogether # <modify>
if TYPE_CHECKING:
from psi.config.config import TrainConfig #, LaunchConfig
from psi.config.model_psi0 import Psi0ModelConfig
from psi.config.transform import Psi0ModelTransform
from psi.config.data_seed import SeedDataConfig
from psi.config.data_mix import MixedDataConfig
from accelerate import Accelerator
# from utils.overwatch import initialize_overwatch
from psi.utils import (
initialize_overwatch,
shorten,
move_to_device,
batch_str_to_tensor
)
overwatch = initialize_overwatch(__name__)
# from .base import Trainer
from .trainer import Trainer, worker_init_fn
from psi.utils import flatten, shorten, move_to_device, rmse, seed_everything
from psi.models.psi0 import Psi0Model
class PosttrainV1Trainer(Trainer):
TASK_FAMILY_TO_ID = {"text": 0, "audio": 1, "video": 2}
TEMPLATE_TO_ID = {
"text": 0,
"audio_text": 1,
"audio_only": 2,
"audio_command_text": 3,
"audio_command_only": 4,
"video_text": 5,
"video_only": 6,
"video_clip_text": 7,
"video_clip_only": 8,
}
def __init__(self, cfg, device: torch.device):
super().__init__(cfg, device)
self.noise_scheduler_name = self.model_cfg.noise_scheduler
self.train_diffusion_steps = self.model_cfg.train_diffusion_steps
self.eval_diffusion_steps = self.model_cfg.eval_diffusion_steps
self.ac_chunk = self.model_cfg.action_chunk_size
self.ac_dim = self.model_cfg.action_dim
self.maxmin = self.data_cfg.transform.field
# 机械臂构型,按初始输入loss权重分配各关节
if self.model_cfg.action_dim == 7:
w_xyz, w_rpy, w_gripper = self.model_cfg.loss_w
self.loss_w = (
torch.from_numpy(np.array([w_xyz] * 3 + [w_rpy] * 3 + [w_gripper]))
.float()
.to(self.device)
) # (7,)
else:
# 全身构型,loss权重均分
self.loss_w = torch.tensor([1.0 / self.model_cfg.action_dim] * self.model_cfg.action_dim, dtype=torch.float32).to(self.device)
assert self.loss_w.sum() == 1.0, "Weights better sum to 1.0 to keep loss range consistent"
# <modify>+70D分组loss权重:在原有逐维均权之外,再叠加一层按动作组的可配权重。
self.loss_group_w = self._build_group_loss_weight()
# diffusion
if self.noise_scheduler_name == "ddpm":
from diffusers.schedulers.scheduling_ddim import DDIMScheduler
self.noise_scheduler = DDIMScheduler( # FIXME configure args
num_train_timesteps=self.train_diffusion_steps,
beta_start=0.0001,
beta_end=0.02,
beta_schedule="squaredcos_cap_v2",
clip_sample=True,
set_alpha_to_one=True,