code-Psi0-yth-v21 / src /psi /trainers /posttrain_v1.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
c4f8e4a verified
Raw History Blame Contribute Delete
50.8 kB
from __future__ import annotations import os import json import torch import wandb import pickle import numpy as np from tqdm import tqdm from copy import copy, deepcopy import contextlib import torch.nn.functional as F from typing import Dict, Optional, List, Union, Any, TYPE_CHECKING from torch.utils.data import DataLoader # <modify> 支持按 checkpoint 类型加载 Qwen3-VL 或 Qwen2.5-Omni # 原代码: # from transformers import Qwen2_5OmniThinkerForConditionalGeneration, Qwen2_5OmniProcessor #, Qwen2TokenizerFast, Qwen3VLProcessor from transformers import ( AutoConfig, AutoProcessor, Qwen2_5OmniProcessor, Qwen2_5OmniThinkerForConditionalGeneration, Qwen3VLForConditionalGeneration, ) from psi.trainers.qwen2_5omni_mixin import PaddedCollatorForTogether # <modify> if TYPE_CHECKING: from psi.config.config import TrainConfig #, LaunchConfig from psi.config.model_psi0 import Psi0ModelConfig from psi.config.transform import Psi0ModelTransform from psi.config.data_seed import SeedDataConfig from psi.config.data_mix import MixedDataConfig from accelerate import Accelerator # from utils.overwatch import initialize_overwatch from psi.utils import ( initialize_overwatch, shorten, move_to_device, batch_str_to_tensor ) overwatch = initialize_overwatch(__name__) # from .base import Trainer from .trainer import Trainer, worker_init_fn from psi.utils import flatten, shorten, move_to_device, rmse, seed_everything from psi.models.psi0 import Psi0Model class PosttrainV1Trainer(Trainer): TASK_FAMILY_TO_ID = {"text": 0, "audio": 1, "video": 2} TEMPLATE_TO_ID = { "text": 0, "audio_text": 1, "audio_only": 2, "audio_command_text": 3, "audio_command_only": 4, "video_text": 5, "video_only": 6, "video_clip_text": 7, "video_clip_only": 8, } def __init__(self, cfg, device: torch.device): super().__init__(cfg, device) self.noise_scheduler_name = self.model_cfg.noise_scheduler self.train_diffusion_steps = self.model_cfg.train_diffusion_steps self.eval_diffusion_steps = self.model_cfg.eval_diffusion_steps self.ac_chunk = self.model_cfg.action_chunk_size self.ac_dim = self.model_cfg.action_dim self.maxmin = self.data_cfg.transform.field # 机械臂构型,按初始输入loss权重分配各关节 if self.model_cfg.action_dim == 7: w_xyz, w_rpy, w_gripper = self.model_cfg.loss_w self.loss_w = ( torch.from_numpy(np.array([w_xyz] * 3 + [w_rpy] * 3 + [w_gripper])) .float() .to(self.device) ) # (7,) else: # 全身构型,loss权重均分 self.loss_w = torch.tensor([1.0 / self.model_cfg.action_dim] * self.model_cfg.action_dim, dtype=torch.float32).to(self.device) assert self.loss_w.sum() == 1.0, "Weights better sum to 1.0 to keep loss range consistent" # <modify>+70D分组loss权重:在原有逐维均权之外,再叠加一层按动作组的可配权重。 self.loss_group_w = self._build_group_loss_weight() # diffusion if self.noise_scheduler_name == "ddpm": from diffusers.schedulers.scheduling_ddim import DDIMScheduler self.noise_scheduler = DDIMScheduler( # FIXME configure args num_train_timesteps=self.train_diffusion_steps, beta_start=0.0001, beta_end=0.02, beta_schedule="squaredcos_cap_v2", clip_sample=True, set_alpha_to_one=True, steps_offset=0, prediction_type="epsilon", ) # flow-matching elif self.noise_scheduler_name == "flow": from diffusers.schedulers.scheduling_flow_match_euler_discrete import ( FlowMatchEulerDiscreteScheduler, ) self.noise_scheduler = FlowMatchEulerDiscreteScheduler( num_train_timesteps=self.train_diffusion_steps, # MUST be 1000 as per pretrained SD3 ) assert self.train_cfg.mixed_precision == "bf16", "keep qwen3vl in bf16 while training action header in fp32" # assert self.model_cfg.n_conditions == len(self.data_cfg.transform.repack.conditions), "inconsistent confs" # type: ignore # <modify> @property def data_cfg(self) -> SeedDataConfig | MixedDataConfig: return self.cfg.data # type: ignore @property def model_cfg(self) -> Psi0ModelConfig: return self.cfg.model # type: ignore # <modify>+梯度保护:先清理 nan/inf,再做逐元素梯度裁剪,返回本步是否见到过非有限梯度。 def _protect_gradients(self) -> bool: max_grad_value = self.train_cfg.max_grad_value had_nonfinite_grad = False if hasattr(self.accelerator, "unscale_gradients"): self.accelerator.unscale_gradients() for param in self.model.parameters(): if param.grad is None: continue if not torch.isfinite(param.grad).all(): had_nonfinite_grad = True if max_grad_value is None: param.grad.data.nan_to_num_(nan=0.0, posinf=0.0, neginf=0.0) else: param.grad.data.nan_to_num_( nan=0.0, posinf=float(max_grad_value), neginf=-float(max_grad_value), ) if max_grad_value is not None: param.grad.data.clamp_(-float(max_grad_value), float(max_grad_value)) return had_nonfinite_grad # TextOp分组loss日志:39D新布局,70D旧布局保留兜底。 def _action_group_losses(self, loss_by_dim: torch.Tensor) -> dict[str, torch.Tensor]: if loss_by_dim.shape[-1] == 39: group_slices = { "joint_abs": slice(0, 29), "root_delta_xy": slice(29, 31), "root_rot5_delta": slice(31, 36), "root_height": slice(36, 37), "foot_contact": slice(37, 39), } elif loss_by_dim.shape[-1] == 70: group_slices = { "joint_abs": slice(0, 29), "joint_delta": slice(29, 58), "root_delta_local": slice(58, 61), "root_rot6d_delta": slice(61, 67), "root_height": slice(67, 68), "foot_contact": slice(68, 70), } else: return {} return { f"loss_group/{name}": loss_by_dim[group_slice].mean() for name, group_slice in group_slices.items() } # TextOp分组loss权重:把配置里的组权重展开成逐维权重向量。 def _build_group_loss_weight(self) -> torch.Tensor: loss_group_w = torch.ones(self.model_cfg.action_dim, dtype=torch.float32, device=self.device) group_cfg = self.train_cfg.loss_group_weight if self.model_cfg.action_dim == 39: group_slices = { "joint_abs": slice(0, 29), "root_delta_xy": slice(29, 31), "root_rot5_delta": slice(31, 36), "root_height": slice(36, 37), "foot_contact": slice(37, 39), } elif self.model_cfg.action_dim == 70: group_slices = { "joint_abs": slice(0, 29), "joint_delta": slice(29, 58), "root_delta_local": slice(58, 61), "root_rot6d_delta": slice(61, 67), "root_height": slice(67, 68), "foot_contact": slice(68, 70), } else: return loss_group_w for group_name, group_slice in group_slices.items(): loss_group_w[group_slice] = float(getattr(group_cfg, group_name)) return loss_group_w def _batch_task_logs(self, losses: dict[str, torch.Tensor]) -> dict[str, float]: task_logs: dict[str, float] = {} task_specs = ( ("_task_loss_sums", "_task_loss_counts", "loss_task", self.TASK_FAMILY_TO_ID), ("_template_loss_sums", "_template_loss_counts", "loss_template", self.TEMPLATE_TO_ID), ) for sum_key, count_key, prefix, mapping in task_specs: if sum_key not in losses or count_key not in losses: continue gathered_sums = self.accelerator.gather(losses[sum_key].detach().float()).reshape(-1, len(mapping)).sum(dim=0) gathered_counts = self.accelerator.gather(losses[count_key].detach().float()).reshape(-1, len(mapping)).sum(dim=0) for task_name, task_id in mapping.items(): if gathered_counts[task_id].item() > 0: task_logs[f"{prefix}/{task_name}"] = (gathered_sums[task_id] / gathered_counts[task_id]).item() return task_logs def _batch_label(self, batch: dict[str, Any], key: str) -> str: value = batch.get(key, "") if isinstance(value, list) and value: return str(value[0]) if isinstance(value, str): return value return "" def _init_task_metric_tensors(self) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: task_loss_sums = torch.zeros(len(self.TASK_FAMILY_TO_ID), device=self.device, dtype=torch.float32) task_loss_counts = torch.zeros(len(self.TASK_FAMILY_TO_ID), device=self.device, dtype=torch.float32) template_loss_sums = torch.zeros(len(self.TEMPLATE_TO_ID), device=self.device, dtype=torch.float32) template_loss_counts = torch.zeros(len(self.TEMPLATE_TO_ID), device=self.device, dtype=torch.float32) return task_loss_sums, task_loss_counts, template_loss_sums, template_loss_counts def _accumulate_task_metrics( self, batch: dict[str, Any], loss_value: torch.Tensor, batch_size: int, task_loss_sums: torch.Tensor, task_loss_counts: torch.Tensor, template_loss_sums: torch.Tensor, template_loss_counts: torch.Tensor, ) -> None: task_family = self._batch_label(batch, "task_family") if task_family in self.TASK_FAMILY_TO_ID: task_id = self.TASK_FAMILY_TO_ID[task_family] task_loss_sums[task_id] += loss_value.detach().float() * float(batch_size) task_loss_counts[task_id] += float(batch_size) task_name = self._batch_label(batch, "task_name") if task_name in self.TEMPLATE_TO_ID: template_id = self.TEMPLATE_TO_ID[task_name] template_loss_sums[template_id] += loss_value.detach().float() * float(batch_size) template_loss_counts[template_id] += float(batch_size) # <modify> 根据 model_type 自动加载 Ψ₀/Qwen3-VL 预训练模型或 Qwen2.5-Omni def init_vlm_models(self): model_path = self.model_cfg.model_name_or_path overwatch.info(f"Loading VLM backbone from {model_path}") model_config = AutoConfig.from_pretrained(model_path, trust_remote_code=True) model_type = str(getattr(model_config, "model_type", "")).lower() # 原 Qwen2.5-Omni 专用加载逻辑: # vlm_model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained( # self.model_cfg.model_name_or_path, # attn_implementation="flash_attention_2", # dtype=torch.bfloat16 # ) # self.vlm_processor = Qwen2_5OmniProcessor.from_pretrained(self.model_cfg.model_name_or_path) if "omni" in model_type: vlm_model = Qwen2_5OmniThinkerForConditionalGeneration.from_pretrained( model_path, attn_implementation="flash_attention_2", dtype=torch.bfloat16, ) self.vlm_processor = Qwen2_5OmniProcessor.from_pretrained(model_path) elif model_type == "qwen3_vl": vlm_model = Qwen3VLForConditionalGeneration.from_pretrained( model_path, attn_implementation="flash_attention_2", dtype=torch.bfloat16, ) self.vlm_processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True) else: raise ValueError(f"Unsupported VLM model_type={model_type!r} from {model_path}") if not hasattr(vlm_model.config, "hidden_size"): vlm_model.config.hidden_size = vlm_model.config.text_config.hidden_size self.tokenizer = self.vlm_processor.tokenizer assert self.cfg.train.lora == False overwatch.info(f"Loaded VLM model_type={model_type}, hidden_size={vlm_model.config.hidden_size}") return vlm_model # <modify> 保留旧函数名,避免其它地方仍调用旧入口 def init_qwen2_5omni_models(self): return self.init_vlm_models() def init_models(self): # init pretrained vlm # <modify> 原逻辑只加载 Qwen2.5-Omni;现在按 checkpoint 类型自动加载 # 原代码:vlm_model = self.init_qwen2_5omni_models() vlm_model = self.init_vlm_models() self.model = Psi0Model( model_cfg=self.model_cfg, vlm_model=vlm_model, # dtype=self.dtype ) # print("L129", self.model.action_header.time_ins_embed.out_net[0].weight.dtype) # debug if self.train_cfg.data_parallel == "deepspeed": # HACK to set correct config for deepspeed self.model.config = vlm_model.config self.model.config.hidden_size = vlm_model.config.text_config.hidden_size # <modify_tune_vlm> 支持按 tune_vlm 开关冻结/解冻 VLM,默认仍保持冻结行为。 # 原逻辑: # assert self.model_cfg.tune_vlm == False # for p in self.model.vlm_model.parameters(): # p.requires_grad = False if not self.model_cfg.tune_vlm: for p in self.model.vlm_model.parameters(): p.requires_grad = False overwatch.info("VLM parameters are frozen (tune_vlm=False)") else: # <modify_tune_vlm_text_only> text-only训练时只解冻实际参与action loss的语言模型主干,避免DDP检测到视觉/audio等未用参数。 # 原逻辑: # for p in self.model.vlm_model.parameters(): # p.requires_grad = True # if hasattr(self.model.vlm_model, "model") and \ # hasattr(self.model.vlm_model.model, "language_model") and \ # hasattr(self.model.vlm_model.model.language_model, "norm"): # for p in self.model.vlm_model.model.language_model.norm.parameters(): # p.requires_grad = False # overwatch.info("VLM final norm layer frozen (not used in forward pass)") for p in self.model.vlm_model.parameters(): p.requires_grad = False language_model = None if hasattr(self.model.vlm_model, "model") and hasattr(self.model.vlm_model.model, "language_model"): language_model = self.model.vlm_model.model.language_model elif hasattr(self.model.vlm_model, "model"): # Qwen2.5-Omni thinker uses vlm_model.model as the text backbone directly. language_model = self.model.vlm_model.model elif hasattr(self.model.vlm_model, "language_model"): language_model = self.model.vlm_model.language_model if language_model is None: raise ValueError("tune_vlm=True but cannot find VLM language_model to tune.") for p in language_model.parameters(): p.requires_grad = True if hasattr(language_model, "norm"): for p in language_model.norm.parameters(): p.requires_grad = False overwatch.info("VLM final norm layer frozen (not used in forward pass)") if hasattr(self.model.vlm_model, "lm_head"): for p in self.model.vlm_model.lm_head.parameters(): p.requires_grad = False overwatch.info("VLM lm_head frozen (action loss uses hidden states only)") overwatch.info("Only VLM language_model parameters are trainable (tune_vlm=True)") num_parameters = sum(p.numel() for p in self.model.parameters() if p.requires_grad) overwatch.info(f"Model has {num_parameters:,} trainable parameters") def create_optimizers(self): # Separate parameters into groups action_header_params = [] vlm_model_params = [] other_params = [] for name, param in self.model.named_parameters(): # <modify_tune_vlm_text_only> optimizer只接收真正可训练参数,避免冻结模块被放进参数组。 if not param.requires_grad: continue if name.startswith("action_header."): action_header_params.append(param) elif name.startswith("vlm_model."): if self.model_cfg.tune_vlm: vlm_model_params.append(param) else: other_params.append(param) param_groups = [] if action_header_params: param_groups.append({ 'params': action_header_params, 'lr': self.cfg.train.learning_rate, 'group_name': 'action_header' }) if vlm_model_params: param_groups.append({ 'params': vlm_model_params, 'lr': self.model_cfg.lang_backbone_lr, 'group_name': 'vlm_model' }) if other_params: param_groups.append({ 'params': other_params, 'lr': self.cfg.train.learning_rate, # Default group 'group_name': 'other' }) optimizer_kwargs = dict(self.cfg.train.lr_scheduler_kwargs) if self.cfg.train.optimizer_foreach is not None: optimizer_kwargs["foreach"] = self.cfg.train.optimizer_foreach self.optimizer = torch.optim.AdamW( param_groups, **optimizer_kwargs, # type: ignore ) def create_lr_schedulers( self, num_training_steps: int | None = None, optimizer: torch.optim.Optimizer | None = None, ): from transformers.optimization import get_scheduler assert num_training_steps is not None self.lr_scheduler = get_scheduler( self.cfg.train.lr_scheduler_type, num_training_steps=num_training_steps * self.world_size, optimizer=optimizer if optimizer is not None else self.optimizer, num_warmup_steps=self.num_warmup_steps * self.world_size, scheduler_specific_kwargs=self.cfg.train.scheduler_specific_kwargs, ) def create_optimizer_and_scheduler(self, num_training_steps: int | None = None): optimizer = self.create_optimizers() self.create_lr_schedulers( num_training_steps=num_training_steps, optimizer=optimizer ) def create_datasets(self): # TODO use parent impl. transform_kwargs=dict( vlm_processor=self.vlm_processor, ) self.train_dataset = self.data_cfg(split="train", transform_kwargs=transform_kwargs) self.val_dataset = self.data_cfg(split="val", transform_kwargs=transform_kwargs) return self.train_dataset, self.val_dataset def create_dataloaders(self, train_dataset, val_dataset): from psi.data.dataset import MixtureDataset if isinstance(self.data_cfg, MixedDataConfig) or isinstance(self.train_dataset, MixtureDataset): assert isinstance(self.train_dataset, MixtureDataset) sampler = getattr(self.data_cfg, "sampler", "batch_mixture") if sampler == "batch_mixture": from psi.data.sampler import BatchMixtureSampler batch_sampler = BatchMixtureSampler( dataset_lens=[d.dataset_length for d in self.train_dataset.datasets], mixture_ratios=self.train_dataset.ratios, num_samples_per_epoch=len(self.train_dataset), batch_size=self.train_cfg.train_batch_size, ) elif sampler == "task_mixture": from psi.data.sampler import TaskMixtureSampler batch_sampler = TaskMixtureSampler( dataset_lens=[d.dataset_length for d in self.train_dataset.datasets], mixture_ratios=self.train_dataset.ratios, num_samples_per_epoch=len(self.train_dataset), batch_size=self.train_cfg.train_batch_size, ) elif sampler == "token_mixture": from psi.data.sampler import TokenMixtureSampler batch_sampler = TokenMixtureSampler( specs=self.train_dataset.specs, tokens_per_batch=self.data_cfg.tokens_per_device, num_batches_per_rank=self.num_steps_per_epoch, ) else: raise ValueError(f"Unknown sampler type: {sampler}") g = shuffle = batch_size = drop_last = None else: batch_sampler = None shuffle = True batch_size = self.train_cfg.train_batch_size drop_last = True g = torch.Generator() g.manual_seed(self.cfg.seed or 42) collator = PaddedCollatorForTogether( model_max_length=self.tokenizer.model_max_length, pad_token_id=self.tokenizer.pad_token_id, padding_side="right", ) dataloader_num_workers = max(int(getattr(self.train_cfg, "num_workers", 1)), 0) if batch_sampler is not None: self.train_dataloader = DataLoader( train_dataset, batch_sampler=batch_sampler, collate_fn=collator, num_workers=dataloader_num_workers, worker_init_fn=worker_init_fn, persistent_workers=dataloader_num_workers > 0 ) self.train_sampler = batch_sampler # !!important else: self.train_dataloader = DataLoader( train_dataset, shuffle=shuffle, # batch_sampler=batch_sampler, generator=g, # batch_size=batch_size, # collate_fn=collator, num_workers=dataloader_num_workers, worker_init_fn=worker_init_fn, persistent_workers=dataloader_num_workers > 0, drop_last=drop_last or False ) val_num_workers = getattr(self.train_cfg, "val_num_workers", None) val_dataloader_num_workers = ( max(int(val_num_workers), 0) if val_num_workers is not None else min(dataloader_num_workers, 2) ) if self.train_cfg.val_num_batches != -1 and val_dataloader_num_workers > 0: overwatch.info( "Validation uses only a prefix of the val dataloader; forcing val num_workers=0 " "to avoid noisy worker aborts during early iterator shutdown." ) val_dataloader_num_workers = 0 val_dataloader_kwargs = { "num_workers": val_dataloader_num_workers, "drop_last": False, # "pin_memory": True, "persistent_workers": False, "shuffle": True, } # 1 small enough to fit im Mem. 2. no need distributed sampler def build_val_dataloader(dataset): return torch.utils.data.DataLoader( dataset, batch_size=self.train_cfg.val_batch_size, collate_fn=collator, **val_dataloader_kwargs, ) if isinstance(val_dataset, dict): self.val_dataloader = { task_name: build_val_dataloader(dataset) for task_name, dataset in val_dataset.items() } else: self.val_dataloader = build_val_dataloader(val_dataset) return self.train_dataloader, self.val_dataloader @property def task_run_name(self): dataset_name = shorten(self.data_cfg.transform.repack.dataset_name) # type:ignore return ( f".{dataset_name}" f".{self.noise_scheduler_name}{self.train_diffusion_steps}" f".{shorten(self.train_cfg.lr_scheduler_type)}" f".lr{self.train_cfg.learning_rate:.1e}" ) def prepare(self, accelerator: Accelerator) -> DataLoader: from psi.data.dataset import MixtureDataset # NOTE: if use deepspeed, model, optimizer, lr_scheduler must be prepared together !!! # print("L301", self.model.action_header.time_ins_embed.out_net[0].weight.dtype) # debug self.model, self.optimizer, self.lr_scheduler = accelerator.prepare( self.model, self.optimizer, self.lr_scheduler ) # self.model.action_header.to(torch.float32) # HACK keep action header in fp32 # print("L307", self.model.module.action_header.time_ins_embed.out_net[0].weight.dtype) # debug if not ( isinstance(self.train_dataset, MixtureDataset) and getattr(self.data_cfg, "sampler", None) is not None ): # dont prepare dataloader for mixed dataset as custom sampler is used self.train_dataloader = accelerator.prepare(self.train_dataloader) # Gradient Checkpoint Setup for tuned VLM backbones. DDP needs this too; # otherwise activation memory dominates once the language backbone is unfrozen. if self.model_cfg.gradient_checkpointing and self.model_cfg.tune_vlm: if self.train_cfg.data_parallel == "deepspeed": assert "DeepSpeedEngine" in self.model.__class__.__name__, "deepspeed is not properly initialized!" model_for_gc = self.model else: model_for_gc = accelerator.unwrap_model(self.model) vlm_model = model_for_gc.vlm_model if hasattr(vlm_model, "config"): vlm_model.config.use_cache = False if hasattr(vlm_model, "gradient_checkpointing_enable"): try: vlm_model.gradient_checkpointing_enable( gradient_checkpointing_kwargs={"use_reentrant": False} ) except TypeError: vlm_model.gradient_checkpointing_enable() if hasattr(vlm_model, "enable_input_require_grads"): vlm_model.enable_input_require_grads() else: input_embeddings = vlm_model.get_input_embeddings() if not getattr(input_embeddings, "_psi_input_require_grads_hook", False): def make_inputs_require_grad(module, input, output): output.requires_grad_(True) input_embeddings.register_forward_hook(make_inputs_require_grad) input_embeddings._psi_input_require_grads_hook = True overwatch.info("VLM gradient checkpointing enabled") val_dataloader = getattr(self, "val_dataloader", None) if isinstance(val_dataloader, dict): self.val_dataloader = { task_name: accelerator.prepare(dataloader) for task_name, dataloader in val_dataloader.items() } elif val_dataloader is not None: # not using if self.val_dataloader to avoid DataLoader.__len__() being called on iterable dataset self.val_dataloader = accelerator.prepare(self.val_dataloader) if self.cfg.train.overfit_single_batch: overwatch.warning("Overfitting a single batch: reusing first batch every step. set cfg.data.image_aug = False for true memorization.") first_batch = next(iter(self.train_dataloader)) class SingleBatchLoader: def __iter__(self): while True: yield first_batch def __len__(self): return 1 self.train_dataloader = SingleBatchLoader() self.val_dataloader = SingleBatchLoader() self.accelerator = accelerator return self.train_dataloader # type: ignore def training_step( self, batch: dict[str, Union[torch.Tensor, Any]], ) -> tuple[bool, dict[str, Any]]: # <modify_audio_grad_accum> 音频整段输入训练使用梯度累积;非sync步不会进入grad_norm计算, # 这里先给日志字段默认值,避免第一轮accumulation访问未初始化属性。 self._grad_norm_act = torch.tensor(0.0, device=self.device) if self.train_cfg.data_parallel == "deepspeed": gac = contextlib.nullcontext() else: # otherwise gradient accumulation is managed by deepspeed gac = self.accelerator.accumulate(self.model) with gac: with self.accelerator.autocast(): batch = move_to_device(batch, self.device) # type: ignore losses = self.forward_and_loss(self.model, batch) self.accelerator.backward(losses["loss"]) # overwatch.debug(self.global_step, self.device, batch["dataset_name"]) had_nonfinite_grad = False grad_norm_value = 0.0 if self.accelerator.sync_gradients: # <modify>+梯度保护:在 norm clip 前先清理 nan/inf,并做逐元素 value clip。 had_nonfinite_grad = self._protect_gradients() if self.train_cfg.max_grad_norm is not None: self._grad_norm_act = self.accelerator.clip_grad_norm_( self.model.parameters(), self.cfg.train.max_grad_norm ) else: self._grad_norm_act = self.get_total_grad_norm(self.model.parameters()) grad_norm_value = self._grad_norm_act.item() if torch.is_tensor(self._grad_norm_act) else float(self._grad_norm_act or 0.0) if self.train_cfg.data_parallel == "ddp": # NOTE: It is normal that param.grad is None for deepspeed and fsdp for name, param in self.model.named_parameters(): if param.requires_grad and param.grad is None: overwatch.critical(f"[Unused] {name} did not receive a gradient.") # <modify>+梯度保护:如果 loss / grad_norm / grad 本身已经出现非有限值,就跳过本次参数更新。 loss_value = losses["loss"].detach().float() should_skip_update = False if self.train_cfg.skip_step_on_nonfinite_grad: should_skip_update = ( (not torch.isfinite(loss_value).item()) or (not np.isfinite(grad_norm_value)) or had_nonfinite_grad ) # 原逻辑: # self.optimizer.step() # self.lr_scheduler.step() # self.optimizer.zero_grad() if should_skip_update: if self.accelerator.is_main_process: overwatch.warning( f"[modify_grad_protect] skip optimizer step, " f"loss={loss_value.item():.6f}, grad_norm={grad_norm_value}, " f"lr={self.lr:.6e}, had_nonfinite_grad={had_nonfinite_grad}" ) self.optimizer.zero_grad() else: self.optimizer.step() self.lr_scheduler.step() self.optimizer.zero_grad() # Log batch["raw_image"] to wandb every 100 steps if ( hasattr(self, "global_step") and self.global_step % self.cfg.log.log_freq == 0 and "raw_images" in batch and self.accelerator.is_main_process and self.cfg.log.report_to == "wandb" ): raw_imgs = batch["raw_images"] # Support both torch.Tensor and numpy if isinstance(raw_imgs, torch.Tensor): raw_imgs = raw_imgs.detach().cpu().numpy() # Log up to 4 images for visualization: concat them horizontally into one image img_arrays = [] for i in range(min(10, len(raw_imgs))): img = raw_imgs[i][0] # t=0 img_arrays.append(img) # Images are assumed to have same shape and 3 channels; concat directly concat_img = np.concatenate(img_arrays, axis=1) wandb.log({"raw_images": [wandb.Image(concat_img, caption=f"raw images {self.global_step}")]}, step=self.global_step) self.train_loss_tracker = step_loss = self.accelerator.gather(losses["loss"]).mean().item() # type: ignore # <modify>+70D分组loss日志:把forward里返回的loss_group/*一起写入wandb。 loss_logs = {} for loss_name, loss_value in losses.items(): if not loss_name.startswith("loss_group/"): continue loss_logs[loss_name] = self.accelerator.gather(loss_value.detach()).mean().item() # type: ignore loss_logs.update(self._batch_task_logs(losses)) return (self.accelerator.sync_gradients, { "lr_act": self.lr, "grad_norm_act": self._grad_norm_act, # type: ignore "loss": step_loss, **loss_logs, }) @torch.no_grad() def inference(self, eval_model, repacked_batch): traj2ds = repacked_batch["traj2ds"] if "traj2ds" in repacked_batch else None # (B, 3, H, W) obs = repacked_batch["states"] # (B,1,M) # imgs_in = {"cam0": view_features} # views # TODO support multi-view bsz = obs.shape[0] action_samples = torch.randn( bsz, self.ac_chunk, self.ac_dim, device=self.device ) self.noise_scheduler.set_timesteps(self.eval_diffusion_steps) for timestep in self.noise_scheduler.timesteps: batched_timestep = timestep.expand(bsz).to(self.device) model_pred = eval_model( input_ids=repacked_batch["input_ids"],##### attention_mask=repacked_batch["attention_mask"], # vlm related # pixel_values=repacked_batch["pixel_values"], # image_grid_thw=repacked_batch["image_grid_thw"], #### <modify> 改成可选读取 pixel_values=repacked_batch.get("pixel_values", None), image_grid_thw=repacked_batch.get("image_grid_thw", None), # <modify_audio> 纯音频Qwen-Omni输入透传;没有音频键时保持原text/image行为。 input_features=repacked_batch.get("input_features", None), feature_attention_mask=repacked_batch.get("feature_attention_mask", None), audio_feature_lengths=repacked_batch.get("audio_feature_lengths", None), pixel_values_videos=repacked_batch.get("pixel_values_videos", None), video_grid_thw=repacked_batch.get("video_grid_thw", None), video_second_per_grid=repacked_batch.get("video_second_per_grid", None), action_samples=action_samples, # (B,Tp,Da) states=obs, timestep=batched_timestep, traj2ds=traj2ds, progress=repacked_batch.get("progress", None), future_progress=repacked_batch.get("future_progress", None), return_dict=True, ).action action_samples = self.noise_scheduler.step( model_output=model_pred, timestep=timestep, sample=action_samples # type: ignore ).prev_sample # type: ignore return action_samples def save_checkpoint(self, global_step: int) -> str | None: save_dir = os.path.join(self.project_dir, "checkpoints") os.makedirs(save_dir, exist_ok=True) ckpt_dir = os.path.join(save_dir, f"ckpt_{global_step}") # <modify> 不再额外保存 save_model 生成的分片权重,避免每个ckpt重复占用约10G存储 # self.accelerator.save_model(self.model, ckpt_dir) return super().save_checkpoint(global_step) def evaluate(self) -> dict[str, float] | None: accelerator = self.accelerator global_step = self.global_step eval_model = self.unwrap_model() val_dataloaders = self.val_dataloader if isinstance(val_dataloaders, dict): val_dataloader_items = list(val_dataloaders.items()) else: val_name = str(getattr(self.data_cfg, "val_task", "val")) val_dataloader_items = [(val_name, val_dataloaders)] all_val_loss_list = [] val_task_logs: dict[str, float] = {} task_loss_sums = torch.zeros(len(self.TASK_FAMILY_TO_ID), device=self.device, dtype=torch.float32) task_loss_counts = torch.zeros(len(self.TASK_FAMILY_TO_ID), device=self.device, dtype=torch.float32) template_loss_sums = torch.zeros(len(self.TEMPLATE_TO_ID), device=self.device, dtype=torch.float32) template_loss_counts = torch.zeros(len(self.TEMPLATE_TO_ID), device=self.device, dtype=torch.float32) for val_name, val_dataloader in val_dataloader_items: total_val_batches = ( len(val_dataloader) if self.train_cfg.val_num_batches == -1 else min(self.train_cfg.val_num_batches, len(val_dataloader)) ) if total_val_batches <= 0: continue val_progress_bar = tqdm( val_dataloader, total=total_val_batches, disable=not accelerator.is_local_main_process, position=1, leave=False, ) val_progress_bar.set_description(f"Eval {val_name} at global step {global_step}") val_loss_list = [] for val_step, val_batch in enumerate(val_progress_bar): val_batch: dict = move_to_device(batch_str_to_tensor(val_batch), self.device) # type: ignore with accelerator.autocast(): val_loss = self.forward_and_loss(eval_model, val_batch) gathered_loss = accelerator.gather(val_loss["loss"]) val_loss_list.append(gathered_loss) all_val_loss_list.append(gathered_loss) task_loss_sums += val_loss.get("_task_loss_sums", torch.zeros_like(task_loss_sums)).detach().float() task_loss_counts += val_loss.get("_task_loss_counts", torch.zeros_like(task_loss_counts)).detach().float() template_loss_sums += val_loss.get("_template_loss_sums", torch.zeros_like(template_loss_sums)).detach().float() template_loss_counts += val_loss.get("_template_loss_counts", torch.zeros_like(template_loss_counts)).detach().float() if val_step + 1 >= total_val_batches: if accelerator.is_local_main_process: val_progress_bar.close() if hasattr(val_dataloader, "end"): val_dataloader.end() # type: ignore break if val_loss_list: val_task_logs[f"loss_val_task/{val_name}"] = torch.cat(val_loss_list).mean().item() if not all_val_loss_list: return None avg_val_loss = torch.cat(all_val_loss_list).mean().item() task_loss_sums_all = accelerator.gather(task_loss_sums).reshape(-1, len(self.TASK_FAMILY_TO_ID)).sum(dim=0) task_loss_counts_all = accelerator.gather(task_loss_counts).reshape(-1, len(self.TASK_FAMILY_TO_ID)).sum(dim=0) template_loss_sums_all = accelerator.gather(template_loss_sums).reshape(-1, len(self.TEMPLATE_TO_ID)).sum(dim=0) template_loss_counts_all = accelerator.gather(template_loss_counts).reshape(-1, len(self.TEMPLATE_TO_ID)).sum(dim=0) task_logs = {} for task_name, task_id in self.TASK_FAMILY_TO_ID.items(): if task_loss_counts_all[task_id].item() > 0: task_logs[f"loss_task/{task_name}"] = ( task_loss_sums_all[task_id] / task_loss_counts_all[task_id] ).item() for template_name, template_id in self.TEMPLATE_TO_ID.items(): if template_loss_counts_all[template_id].item() > 0: task_logs[f"loss_template/{template_name}"] = ( template_loss_sums_all[template_id] / template_loss_counts_all[template_id] ).item() # <modify>+eval日志精简:原来的action L1指标不再返回,wandb中eval下只保留eval/loss。 # 原逻辑: # action_l1_err_list = np.concatenate(action_l1_err_list, axis=0) # (len_val_dataset*Ta, Da) # action_l1_err_list_denormed = self.maxmin.denormalize_L1_action_err( action_l1_err_list) # type: ignore # # # action L1 errors # avg_action_errors_denormed = action_l1_err_list_denormed.mean(0) # (Da,) NOTE only if the error is L1 (linear) # labels_denormed = [ # "hand_left_l1", # "hand_right_l1", # "arm_left_l1", # "arm_right_l1", # ] # avg_lr_action_err_denormed = np.split( # avg_action_errors_denormed, [7, 14, 21], axis=-1 # ) # l1_action_errs = [ # float(np.mean(x)) for x in avg_lr_action_err_denormed # ] # log metrics return { "loss": avg_val_loss, **val_task_logs, **task_logs, } def _forward_and_loss_single(self, model, batch) -> dict[str, torch.Tensor]: bsz, *_, h, w = batch["actions"].shape if self.model_cfg.rtc and self.noise_scheduler_name != "flow": raise ValueError("RTC training in posttrain_v1 currently supports flow scheduler only") # for debugging purposes # gen = torch.Generator(device=self.device) # gen.manual_seed(42) # or any fixed seed you want gen = None if self.noise_scheduler_name == "ddpm": timesteps = torch.randint( low=0, high=self.train_diffusion_steps, size=(bsz,), device=self.device ).long() elif self.noise_scheduler_name == "flow": # For reproducibility/debugging sigmas = torch.rand((bsz,), device=self.device, generator=gen).float() * 0.95 + 0.05 timesteps = sigmas * self.train_diffusion_steps # <modify> 支持RTC训练:随机选择prefix delay,并把prefix位置设成clean action条件 # 原逻辑:sigmas = sigmas.view(-1, 1, 1) if self.model_cfg.rtc: delay = torch.randint( low=0, high=self.model_cfg.max_delay, size=(bsz,), device=self.device, dtype=torch.long, generator=gen, ) prefix_mask = torch.arange(batch["actions"].shape[1], device=self.device)[None, :] < delay[:, None] sigmas = torch.where( prefix_mask, torch.tensor(0.0, device=self.device, dtype=sigmas.dtype), sigmas[:, None], ) timesteps = sigmas * self.train_diffusion_steps else: sigmas = sigmas.view(-1, 1, 1) else: raise ValueError(f"Unknown noise scheduler: {self.noise_scheduler_name}") # noise_action = torch.randn_like(batch["actions"]) noise_action = torch.randn(batch["actions"].shape, device=self.device, generator=gen) if self.noise_scheduler_name == "ddpm": noisy_actions = self.noise_scheduler.add_noise( batch["actions"], noise_action, timesteps # type: ignore ) target_action = noise_action # ddpm epsilon-prediction elif self.noise_scheduler_name == "flow": # <modify> RTC时sigmas是(B,Tp),普通flow时是(B,1,1),这里统一扩到actions维度 # 原逻辑:sigmas_action = sigmas.view(-1, 1, 1) sigmas_action = sigmas # type: ignore while len(sigmas_action.shape) < len(batch["actions"].shape): sigmas_action = sigmas_action.unsqueeze(-1) noisy_actions = (1 - sigmas_action) * batch["actions"] + sigmas_action * noise_action target_action = noise_action - batch["actions"] # flow-matching velocity prediction else: raise ValueError model_output = model( input_ids=batch["input_ids"],##### attention_mask=batch["attention_mask"], # vlm related # pixel_values=batch["pixel_values"], # image_grid_thw=batch["image_grid_thw"], #### <modify> 改成可选输入 pixel_values=batch.get("pixel_values", None), image_grid_thw=batch.get("image_grid_thw", None), # <modify_audio> 纯音频Qwen-Omni输入透传;没有音频键时保持原text/image行为。 input_features=batch.get("input_features", None), feature_attention_mask=batch.get("feature_attention_mask", None), audio_feature_lengths=batch.get("audio_feature_lengths", None), pixel_values_videos=batch.get("pixel_values_videos", None), video_grid_thw=batch.get("video_grid_thw", None), video_second_per_grid=batch.get("video_second_per_grid", None), action_samples=noisy_actions, # (B,Tp,Da) states=batch["states"], # (B,1,M) timestep=timesteps, traj2ds=batch["traj2ds"] if "traj2ds" in batch else None, # (B, C, 3, H, W) progress=batch.get("progress", None), future_progress=batch.get("future_progress", None), return_dict=True, ) action_pred = model_output.action loss_action = F.mse_loss( action_pred.float(), target_action.float(), reduction="none" ) # (B, Tp, Da) # for post pre training, we only care about the overall loss # """ # sum(1) -> to keep gradient consistent when training with different action chunks # mean(0) -> to average over the batch # sum() -> to sum over the weighted loss of each action dimension # """ if "actions_mask" not in batch: batch["actions_mask"] = torch.ones_like(batch["actions"]) # <modify> RTC训练时prefix是已知旧动作,只在postfix上计算loss # 原逻辑:loss_action = (loss_action * batch["actions_mask"]).sum(1) action_mask = batch["actions_mask"] if self.model_cfg.rtc: postfix_mask = (~prefix_mask)[:, :, None].float() # type: ignore action_mask = action_mask * postfix_mask loss_action = (loss_action * action_mask).sum(1) # (B, Da) # <modify>+70D分组loss日志:先保留逐维loss,再按原逻辑计算总loss。 loss_action_by_dim = loss_action.mean(0) # <modify>+70D分组loss权重:在原有 self.loss_w 的基础上,继续乘一层组权重。 # 原逻辑: # loss_action = (loss_action_by_dim * self.loss_w).sum() loss_action = (loss_action_by_dim * self.loss_w * self.loss_group_w).sum() group_losses = self._action_group_losses(loss_action_by_dim) return {"loss": loss_action, **group_losses} def forward_and_loss(self, model, batch) -> dict[str, torch.Tensor]: task_batches = batch.get("task_batches", None) if not isinstance(task_batches, dict): losses = self._forward_and_loss_single(model, batch) task_loss_sums, task_loss_counts, template_loss_sums, template_loss_counts = self._init_task_metric_tensors() self._accumulate_task_metrics( batch, losses["loss"], int(batch["actions"].shape[0]), task_loss_sums, task_loss_counts, template_loss_sums, template_loss_counts, ) return { **losses, "_task_loss_sums": task_loss_sums, "_task_loss_counts": task_loss_counts, "_template_loss_sums": template_loss_sums, "_template_loss_counts": template_loss_counts, } total_batch_size = sum(int(task_batch["actions"].shape[0]) for task_batch in task_batches.values()) if total_batch_size <= 0: raise ValueError("Empty mixed batch received in forward_and_loss") task_loss_sums, task_loss_counts, template_loss_sums, template_loss_counts = self._init_task_metric_tensors() total_loss = None merged_group_losses: dict[str, torch.Tensor] = {} for task_batch in task_batches.values(): task_batch_size = int(task_batch["actions"].shape[0]) task_weight = float(task_batch_size) / float(total_batch_size) task_losses = self._forward_and_loss_single(model, task_batch) weighted_loss = task_losses["loss"] * task_weight total_loss = weighted_loss if total_loss is None else total_loss + weighted_loss for loss_name, loss_value in task_losses.items(): if loss_name == "loss": continue merged_group_losses[loss_name] = ( loss_value * task_weight if loss_name not in merged_group_losses else merged_group_losses[loss_name] + loss_value * task_weight ) self._accumulate_task_metrics( task_batch, task_losses["loss"], task_batch_size, task_loss_sums, task_loss_counts, template_loss_sums, template_loss_counts, ) assert total_loss is not None return { "loss": total_loss, **merged_group_losses, "_task_loss_sums": task_loss_sums, "_task_loss_counts": task_loss_counts, "_template_loss_sums": template_loss_sums, "_template_loss_counts": template_loss_counts, }
from __future__ import annotations
import os
import json
import torch
import wandb
import pickle
import numpy as np
from tqdm import tqdm
from copy import copy, deepcopy
import contextlib
import torch.nn.functional as F
from typing import Dict, Optional, List, Union, Any, TYPE_CHECKING
from torch.utils.data import DataLoader
# <modify> 支持按 checkpoint 类型加载 Qwen3-VL 或 Qwen2.5-Omni
# 原代码:
# from transformers import Qwen2_5OmniThinkerForConditionalGeneration, Qwen2_5OmniProcessor #, Qwen2TokenizerFast, Qwen3VLProcessor
from transformers import (
AutoConfig,
AutoProcessor,
Qwen2_5OmniProcessor,
Qwen2_5OmniThinkerForConditionalGeneration,
Qwen3VLForConditionalGeneration,
)
from psi.trainers.qwen2_5omni_mixin import PaddedCollatorForTogether # <modify>
if TYPE_CHECKING:
from psi.config.config import TrainConfig #, LaunchConfig
from psi.config.model_psi0 import Psi0ModelConfig
from psi.config.transform import Psi0ModelTransform
from psi.config.data_seed import SeedDataConfig
from psi.config.data_mix import MixedDataConfig
from accelerate import Accelerator
# from utils.overwatch import initialize_overwatch
from psi.utils import (
initialize_overwatch,
shorten,
move_to_device,
batch_str_to_tensor
)
overwatch = initialize_overwatch(__name__)
# from .base import Trainer
from .trainer import Trainer, worker_init_fn
from psi.utils import flatten, shorten, move_to_device, rmse, seed_everything
from psi.models.psi0 import Psi0Model
class PosttrainV1Trainer(Trainer):
TASK_FAMILY_TO_ID = {"text": 0, "audio": 1, "video": 2}
TEMPLATE_TO_ID = {
"text": 0,
"audio_text": 1,
"audio_only": 2,
"audio_command_text": 3,
"audio_command_only": 4,
"video_text": 5,
"video_only": 6,
"video_clip_text": 7,
"video_clip_only": 8,
}
def __init__(self, cfg, device: torch.device):
super().__init__(cfg, device)
self.noise_scheduler_name = self.model_cfg.noise_scheduler
self.train_diffusion_steps = self.model_cfg.train_diffusion_steps
self.eval_diffusion_steps = self.model_cfg.eval_diffusion_steps
self.ac_chunk = self.model_cfg.action_chunk_size
self.ac_dim = self.model_cfg.action_dim
self.maxmin = self.data_cfg.transform.field
# 机械臂构型,按初始输入loss权重分配各关节
if self.model_cfg.action_dim == 7:
w_xyz, w_rpy, w_gripper = self.model_cfg.loss_w
self.loss_w = (
torch.from_numpy(np.array([w_xyz] * 3 + [w_rpy] * 3 + [w_gripper]))
.float()
.to(self.device)
) # (7,)
else:
# 全身构型,loss权重均分
self.loss_w = torch.tensor([1.0 / self.model_cfg.action_dim] * self.model_cfg.action_dim, dtype=torch.float32).to(self.device)
assert self.loss_w.sum() == 1.0, "Weights better sum to 1.0 to keep loss range consistent"
# <modify>+70D分组loss权重:在原有逐维均权之外,再叠加一层按动作组的可配权重。
self.loss_group_w = self._build_group_loss_weight()
# diffusion
if self.noise_scheduler_name == "ddpm":
from diffusers.schedulers.scheduling_ddim import DDIMScheduler
self.noise_scheduler = DDIMScheduler( # FIXME configure args
num_train_timesteps=self.train_diffusion_steps,
beta_start=0.0001,
beta_end=0.02,
beta_schedule="squaredcos_cap_v2",
clip_sample=True,
set_alpha_to_one=True,