from __future__ import annotations import os import json import torch import wandb import pickle import numpy as np from tqdm import tqdm from copy import copy, deepcopy import contextlib import torch.nn.functional as F from typing import Dict, Optional, List, Union, Any, TYPE_CHECKING from torch.utils.data import DataLoader from transformers import Qwen3VLForConditionalGeneration, AutoProcessor #, Qwen2TokenizerFast, Qwen3VLProcessor from psi.trainers.qwen3vl_mixin import PaddedCollatorForTogether if TYPE_CHECKING: from psi.config.config import TrainConfig #, LaunchConfig from psi.config.model_psi0 import Psi0ModelConfig from psi.config.transform import Psi0ModelTransform from psi.config.data_he import HERawDataConfig from psi.config.data_mix import MixedDataConfig from accelerate import Accelerator # from utils.overwatch import initialize_overwatch from psi.utils import ( initialize_overwatch, shorten, move_to_device, batch_str_to_tensor ) overwatch = initialize_overwatch(__name__) # from .base import Trainer from .trainer import Trainer, worker_init_fn from psi.utils import flatten, shorten, move_to_device, rmse, seed_everything from psi.models.psi0 import Psi0Model class PosttrainTrainer(Trainer): def __init__(self, cfg, device: torch.device): super().__init__(cfg, device) self.noise_scheduler_name = self.model_cfg.noise_scheduler self.train_diffusion_steps = self.model_cfg.train_diffusion_steps self.eval_diffusion_steps = self.model_cfg.eval_diffusion_steps self.ac_chunk = self.model_cfg.action_chunk_size self.ac_dim = self.model_cfg.action_dim self.maxmin = self.data_cfg.transform.field if self.model_cfg.action_dim == 7: w_xyz, w_rpy, w_gripper = self.model_cfg.loss_w self.loss_w = ( torch.from_numpy(np.array([w_xyz] * 3 + [w_rpy] * 3 + [w_gripper])) .float() .to(self.device) ) # (7,) else: self.loss_w = torch.tensor([1.0 / self.model_cfg.action_dim] * self.model_cfg.action_dim, dtype=torch.float32).to(self.device) assert self.loss_w.sum() == 1.0, "Weights better sum to 1.0 to keep loss range consistent" if self.noise_scheduler_name == "ddpm": from diffusers.schedulers.scheduling_ddim import DDIMScheduler self.noise_scheduler = DDIMScheduler( # FIXME configure args num_train_timesteps=self.train_diffusion_steps, beta_start=0.0001, beta_end=0.02, beta_schedule="squaredcos_cap_v2", clip_sample=True, set_alpha_to_one=True, steps_offset=0, prediction_type="epsilon", ) elif self.noise_scheduler_name == "flow": from diffusers.schedulers.scheduling_flow_match_euler_discrete import ( FlowMatchEulerDiscreteScheduler, ) self.noise_scheduler = FlowMatchEulerDiscreteScheduler( num_train_timesteps=self.train_diffusion_steps, # MUST be 1000 as per pretrained SD3 ) assert self.train_cfg.mixed_precision == "bf16", "keep qwen3vl in bf16 while training action header in fp32" # assert self.model_cfg.n_conditions == len(self.data_cfg.transform.repack.conditions), "inconsistent confs" # type: ignore @property def data_cfg(self) -> HERawDataConfig | MixedDataConfig: return self.cfg.data # type: ignore @property def model_cfg(self) -> Psi0ModelConfig: return self.cfg.model # type: ignore def init_qwen3vl_models(self): vlm_model = Qwen3VLForConditionalGeneration.from_pretrained( self.model_cfg.model_name_or_path, attn_implementation="flash_attention_2", dtype=torch.bfloat16 ) self.vlm_processor = AutoProcessor.from_pretrained(self.model_cfg.model_name_or_path) self.tokenizer = self.vlm_processor.tokenizer assert self.cfg.train.lora == False return vlm_model def init_models(self): # init pretrained vlm vlm_model = self.init_qwen3vl_models() self.model = Psi0Model( model_cfg=self.model_cfg, vlm_model=vlm_model, # dtype=self.dtype ) # print("L129", self.model.action_header.time_ins_embed.out_net[0].weight.dtype) # debug if self.train_cfg.data_parallel == "deepspeed": # HACK to set correct config for deepspeed self.model.config = vlm_model.config self.model.config.hidden_size = vlm_model.config.text_config.hidden_size # only support freeze vlm now assert self.model_cfg.tune_vlm == False for p in self.model.vlm_model.parameters(): p.requires_grad = False num_parameters = sum(p.numel() for p in self.model.parameters() if p.requires_grad) overwatch.info(f"Model has {num_parameters:,} trainable parameters") def create_optimizers(self): # Separate parameters into groups action_header_params = [] vlm_model_params = [] other_params = [] for name, param in self.model.named_parameters(): if name.startswith("action_header."): action_header_params.append(param) elif name.startswith("vlm_model."): if self.model_cfg.tune_vlm: vlm_model_params.append(param) else: other_params.append(param) param_groups = [] if action_header_params: param_groups.append({ 'params': action_header_params, 'lr': self.cfg.train.learning_rate, 'group_name': 'action_header' }) if vlm_model_params: param_groups.append({ 'params': vlm_model_params, 'lr': self.model_cfg.lang_backbone_lr, 'group_name': 'vlm_model' }) if other_params: param_groups.append({ 'params': other_params, 'lr': self.cfg.train.learning_rate, # Default group 'group_name': 'other' }) optimizer_kwargs = dict(self.cfg.train.lr_scheduler_kwargs) if self.cfg.train.optimizer_foreach is not None: optimizer_kwargs["foreach"] = self.cfg.train.optimizer_foreach self.optimizer = torch.optim.AdamW( param_groups, **optimizer_kwargs, # type: ignore ) def create_lr_schedulers( self, num_training_steps: int | None = None, optimizer: torch.optim.Optimizer | None = None, ): from transformers.optimization import get_scheduler assert num_training_steps is not None self.lr_scheduler = get_scheduler( self.cfg.train.lr_scheduler_type, num_training_steps=num_training_steps * self.world_size, optimizer=optimizer if optimizer is not None else self.optimizer, num_warmup_steps=self.num_warmup_steps * self.world_size, scheduler_specific_kwargs=self.cfg.train.scheduler_specific_kwargs, ) def create_optimizer_and_scheduler(self, num_training_steps: int | None = None): optimizer = self.create_optimizers() self.create_lr_schedulers( num_training_steps=num_training_steps, optimizer=optimizer ) def create_datasets(self): # TODO use parent impl. transform_kwargs=dict( vlm_processor=self.vlm_processor, ) self.train_dataset = self.data_cfg(split="train", transform_kwargs=transform_kwargs) self.val_dataset = self.data_cfg(split="val", transform_kwargs=transform_kwargs) return self.train_dataset, self.val_dataset def create_dataloaders(self, train_dataset, val_dataset): from psi.data.dataset import MixtureDataset if isinstance(self.data_cfg, MixedDataConfig): assert isinstance(self.train_dataset, MixtureDataset) if self.data_cfg.sampler == "batch_mixture": from psi.data.sampler import BatchMixtureSampler batch_sampler = BatchMixtureSampler( dataset_lens=[d.dataset_length for d in self.train_dataset.datasets], mixture_ratios=self.train_dataset.ratios, num_samples_per_epoch=len(self.train_dataset), batch_size=self.train_cfg.train_batch_size, ) elif self.data_cfg.sampler == "token_mixture": from psi.data.sampler import TokenMixtureSampler batch_sampler = TokenMixtureSampler( specs=self.train_dataset.specs, tokens_per_batch=self.data_cfg.tokens_per_device, num_batches_per_rank=self.num_steps_per_epoch, ) else: raise ValueError(f"Unknown sampler type: {self.data_cfg.sampler}") g = shuffle = batch_size = drop_last = None else: batch_sampler = None shuffle = True batch_size = self.train_cfg.train_batch_size drop_last = True g = torch.Generator() g.manual_seed(self.cfg.seed or 42) collator = PaddedCollatorForTogether( model_max_length=self.tokenizer.model_max_length, pad_token_id=self.tokenizer.pad_token_id, padding_side="right", ) if batch_sampler is not None: self.train_dataloader = DataLoader( train_dataset, batch_sampler=batch_sampler, collate_fn=collator, num_workers=1, worker_init_fn=worker_init_fn, persistent_workers=True ) self.train_sampler = batch_sampler # !!important else: self.train_dataloader = DataLoader( train_dataset, shuffle=shuffle, # batch_sampler=batch_sampler, generator=g, # batch_size=batch_size, # collate_fn=collator, num_workers=1, worker_init_fn=worker_init_fn, persistent_workers=True, drop_last=drop_last or False ) val_dataloader_kwargs = { "num_workers": 1, "drop_last": False, # "pin_memory": True, "persistent_workers": True, "shuffle": True, } # 1 small enough to fit im Mem. 2. no need distributed sampler self.val_dataloader = torch.utils.data.DataLoader( val_dataset, batch_size=self.train_cfg.val_batch_size, collate_fn=collator, **val_dataloader_kwargs, ) return self.train_dataloader, self.val_dataloader @property def task_run_name(self): dataset_name = shorten(self.data_cfg.transform.repack.dataset_name) # type:ignore return ( f".{dataset_name}" f".{self.noise_scheduler_name}{self.train_diffusion_steps}" f".{shorten(self.train_cfg.lr_scheduler_type)}" f".lr{self.train_cfg.learning_rate:.1e}" ) def prepare(self, accelerator: Accelerator) -> DataLoader: # NOTE: if use deepspeed, model, optimizer, lr_scheduler must be prepared together !!! # print("L301", self.model.action_header.time_ins_embed.out_net[0].weight.dtype) # debug self.model, self.optimizer, self.lr_scheduler = accelerator.prepare( self.model, self.optimizer, self.lr_scheduler ) # self.model.action_header.to(torch.float32) # HACK keep action header in fp32 # print("L307", self.model.module.action_header.time_ins_embed.out_net[0].weight.dtype) # debug if not (isinstance(self.data_cfg, MixedDataConfig) and self.data_cfg.sampler is not None): # dont prepare dataloader for mixed dataset as custom sampler is used self.train_dataloader = accelerator.prepare(self.train_dataloader) if self.train_cfg.data_parallel == "deepspeed": # Gradient Checkpoint Setup if self.model_cfg.gradient_checkpointing and self.model_cfg.tune_vlm: assert "DeepSpeedEngine" in self.model.__class__.__name__, "deepspeed is not properly initialized!" if hasattr(self.model, "enable_input_require_grads"): self.model.vlm_model.enable_input_require_grads() else: def make_inputs_require_grad(module, input, output): output.requires_grad_(True) self.model.vlm_model.get_input_embeddings().register_forward_hook(make_inputs_require_grad) val_dataloader = getattr(self, "val_dataloader", None) if val_dataloader is not None: # not using if self.val_dataloader to avoid DataLoader.__len__() being called on iterable dataset self.val_dataloader = accelerator.prepare(self.val_dataloader) if self.cfg.train.overfit_single_batch: overwatch.warning("Overfitting a single batch: reusing first batch every step. set cfg.data.image_aug = False for true memorization.") first_batch = next(iter(self.train_dataloader)) class SingleBatchLoader: def __iter__(self): while True: yield first_batch def __len__(self): return 1 self.train_dataloader = SingleBatchLoader() self.val_dataloader = SingleBatchLoader() self.accelerator = accelerator return self.train_dataloader # type: ignore def training_step( self, batch: dict[str, Union[torch.Tensor, Any]], ) -> tuple[bool, dict[str, Any]]: if self.train_cfg.data_parallel == "deepspeed": gac = contextlib.nullcontext() else: # otherwise gradient accumulation is managed by deepspeed gac = self.accelerator.accumulate(self.model) with gac: with self.accelerator.autocast(): batch = move_to_device(batch, self.device) # type: ignore losses = self.forward_and_loss(self.model, batch) self.accelerator.backward(losses["loss"]) # overwatch.debug(self.global_step, self.device, batch["dataset_name"]) if self.accelerator.sync_gradients: if self.train_cfg.max_grad_norm is not None: self._grad_norm_act = self.accelerator.clip_grad_norm_( self.model.parameters(), self.cfg.train.max_grad_norm ) else: self._grad_norm_act = self.get_total_grad_norm(self.model.parameters()) if self.train_cfg.data_parallel == "ddp": # NOTE: It is normal that param.grad is None for deepspeed and fsdp for name, param in self.model.named_parameters(): if param.requires_grad and param.grad is None: overwatch.critical(f"[Unused] {name} did not receive a gradient.") self.optimizer.step() self.lr_scheduler.step() self.optimizer.zero_grad() # Log batch["raw_image"] to wandb every 100 steps if ( hasattr(self, "global_step") and self.global_step % self.cfg.log.log_freq == 0 and "raw_images" in batch and self.accelerator.is_main_process and self.cfg.log.report_to == "wandb" ): raw_imgs = batch["raw_images"] # Support both torch.Tensor and numpy if isinstance(raw_imgs, torch.Tensor): raw_imgs = raw_imgs.detach().cpu().numpy() # Log up to 4 images for visualization: concat them horizontally into one image img_arrays = [] for i in range(min(10, len(raw_imgs))): img = raw_imgs[i][0] # t=0 img_arrays.append(img) # Images are assumed to have same shape and 3 channels; concat directly concat_img = np.concatenate(img_arrays, axis=1) wandb.log({"raw_images": [wandb.Image(concat_img, caption=f"raw images {self.global_step}")]}, step=self.global_step) self.train_loss_tracker = step_loss = self.accelerator.gather(losses["loss"]).mean().item() # type: ignore return (self.accelerator.sync_gradients, { "lr_act": self.lr, "grad_norm_act": self._grad_norm_act, # type: ignore "loss": step_loss }) @torch.no_grad() def inference(self, eval_model, repacked_batch): traj2ds = repacked_batch["traj2ds"] if "traj2ds" in repacked_batch else None # (B, 3, H, W) obs = repacked_batch["states"] # (B,1,M) # imgs_in = {"cam0": view_features} # views # TODO support multi-view bsz = obs.shape[0] action_samples = torch.randn( bsz, self.ac_chunk, self.ac_dim, device=self.device ) self.noise_scheduler.set_timesteps(self.eval_diffusion_steps) for timestep in self.noise_scheduler.timesteps: batched_timestep = timestep.expand(bsz).to(self.device) model_pred = eval_model( input_ids=repacked_batch["input_ids"],##### attention_mask=repacked_batch["attention_mask"], # vlm related pixel_values=repacked_batch["pixel_values"], image_grid_thw=repacked_batch["image_grid_thw"], #### action_samples=action_samples, # (B,Tp,Da) states=obs, timestep=batched_timestep, traj2ds=traj2ds, return_dict=True, ).action action_samples = self.noise_scheduler.step( model_output=model_pred, timestep=timestep, sample=action_samples # type: ignore ).prev_sample # type: ignore return action_samples def save_checkpoint(self, global_step: int) -> str | None: save_dir = os.path.join(self.project_dir, "checkpoints") os.makedirs(save_dir, exist_ok=True) ckpt_dir = os.path.join(save_dir, f"ckpt_{global_step}") self.accelerator.save_model(self.model, ckpt_dir) return super().save_checkpoint(global_step) def evaluate(self) -> dict[str, float] | None: accelerator = self.accelerator global_step = self.global_step eval_model = self.unwrap_model() total_val_batches = ( len(self.val_dataloader) if self.train_cfg.val_num_batches == -1 else min(self.train_cfg.val_num_batches, len(self.val_dataloader)) ) val_progress_bar = tqdm( self.val_dataloader, total=total_val_batches, disable=not accelerator.is_local_main_process, position=1, leave=False, ) val_progress_bar.set_description(f"Eval at global step {global_step}") val_loss_list = [] action_l1_err_list = [] for val_step, val_batch in enumerate(val_progress_bar): val_batch: dict = move_to_device(batch_str_to_tensor(val_batch), self.device) # type: ignore # mask = val_batch["mask"] mask = torch.ones_like(val_batch["actions"]) # FIXME gt_actions = val_batch["actions"] # (B, Tp, Da) # gt_actions = self.data_cfg.data_transforms.normalize_action(repacked_batch[1]) # Tp -> predicted action horizon, Da -> action dim B, Tp, Da = gt_actions.shape # validation loss with accelerator.autocast(): val_loss = self.forward_and_loss(eval_model, val_batch) val_loss_list.append(accelerator.gather(val_loss["loss"])) # action prediction loss pred_actions = self.inference(eval_model, val_batch) err_action_l1 = pred_actions - gt_actions # (B, Tp, Da) err_action_l1_all = accelerator.gather( err_action_l1[:, :Tp].contiguous() ) err_action_masks_all = accelerator.gather(mask[:, :Tp].contiguous()) err_action_l1 = err_action_l1_all[ err_action_masks_all.to(torch.bool) # type: ignore ].abs() action_l1_err_list.append(err_action_l1.reshape(-1, Da).float().cpu().numpy()) # (B*world_size*Ta, 7) if val_step + 1 >= total_val_batches: if accelerator.is_local_main_process: val_progress_bar.close() if hasattr(self.val_dataloader, "end"): self.val_dataloader.end() # type: ignore break avg_val_loss = torch.cat(val_loss_list).mean().item() action_l1_err_list = np.concatenate(action_l1_err_list, axis=0) # (len_val_dataset*Ta, Da) action_l1_err_list_denormed = self.maxmin.denormalize_L1_action_err( action_l1_err_list) # type: ignore # action L1 errors avg_action_errors_denormed = action_l1_err_list_denormed.mean(0) # (Da,) NOTE only if the error is L1 (linear) labels_denormed = [ "hand_left_l1", "hand_right_l1", "arm_left_l1", "arm_right_l1", ] avg_lr_action_err_denormed = np.split( avg_action_errors_denormed, [7, 14, 21], axis=-1 ) l1_action_errs = [ float(np.mean(x)) for x in avg_lr_action_err_denormed ] # log metrics return { "loss": avg_val_loss, **dict(zip(labels_denormed, l1_action_errs)) } def forward_and_loss(self, model, batch) -> dict[str, torch.Tensor]: bsz, *_, h, w = batch["actions"].shape # for debugging purposes # gen = torch.Generator(device=self.device) # gen.manual_seed(42) # or any fixed seed you want gen = None if self.noise_scheduler_name == "ddpm": timesteps = torch.randint( low=0, high=self.train_diffusion_steps, size=(bsz,), device=self.device ).long() elif self.noise_scheduler_name == "flow": # For reproducibility/debugging sigmas = torch.rand((bsz,), device=self.device, generator=gen).float() timesteps = sigmas * self.train_diffusion_steps sigmas = sigmas.view(-1, 1, 1) else: raise ValueError(f"Unknown noise scheduler: {self.noise_scheduler_name}") # noise_action = torch.randn_like(batch["actions"]) noise_action = torch.randn(batch["actions"].shape, device=self.device, generator=gen) if self.noise_scheduler_name == "ddpm": noisy_actions = self.noise_scheduler.add_noise( batch["actions"], noise_action, timesteps # type: ignore ) target_action = noise_action # ddpm epsilon-prediction elif self.noise_scheduler_name == "flow": sigmas_action = sigmas.view(-1, 1, 1) # B, Tp, action_dim # type: ignore noisy_actions = (1 - sigmas_action) * batch["actions"] + sigmas_action * noise_action target_action = noise_action - batch["actions"] # flow-matching velocity prediction else: raise ValueError model_output = model( input_ids=batch["input_ids"],##### attention_mask=batch["attention_mask"], # vlm related pixel_values=batch["pixel_values"], image_grid_thw=batch["image_grid_thw"], #### action_samples=noisy_actions, # (B,Tp,Da) states=batch["states"], # (B,1,M) timestep=timesteps, traj2ds=batch["traj2ds"] if "traj2ds" in batch else None, # (B, C, 3, H, W) return_dict=True, ) action_pred = model_output.action loss_action = F.mse_loss( action_pred.float(), target_action.float(), reduction="none" ) # (B, Tp, Da) # for post pre training, we only care about the overall loss # """ # sum(1) -> to keep gradient consistent when training with different action chunks # mean(0) -> to average over the batch # sum() -> to sum over the weighted loss of each action dimension # """ if "actions_mask" not in batch: batch["actions_mask"] = torch.ones_like(batch["actions"]) loss_action = (loss_action * batch["actions_mask"]).sum(1) # (B, Da) loss_action = (loss_action.mean(0) * self.loss_w).sum() return {"loss": loss_action}