from __future__ import annotations from dotenv import load_dotenv assert load_dotenv(), "Failed to load .env file. Make sure it exists and is properly formatted." import importlib import os import sys from pathlib import Path from typing import Any import torch import tyro from tqdm import tqdm from psi.data.dataset import MixtureDataset from psi.trainers import Trainer from psi.trainers.qwen2_5omni_mixin import PaddedCollatorForTogether from psi.utils import batch_str_to_tensor, initialize_overwatch, move_to_device, seed_everything overwatch = initialize_overwatch(__name__) def _dataset_items(dataset: Any, name: str): if isinstance(dataset, MixtureDataset): for idx, sub_dataset in enumerate(dataset.datasets): yield from _dataset_items(sub_dataset, f"{name}_mix{idx}") return if isinstance(dataset, dict): for key, sub_dataset in dataset.items(): yield from _dataset_items(sub_dataset, f"{name}_{key}") return yield name, dataset def _save_batch(trainer, collator, instances, split_name: str, save_dtype: torch.dtype) -> int: batch = batch_str_to_tensor(collator(instances)) batch = move_to_device(batch, trainer.device) with torch.no_grad(): hidden = trainer.model.encode_motion_hidden(batch["actions"]) saved = 0 for i in range(int(batch["input_ids"].shape[0])): cache_path, latent_hash = trainer._latent_cache_key(batch, i) cache_path.parent.mkdir(parents=True, exist_ok=True) obj = { "hidden_states": hidden[i].detach().to(dtype=save_dtype).cpu(), "latent_hash": latent_hash, "split": split_name, "task_name": trainer._batch_item(batch, "task_name", i, default="unknown"), "motion_dir": trainer._batch_item(batch, "motion_dir", i, default=""), "video_path": trainer._batch_item(batch, "video_path", i, default=""), "frame_id": str(trainer._batch_item(batch, "frame_id", i, default="")), } torch.save(obj, cache_path) saved += 1 return saved def save_cache(config): if config.seed: seed_everything(config.seed) cache_dir = getattr(config.model, "motion_latent_cache_dir", None) if not cache_dir: raise ValueError("--model.motion-latent-cache-dir must be set") teacher_checkpoint = getattr(config.model, "motion_teacher_checkpoint", None) if not teacher_checkpoint: raise ValueError("--model.motion-teacher-checkpoint must point to the stage-1 checkpoint") if torch.cuda.is_available(): torch.cuda.set_device(0) device = torch.device("cuda:0") else: device = torch.device("cpu") trainer = Trainer.instantiate(config, device) trainer.init_models() trainer.model.motion_encoder.to(device) trainer.model.motion_encoder.eval() train_dataset, val_dataset = trainer.create_datasets() datasets = [("train", train_dataset)] if val_dataset is not None: datasets.append(("val", val_dataset)) collator = PaddedCollatorForTogether( model_max_length=trainer.tokenizer.model_max_length, pad_token_id=trainer.tokenizer.pad_token_id, padding_side="right", ) batch_size = max(1, int(config.train.val_batch_size)) save_dtype = torch.bfloat16 if config.train.mixed_precision == "bf16" else torch.float32 total_saved = 0 for split_name, split_dataset in datasets: for dataset_name, dataset in _dataset_items(split_dataset, split_name): pending = [] progress = tqdm( range(len(dataset)), desc=f"Cache {dataset_name}", disable=not overwatch.is_rank_zero(), ) for idx in progress: pending.append(dataset[idx]) if len(pending) >= batch_size: total_saved += _save_batch(trainer, collator, pending, split_name, save_dtype) pending = [] progress.set_postfix(saved=total_saved) if pending: total_saved += _save_batch(trainer, collator, pending, split_name, save_dtype) cache_root = Path(str(cache_dir)) manifest_path = cache_root / "manifest.txt" if overwatch.is_rank_zero(): manifest_path.parent.mkdir(parents=True, exist_ok=True) with manifest_path.open("w") as f: f.write(f"teacher_checkpoint={teacher_checkpoint}\n") f.write(f"saved={total_saved}\n") overwatch.info(f"Saved {total_saved} motion latent cache entries to {cache_root}") def main(): module = importlib.import_module(f"psi.config.train.{sys.argv[1]}") config_cls = getattr(module, "DynamicLaunchConfig") config = tyro.cli( config_cls, config=(tyro.conf.ConsolidateSubcommandArgs,), args=sys.argv[2:], ) save_cache(config) if __name__ == "__main__": main()