File size: 4,953 Bytes
3c8c0d5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | from __future__ import annotations
from dotenv import load_dotenv
assert load_dotenv(), "Failed to load .env file. Make sure it exists and is properly formatted."
import importlib
import os
import sys
from pathlib import Path
from typing import Any
import torch
import tyro
from tqdm import tqdm
from psi.data.dataset import MixtureDataset
from psi.trainers import Trainer
from psi.trainers.qwen2_5omni_mixin import PaddedCollatorForTogether
from psi.utils import batch_str_to_tensor, initialize_overwatch, move_to_device, seed_everything
overwatch = initialize_overwatch(__name__)
def _dataset_items(dataset: Any, name: str):
if isinstance(dataset, MixtureDataset):
for idx, sub_dataset in enumerate(dataset.datasets):
yield from _dataset_items(sub_dataset, f"{name}_mix{idx}")
return
if isinstance(dataset, dict):
for key, sub_dataset in dataset.items():
yield from _dataset_items(sub_dataset, f"{name}_{key}")
return
yield name, dataset
def _save_batch(trainer, collator, instances, split_name: str, save_dtype: torch.dtype) -> int:
batch = batch_str_to_tensor(collator(instances))
batch = move_to_device(batch, trainer.device)
with torch.no_grad():
hidden = trainer.model.encode_motion_hidden(batch["actions"])
saved = 0
for i in range(int(batch["input_ids"].shape[0])):
cache_path, latent_hash = trainer._latent_cache_key(batch, i)
cache_path.parent.mkdir(parents=True, exist_ok=True)
obj = {
"hidden_states": hidden[i].detach().to(dtype=save_dtype).cpu(),
"latent_hash": latent_hash,
"split": split_name,
"task_name": trainer._batch_item(batch, "task_name", i, default="unknown"),
"motion_dir": trainer._batch_item(batch, "motion_dir", i, default=""),
"video_path": trainer._batch_item(batch, "video_path", i, default=""),
"frame_id": str(trainer._batch_item(batch, "frame_id", i, default="")),
}
torch.save(obj, cache_path)
saved += 1
return saved
def save_cache(config):
if config.seed:
seed_everything(config.seed)
cache_dir = getattr(config.model, "motion_latent_cache_dir", None)
if not cache_dir:
raise ValueError("--model.motion-latent-cache-dir must be set")
teacher_checkpoint = getattr(config.model, "motion_teacher_checkpoint", None)
if not teacher_checkpoint:
raise ValueError("--model.motion-teacher-checkpoint must point to the stage-1 checkpoint")
if torch.cuda.is_available():
torch.cuda.set_device(0)
device = torch.device("cuda:0")
else:
device = torch.device("cpu")
trainer = Trainer.instantiate(config, device)
trainer.init_models()
trainer.model.motion_encoder.to(device)
trainer.model.motion_encoder.eval()
train_dataset, val_dataset = trainer.create_datasets()
datasets = [("train", train_dataset)]
if val_dataset is not None:
datasets.append(("val", val_dataset))
collator = PaddedCollatorForTogether(
model_max_length=trainer.tokenizer.model_max_length,
pad_token_id=trainer.tokenizer.pad_token_id,
padding_side="right",
)
batch_size = max(1, int(config.train.val_batch_size))
save_dtype = torch.bfloat16 if config.train.mixed_precision == "bf16" else torch.float32
total_saved = 0
for split_name, split_dataset in datasets:
for dataset_name, dataset in _dataset_items(split_dataset, split_name):
pending = []
progress = tqdm(
range(len(dataset)),
desc=f"Cache {dataset_name}",
disable=not overwatch.is_rank_zero(),
)
for idx in progress:
pending.append(dataset[idx])
if len(pending) >= batch_size:
total_saved += _save_batch(trainer, collator, pending, split_name, save_dtype)
pending = []
progress.set_postfix(saved=total_saved)
if pending:
total_saved += _save_batch(trainer, collator, pending, split_name, save_dtype)
cache_root = Path(str(cache_dir))
manifest_path = cache_root / "manifest.txt"
if overwatch.is_rank_zero():
manifest_path.parent.mkdir(parents=True, exist_ok=True)
with manifest_path.open("w") as f:
f.write(f"teacher_checkpoint={teacher_checkpoint}\n")
f.write(f"saved={total_saved}\n")
overwatch.info(f"Saved {total_saved} motion latent cache entries to {cache_root}")
def main():
module = importlib.import_module(f"psi.config.train.{sys.argv[1]}")
config_cls = getattr(module, "DynamicLaunchConfig")
config = tyro.cli(
config_cls,
config=(tyro.conf.ConsolidateSubcommandArgs,),
args=sys.argv[2:],
)
save_cache(config)
if __name__ == "__main__":
main()
|