code-HFD_lzk / scripts /save_motion_teacher_hidden_cache.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
55e0599 verified
Raw History Blame Contribute Delete
4.95 kB
from __future__ import annotations
from dotenv import load_dotenv
assert load_dotenv(), "Failed to load .env file. Make sure it exists and is properly formatted."
import importlib
import os
import sys
from pathlib import Path
from typing import Any
import torch
import tyro
from tqdm import tqdm
from psi.data.dataset import MixtureDataset
from psi.trainers import Trainer
from psi.trainers.qwen2_5omni_mixin import PaddedCollatorForTogether
from psi.utils import batch_str_to_tensor, initialize_overwatch, move_to_device, seed_everything
overwatch = initialize_overwatch(__name__)
def _dataset_items(dataset: Any, name: str):
if isinstance(dataset, MixtureDataset):
for idx, sub_dataset in enumerate(dataset.datasets):
yield from _dataset_items(sub_dataset, f"{name}_mix{idx}")
return
if isinstance(dataset, dict):
for key, sub_dataset in dataset.items():
yield from _dataset_items(sub_dataset, f"{name}_{key}")
return
yield name, dataset
def _save_batch(trainer, collator, instances, split_name: str, save_dtype: torch.dtype) -> int:
batch = batch_str_to_tensor(collator(instances))
batch = move_to_device(batch, trainer.device)
with torch.no_grad():
hidden = trainer.model.encode_motion_hidden(batch["actions"])
saved = 0
for i in range(int(batch["input_ids"].shape[0])):
cache_path, latent_hash = trainer._latent_cache_key(batch, i)
cache_path.parent.mkdir(parents=True, exist_ok=True)
obj = {
"hidden_states": hidden[i].detach().to(dtype=save_dtype).cpu(),
"latent_hash": latent_hash,
"split": split_name,
"task_name": trainer._batch_item(batch, "task_name", i, default="unknown"),
"motion_dir": trainer._batch_item(batch, "motion_dir", i, default=""),
"video_path": trainer._batch_item(batch, "video_path", i, default=""),
"frame_id": str(trainer._batch_item(batch, "frame_id", i, default="")),
}
torch.save(obj, cache_path)
saved += 1
return saved
def save_cache(config):
if config.seed:
seed_everything(config.seed)
cache_dir = getattr(config.model, "motion_latent_cache_dir", None)
if not cache_dir:
raise ValueError("--model.motion-latent-cache-dir must be set")
teacher_checkpoint = getattr(config.model, "motion_teacher_checkpoint", None)
if not teacher_checkpoint:
raise ValueError("--model.motion-teacher-checkpoint must point to the stage-1 checkpoint")
if torch.cuda.is_available():
torch.cuda.set_device(0)
device = torch.device("cuda:0")
else:
device = torch.device("cpu")
trainer = Trainer.instantiate(config, device)
trainer.init_models()
trainer.model.motion_encoder.to(device)
trainer.model.motion_encoder.eval()
train_dataset, val_dataset = trainer.create_datasets()
datasets = [("train", train_dataset)]
if val_dataset is not None:
datasets.append(("val", val_dataset))
collator = PaddedCollatorForTogether(
model_max_length=trainer.tokenizer.model_max_length,
pad_token_id=trainer.tokenizer.pad_token_id,
padding_side="right",
)
batch_size = max(1, int(config.train.val_batch_size))
save_dtype = torch.bfloat16 if config.train.mixed_precision == "bf16" else torch.float32
total_saved = 0
for split_name, split_dataset in datasets:
for dataset_name, dataset in _dataset_items(split_dataset, split_name):
pending = []
progress = tqdm(
range(len(dataset)),
desc=f"Cache {dataset_name}",
disable=not overwatch.is_rank_zero(),
)
for idx in progress:
pending.append(dataset[idx])
if len(pending) >= batch_size:
total_saved += _save_batch(trainer, collator, pending, split_name, save_dtype)
pending = []
progress.set_postfix(saved=total_saved)
if pending:
total_saved += _save_batch(trainer, collator, pending, split_name, save_dtype)
cache_root = Path(str(cache_dir))
manifest_path = cache_root / "manifest.txt"
if overwatch.is_rank_zero():
manifest_path.parent.mkdir(parents=True, exist_ok=True)
with manifest_path.open("w") as f:
f.write(f"teacher_checkpoint={teacher_checkpoint}\n")
f.write(f"saved={total_saved}\n")
overwatch.info(f"Saved {total_saved} motion latent cache entries to {cache_root}")
def main():
module = importlib.import_module(f"psi.config.train.{sys.argv[1]}")
config_cls = getattr(module, "DynamicLaunchConfig")
config = tyro.cli(
config_cls,
config=(tyro.conf.ConsolidateSubcommandArgs,),
args=sys.argv[2:],
)
save_cache(config)
if __name__ == "__main__":
main()