File size: 4,953 Bytes
35bef58
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
from __future__ import annotations

from dotenv import load_dotenv
assert load_dotenv(), "Failed to load .env file. Make sure it exists and is properly formatted."

import importlib
import os
import sys
from pathlib import Path
from typing import Any

import torch
import tyro
from tqdm import tqdm

from psi.data.dataset import MixtureDataset
from psi.trainers import Trainer
from psi.trainers.qwen2_5omni_mixin import PaddedCollatorForTogether
from psi.utils import batch_str_to_tensor, initialize_overwatch, move_to_device, seed_everything

overwatch = initialize_overwatch(__name__)


def _dataset_items(dataset: Any, name: str):
    if isinstance(dataset, MixtureDataset):
        for idx, sub_dataset in enumerate(dataset.datasets):
            yield from _dataset_items(sub_dataset, f"{name}_mix{idx}")
        return
    if isinstance(dataset, dict):
        for key, sub_dataset in dataset.items():
            yield from _dataset_items(sub_dataset, f"{name}_{key}")
        return
    yield name, dataset


def _save_batch(trainer, collator, instances, split_name: str, save_dtype: torch.dtype) -> int:
    batch = batch_str_to_tensor(collator(instances))
    batch = move_to_device(batch, trainer.device)
    with torch.no_grad():
        hidden = trainer.model.encode_motion_hidden(batch["actions"])

    saved = 0
    for i in range(int(batch["input_ids"].shape[0])):
        cache_path, latent_hash = trainer._latent_cache_key(batch, i)
        cache_path.parent.mkdir(parents=True, exist_ok=True)
        obj = {
            "hidden_states": hidden[i].detach().to(dtype=save_dtype).cpu(),
            "latent_hash": latent_hash,
            "split": split_name,
            "task_name": trainer._batch_item(batch, "task_name", i, default="unknown"),
            "motion_dir": trainer._batch_item(batch, "motion_dir", i, default=""),
            "video_path": trainer._batch_item(batch, "video_path", i, default=""),
            "frame_id": str(trainer._batch_item(batch, "frame_id", i, default="")),
        }
        torch.save(obj, cache_path)
        saved += 1
    return saved


def save_cache(config):
    if config.seed:
        seed_everything(config.seed)

    cache_dir = getattr(config.model, "motion_latent_cache_dir", None)
    if not cache_dir:
        raise ValueError("--model.motion-latent-cache-dir must be set")
    teacher_checkpoint = getattr(config.model, "motion_teacher_checkpoint", None)
    if not teacher_checkpoint:
        raise ValueError("--model.motion-teacher-checkpoint must point to the stage-1 checkpoint")

    if torch.cuda.is_available():
        torch.cuda.set_device(0)
        device = torch.device("cuda:0")
    else:
        device = torch.device("cpu")

    trainer = Trainer.instantiate(config, device)
    trainer.init_models()
    trainer.model.motion_encoder.to(device)
    trainer.model.motion_encoder.eval()

    train_dataset, val_dataset = trainer.create_datasets()
    datasets = [("train", train_dataset)]
    if val_dataset is not None:
        datasets.append(("val", val_dataset))

    collator = PaddedCollatorForTogether(
        model_max_length=trainer.tokenizer.model_max_length,
        pad_token_id=trainer.tokenizer.pad_token_id,
        padding_side="right",
    )
    batch_size = max(1, int(config.train.val_batch_size))
    save_dtype = torch.bfloat16 if config.train.mixed_precision == "bf16" else torch.float32
    total_saved = 0

    for split_name, split_dataset in datasets:
        for dataset_name, dataset in _dataset_items(split_dataset, split_name):
            pending = []
            progress = tqdm(
                range(len(dataset)),
                desc=f"Cache {dataset_name}",
                disable=not overwatch.is_rank_zero(),
            )
            for idx in progress:
                pending.append(dataset[idx])
                if len(pending) >= batch_size:
                    total_saved += _save_batch(trainer, collator, pending, split_name, save_dtype)
                    pending = []
                    progress.set_postfix(saved=total_saved)
            if pending:
                total_saved += _save_batch(trainer, collator, pending, split_name, save_dtype)

    cache_root = Path(str(cache_dir))
    manifest_path = cache_root / "manifest.txt"
    if overwatch.is_rank_zero():
        manifest_path.parent.mkdir(parents=True, exist_ok=True)
        with manifest_path.open("w") as f:
            f.write(f"teacher_checkpoint={teacher_checkpoint}\n")
            f.write(f"saved={total_saved}\n")
        overwatch.info(f"Saved {total_saved} motion latent cache entries to {cache_root}")


def main():
    module = importlib.import_module(f"psi.config.train.{sys.argv[1]}")
    config_cls = getattr(module, "DynamicLaunchConfig")
    config = tyro.cli(
        config_cls,
        config=(tyro.conf.ConsolidateSubcommandArgs,),
        args=sys.argv[2:],
    )
    save_cache(config)


if __name__ == "__main__":
    main()