File size: 13,780 Bytes
55e0599
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
#!/usr/bin/env python3
"""Validate, launch, or execute the fixed third-party-only small experiment."""
from __future__ import annotations

import argparse
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys

PROJECT = Path(__file__).resolve().parents[1]
for directory in (PROJECT, PROJECT / "src"):
    if str(directory) not in sys.path:
        sys.path.insert(0, str(directory))

PRODUCTION = PROJECT / "data/hfd_third_party_multi_v5/production"
DEFAULT_MANIFEST = PROJECT / "data/hfd_third_party_multi_v5/training_pilot_300/selection.json"
BACKBONE = Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/Qwen2.5-Omni-3B")


def contained_output(path):
    value = Path(path).absolute().resolve()
    if not value.is_relative_to(PROJECT / "runs") or value == PROJECT / "runs":
        raise ValueError("Experiment output must be a child directory of HFD_lzk/runs")
    return value


def build_config(manifest, run_dir, phase):
    from psi.config.train.posttrain_third_party_config import DynamicLaunchConfig
    return DynamicLaunchConfig.model_validate({
        "exp": "third_party", "seed": 26100401, "auto_tag_run": False,
        "timestamp": run_dir.name, "eval": False, "debug": False,
        "log": {"report_to": None, "log_freq": 20},
        "wandb": {"project": "hfd-lzk-third-party", "resume": "never"},
        "train": {
            "name": "posttrain_third_party", "experiment_phase": phase,
            "probe_count": 2 if phase == "smoke" else 12, "coverage_epochs": 3,
            "output_dir": str(run_dir), "data_parallel": "ddp", "mixed_precision": "bf16",
            "train_batch_size": 2, "target_effective_sources_per_gpu": 2,
            "val_batch_size": 1, "num_workers": 2, "val_num_workers": 0,
            "source_length_lookahead_batches": 1, "dynamic_source_refill": False,
            "gradient_accumulation_steps": 1, "source_accumulation_groups": 1,
            "resume_from_checkpoint": None, "resume_load_full_state": False,
            "max_training_steps": 50, "checkpointing_steps": 50,
            "validation_steps": 0, "val_num_batches": 0, "max_checkpoints_to_keep": 2,
            "learning_rate": 8e-5, "warmup_steps": 1, "warmup_ratio": None,
            "cosine_min_lr_rate": 0.05, "max_grad_norm": 1.0,
            "lr_scheduler_kwargs": {"betas": [0.9, 0.999], "weight_decay": 0.0, "eps": 1e-8},
            "full_duplex_semantics_version": 24, "qwen_kv_gradient_horizon_windows": 1,
            "dense_window_accumulation_steps": 1, "full_source_gradient_checkpointing": False,
            "control_loss_weight": 1.0, "control_head_lr_multiplier": 0.25,
            "joint_velocity_loss_weight": 0.1, "joint_acceleration_loss_weight": 0.05,
            "root_continuity_loss_weight": 0.1, "stationary_root_height_loss_weight": 1.0,
            "motion_duration_loss_weight": 0.2, "latest_instruction_loss_weight": 0.1,
            "instruction_memory_loss_weight": 0.05,
            "loss_group_weight": {"joint_abs": 1.0, "root_delta_xy": 3.0,
                                  "root_rot5_delta": 10.0, "root_height": 15.0, "foot_contact": 0.1},
        },
        "data": {
            "frozen_manifest": str(manifest), "production_root": str(PRODUCTION),
            "online_audio": True, "multi_turn_sessions": True, "causal_listen_timing": True,
            "stationary_heading_alignment": True, "safe_stationary_anchors": True,
            "stationary_idle_residual_retargeting": True, "dynamic_task_entry_blend": True,
            "initial_idle_max_steps": 7, "turn_transition_frames": 50,
            "max_speech_onset_steps": 3, "listen_reaction_steps": 1,
            "max_stream_steps": 1919, "long_session_drop_ratio": 0.0,
            "filter_known_tts_translation_mismatches": True,
            "session_voice_variants": 1, "interruption_sessions": False,
            "independent_task_pools": False, "safe_hold_session_variants": 0,
            "self_repair_implicit_ratio": 0.0,
            "transform": {
                "repack": {"action_chunk_size": 30, "rtc_prefix_frames": 10,
                           "textop_actions": True, "pad_action_dim": 39, "pad_state_dim": 39},
                "field": {"action_norm_type": "bounds_q99", "normalize_state": True,
                          "pad_action_dim": 39, "pad_state_dim": 39,
                          "stat_path": str(PROJECT / "assets/stats/seed_all_g1_motion39_textop_stats.json")},
                "model": {"audio_only": True, "text_only": True, "video_only": False, "img_aug": False},
            },
        },
        "model": {
            "model_name_or_path": str(BACKBONE), "tune_vlm": True, "gradient_checkpointing": False,
            "lang_backbone_lr": 2e-6, "streaming_qwen_enabled": True,
            "streaming_qwen_trainable_last_layers": 2, "streaming_qwen_train_final_norm": True,
            "instruction_recency_slots": 3, "state_aware_serial_control": False,
            "predict_motion_duration": True, "motion_duration_distribution_bins": 48,
            "action_dim": 39, "odim": 39, "action_chunk_size": 30, "action_exec_horizon": 20,
            "observation_horizon": 1, "view_feature_dim": 2048,
            "use_future_progress_embedding": True, "future_progress_dim": 5, "rtc": True, "max_delay": 11,
        },
    })


def check_smoke(smoke_run, manifest):
    if smoke_run is None:
        raise ValueError("Formal training requires --smoke-run with a successful short-run directory")
    smoke_run = contained_output(smoke_run)
    events = [json.loads(line) for line in (smoke_run / "events.jsonl").read_text().splitlines() if line]
    if not any(event.get("event") == "smoke_checkpoint_reload_passed" for event in events):
        raise ValueError("Smoke checkpoint save/full-state reload has not passed")
    if not any(event.get("event") == "checkpoint_probe_complete" and event.get("step", 0) > 0 for event in events):
        raise ValueError("Smoke checkpoint prediction/video check has not passed")
    config = json.loads((smoke_run / "run_config.json").read_text())
    if Path(config["data"]["frozen_manifest"]).resolve() != manifest:
        raise ValueError("Smoke run did not use this accepted manifest")
    identity = json.loads((smoke_run / "input_identity.json").read_text())
    if identity["manifest_sha256"] != hashlib.sha256(manifest.read_bytes()).hexdigest():
        raise ValueError("Manifest changed since smoke training")
    state = json.loads((smoke_run / "run_status.json").read_text())
    if state.get("training_returncode") != 0 or state.get("phase") != "completed":
        raise ValueError("Smoke training has not exited successfully")
    for name, expected in json.loads((smoke_run / "code_identity.json").read_text()).items():
        if hashlib.sha256((PROJECT / name).read_bytes()).hexdigest() != expected:
            raise ValueError(f"Training implementation changed since smoke validation: {name}")


def validate(args):
    from scripts.data.prepare_third_party_training import validate_acceptance
    manifest = args.manifest.resolve()
    run_dir = contained_output(args.run_dir)
    report = validate_acceptance(
        selection_path=manifest, production_root=PRODUCTION,
        acceptance_path=manifest.parent / "acceptance.json", deep_payload=True,
    )
    if not BACKBONE.is_dir():
        raise FileNotFoundError(BACKBONE)
    config = build_config(manifest, run_dir, args.phase)
    from scripts.infer.third_party_checkpoint_probe import check_renderer
    check_renderer()
    if getattr(args, "resume_checkpoint", None):
        if args.phase != "train" or args.smoke_run:
            raise ValueError("Use --phase train --resume-checkpoint without --smoke-run")
        from scripts.third_party_resume import prepare_resume
        config, args.resume_record = prepare_resume(
            config, args.resume_checkpoint, manifest=manifest, run_dir=run_dir,
        )
    elif args.phase == "train":
        check_smoke(args.smoke_run, manifest)
    return manifest, run_dir, report, config


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
    parser.add_argument("--run-dir", type=Path, required=True)
    parser.add_argument("--phase", choices=("smoke", "train"), default="smoke")
    parser.add_argument("--smoke-run", type=Path)
    parser.add_argument("--resume-checkpoint", type=Path,
                        help="Resume full state into a separate new run directory")
    mode = parser.add_mutually_exclusive_group()
    mode.add_argument("--check-only", action="store_true")
    mode.add_argument("--launch", action="store_true")
    mode.add_argument("--worker", action="store_true", help=argparse.SUPPRESS)
    args = parser.parse_args()
    os.chdir(PROJECT)
    # This entry is intentionally single-process, even if invoked from a shell
    # that previously ran torchrun/Accelerate/SLURM. Do this before psi imports.
    for key in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE",
                "MASTER_ADDR", "MASTER_PORT", "ACCELERATE_USE_DEEPSPEED",
                "ACCELERATE_USE_FSDP", "ACCELERATE_USE_MEGATRON_LM",
                "ACCELERATE_USE_CPU", "SLURM_PROCID", "SLURM_LOCALID", "SLURM_NTASKS"):
        os.environ.pop(key, None)
    os.environ["OMP_NUM_THREADS"] = "4"
    if args.worker:
        cache_root = PROJECT / "cache/third_party_training"
        for key, name in (("TORCHINDUCTOR_CACHE_DIR", "inductor"),
                          ("TRITON_CACHE_DIR", "triton"),
                          ("CUDA_CACHE_PATH", "cuda")):
            directory = cache_root / name
            directory.mkdir(parents=True, exist_ok=True)
            os.environ[key] = str(directory)
        from scripts.third_party_temp import configure_project_temp
        temporary_storage = configure_project_temp(PROJECT)
    manifest, run_dir, report, config = validate(args)
    if not args.launch and not args.worker:
        print(json.dumps({"passed": True, "phase": args.phase, "train_count": 300,
                          "manifest_sha256": report["manifest_sha256"], "run_dir": str(run_dir),
                          "resume": getattr(args, "resume_record", None),
                          "visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0}, indent=2))
        return 0
    if args.launch:
        shared = [str(PROJECT / "scripts/train_third_party.py"), "--manifest", str(manifest),
                  "--run-dir", str(run_dir), "--phase", args.phase]
        if args.smoke_run:
            shared.extend(["--smoke-run", str(args.smoke_run.resolve())])
        if args.resume_checkpoint:
            shared.extend(["--resume-checkpoint", str(args.resume_checkpoint.resolve())])
        spec = {"schema": 1, "project_root": str(PROJECT), "run_dir": str(run_dir), "gpu": 3,
                "manifest_path": str(manifest), "readiness_file": str(manifest.parent / "acceptance.json"),
                "argv": [sys.executable, *shared, "--worker"],
                "preflight_command": [sys.executable, *shared, "--check-only"]}
        spec_path = manifest.parent / f"{run_dir.name}.launch.json"
        if spec_path.exists() and json.loads(spec_path.read_text()) != spec:
            raise ValueError("Refusing to overwrite a different launch specification")
        spec_path.write_text(json.dumps(spec, indent=2) + "\n")
        return subprocess.call([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"),
                                "start", "--spec", str(spec_path)])
    if os.environ.get("CUDA_VISIBLE_DEVICES") != "3" or os.environ.get("HFD_NPROC_PER_NODE") != "1":
        raise RuntimeError("Use --launch; the worker requires the single-GPU managed environment")
    if int(os.environ.get("WORLD_SIZE", "1")) != 1:
        raise RuntimeError("This accepted pilot entry supports exactly one GPU")
    # Explicit local/offline setup; never source the owner's .env or export credentials.
    os.environ.update({"HFD_SKIP_DOTENV": "1", "PSI_SKIP_VLM_MODEL": "0",
                       "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1",
                       "PSI_FLOW_SIGMA_MIN": "0.05", "PSI_FLOW_SIGMA_MAX": "1.0",
                       "PSI_WORKER_TORCH_THREADS": "1", "PSI_DATALOADER_PIN_MEMORY": "1",
                       "PSI_FOREACH_GRAD_NORM": "1", "OMP_NUM_THREADS": "4"})
    os.environ.pop("_ENV_INFO_JSON", None)
    os.environ["HFD_CONTINUOUS_DATA_ROOT"] = config.data.continuous_audio_root
    os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = config.data.background_assignment_path
    os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str(
        Path(config.data.continuous_audio_root) / "reports/production_complete.json")
    run_dir.mkdir(parents=True, exist_ok=True)
    (run_dir / "input_identity.json").write_text(json.dumps({
        "manifest_sha256": report["manifest_sha256"], "phase": args.phase,
        "qwen_initialization": "checkpoint" if args.resume_checkpoint else "pretrained",
        "hfd_initialization": "checkpoint" if args.resume_checkpoint else "fresh", "train_count": 300,
        "temporary_storage": temporary_storage,
        "resume_checkpoint": str(args.resume_checkpoint.resolve()) if args.resume_checkpoint else None,
    }, indent=2) + "\n")
    from scripts.third_party_resume import code_identity
    (run_dir / "code_identity.json").write_text(json.dumps(code_identity(), indent=2) + "\n")
    if args.resume_checkpoint:
        (run_dir / "resume_identity.json").write_text(json.dumps(args.resume_record, indent=2) + "\n")
    from scripts.train import train
    train(config)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())