File size: 13,780 Bytes
55e0599 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 | #!/usr/bin/env python3
"""Validate, launch, or execute the fixed third-party-only small experiment."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
PROJECT = Path(__file__).resolve().parents[1]
for directory in (PROJECT, PROJECT / "src"):
if str(directory) not in sys.path:
sys.path.insert(0, str(directory))
PRODUCTION = PROJECT / "data/hfd_third_party_multi_v5/production"
DEFAULT_MANIFEST = PROJECT / "data/hfd_third_party_multi_v5/training_pilot_300/selection.json"
BACKBONE = Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/Qwen2.5-Omni-3B")
def contained_output(path):
value = Path(path).absolute().resolve()
if not value.is_relative_to(PROJECT / "runs") or value == PROJECT / "runs":
raise ValueError("Experiment output must be a child directory of HFD_lzk/runs")
return value
def build_config(manifest, run_dir, phase):
from psi.config.train.posttrain_third_party_config import DynamicLaunchConfig
return DynamicLaunchConfig.model_validate({
"exp": "third_party", "seed": 26100401, "auto_tag_run": False,
"timestamp": run_dir.name, "eval": False, "debug": False,
"log": {"report_to": None, "log_freq": 20},
"wandb": {"project": "hfd-lzk-third-party", "resume": "never"},
"train": {
"name": "posttrain_third_party", "experiment_phase": phase,
"probe_count": 2 if phase == "smoke" else 12, "coverage_epochs": 3,
"output_dir": str(run_dir), "data_parallel": "ddp", "mixed_precision": "bf16",
"train_batch_size": 2, "target_effective_sources_per_gpu": 2,
"val_batch_size": 1, "num_workers": 2, "val_num_workers": 0,
"source_length_lookahead_batches": 1, "dynamic_source_refill": False,
"gradient_accumulation_steps": 1, "source_accumulation_groups": 1,
"resume_from_checkpoint": None, "resume_load_full_state": False,
"max_training_steps": 50, "checkpointing_steps": 50,
"validation_steps": 0, "val_num_batches": 0, "max_checkpoints_to_keep": 2,
"learning_rate": 8e-5, "warmup_steps": 1, "warmup_ratio": None,
"cosine_min_lr_rate": 0.05, "max_grad_norm": 1.0,
"lr_scheduler_kwargs": {"betas": [0.9, 0.999], "weight_decay": 0.0, "eps": 1e-8},
"full_duplex_semantics_version": 24, "qwen_kv_gradient_horizon_windows": 1,
"dense_window_accumulation_steps": 1, "full_source_gradient_checkpointing": False,
"control_loss_weight": 1.0, "control_head_lr_multiplier": 0.25,
"joint_velocity_loss_weight": 0.1, "joint_acceleration_loss_weight": 0.05,
"root_continuity_loss_weight": 0.1, "stationary_root_height_loss_weight": 1.0,
"motion_duration_loss_weight": 0.2, "latest_instruction_loss_weight": 0.1,
"instruction_memory_loss_weight": 0.05,
"loss_group_weight": {"joint_abs": 1.0, "root_delta_xy": 3.0,
"root_rot5_delta": 10.0, "root_height": 15.0, "foot_contact": 0.1},
},
"data": {
"frozen_manifest": str(manifest), "production_root": str(PRODUCTION),
"online_audio": True, "multi_turn_sessions": True, "causal_listen_timing": True,
"stationary_heading_alignment": True, "safe_stationary_anchors": True,
"stationary_idle_residual_retargeting": True, "dynamic_task_entry_blend": True,
"initial_idle_max_steps": 7, "turn_transition_frames": 50,
"max_speech_onset_steps": 3, "listen_reaction_steps": 1,
"max_stream_steps": 1919, "long_session_drop_ratio": 0.0,
"filter_known_tts_translation_mismatches": True,
"session_voice_variants": 1, "interruption_sessions": False,
"independent_task_pools": False, "safe_hold_session_variants": 0,
"self_repair_implicit_ratio": 0.0,
"transform": {
"repack": {"action_chunk_size": 30, "rtc_prefix_frames": 10,
"textop_actions": True, "pad_action_dim": 39, "pad_state_dim": 39},
"field": {"action_norm_type": "bounds_q99", "normalize_state": True,
"pad_action_dim": 39, "pad_state_dim": 39,
"stat_path": str(PROJECT / "assets/stats/seed_all_g1_motion39_textop_stats.json")},
"model": {"audio_only": True, "text_only": True, "video_only": False, "img_aug": False},
},
},
"model": {
"model_name_or_path": str(BACKBONE), "tune_vlm": True, "gradient_checkpointing": False,
"lang_backbone_lr": 2e-6, "streaming_qwen_enabled": True,
"streaming_qwen_trainable_last_layers": 2, "streaming_qwen_train_final_norm": True,
"instruction_recency_slots": 3, "state_aware_serial_control": False,
"predict_motion_duration": True, "motion_duration_distribution_bins": 48,
"action_dim": 39, "odim": 39, "action_chunk_size": 30, "action_exec_horizon": 20,
"observation_horizon": 1, "view_feature_dim": 2048,
"use_future_progress_embedding": True, "future_progress_dim": 5, "rtc": True, "max_delay": 11,
},
})
def check_smoke(smoke_run, manifest):
if smoke_run is None:
raise ValueError("Formal training requires --smoke-run with a successful short-run directory")
smoke_run = contained_output(smoke_run)
events = [json.loads(line) for line in (smoke_run / "events.jsonl").read_text().splitlines() if line]
if not any(event.get("event") == "smoke_checkpoint_reload_passed" for event in events):
raise ValueError("Smoke checkpoint save/full-state reload has not passed")
if not any(event.get("event") == "checkpoint_probe_complete" and event.get("step", 0) > 0 for event in events):
raise ValueError("Smoke checkpoint prediction/video check has not passed")
config = json.loads((smoke_run / "run_config.json").read_text())
if Path(config["data"]["frozen_manifest"]).resolve() != manifest:
raise ValueError("Smoke run did not use this accepted manifest")
identity = json.loads((smoke_run / "input_identity.json").read_text())
if identity["manifest_sha256"] != hashlib.sha256(manifest.read_bytes()).hexdigest():
raise ValueError("Manifest changed since smoke training")
state = json.loads((smoke_run / "run_status.json").read_text())
if state.get("training_returncode") != 0 or state.get("phase") != "completed":
raise ValueError("Smoke training has not exited successfully")
for name, expected in json.loads((smoke_run / "code_identity.json").read_text()).items():
if hashlib.sha256((PROJECT / name).read_bytes()).hexdigest() != expected:
raise ValueError(f"Training implementation changed since smoke validation: {name}")
def validate(args):
from scripts.data.prepare_third_party_training import validate_acceptance
manifest = args.manifest.resolve()
run_dir = contained_output(args.run_dir)
report = validate_acceptance(
selection_path=manifest, production_root=PRODUCTION,
acceptance_path=manifest.parent / "acceptance.json", deep_payload=True,
)
if not BACKBONE.is_dir():
raise FileNotFoundError(BACKBONE)
config = build_config(manifest, run_dir, args.phase)
from scripts.infer.third_party_checkpoint_probe import check_renderer
check_renderer()
if getattr(args, "resume_checkpoint", None):
if args.phase != "train" or args.smoke_run:
raise ValueError("Use --phase train --resume-checkpoint without --smoke-run")
from scripts.third_party_resume import prepare_resume
config, args.resume_record = prepare_resume(
config, args.resume_checkpoint, manifest=manifest, run_dir=run_dir,
)
elif args.phase == "train":
check_smoke(args.smoke_run, manifest)
return manifest, run_dir, report, config
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--phase", choices=("smoke", "train"), default="smoke")
parser.add_argument("--smoke-run", type=Path)
parser.add_argument("--resume-checkpoint", type=Path,
help="Resume full state into a separate new run directory")
mode = parser.add_mutually_exclusive_group()
mode.add_argument("--check-only", action="store_true")
mode.add_argument("--launch", action="store_true")
mode.add_argument("--worker", action="store_true", help=argparse.SUPPRESS)
args = parser.parse_args()
os.chdir(PROJECT)
# This entry is intentionally single-process, even if invoked from a shell
# that previously ran torchrun/Accelerate/SLURM. Do this before psi imports.
for key in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE",
"MASTER_ADDR", "MASTER_PORT", "ACCELERATE_USE_DEEPSPEED",
"ACCELERATE_USE_FSDP", "ACCELERATE_USE_MEGATRON_LM",
"ACCELERATE_USE_CPU", "SLURM_PROCID", "SLURM_LOCALID", "SLURM_NTASKS"):
os.environ.pop(key, None)
os.environ["OMP_NUM_THREADS"] = "4"
if args.worker:
cache_root = PROJECT / "cache/third_party_training"
for key, name in (("TORCHINDUCTOR_CACHE_DIR", "inductor"),
("TRITON_CACHE_DIR", "triton"),
("CUDA_CACHE_PATH", "cuda")):
directory = cache_root / name
directory.mkdir(parents=True, exist_ok=True)
os.environ[key] = str(directory)
from scripts.third_party_temp import configure_project_temp
temporary_storage = configure_project_temp(PROJECT)
manifest, run_dir, report, config = validate(args)
if not args.launch and not args.worker:
print(json.dumps({"passed": True, "phase": args.phase, "train_count": 300,
"manifest_sha256": report["manifest_sha256"], "run_dir": str(run_dir),
"resume": getattr(args, "resume_record", None),
"visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0}, indent=2))
return 0
if args.launch:
shared = [str(PROJECT / "scripts/train_third_party.py"), "--manifest", str(manifest),
"--run-dir", str(run_dir), "--phase", args.phase]
if args.smoke_run:
shared.extend(["--smoke-run", str(args.smoke_run.resolve())])
if args.resume_checkpoint:
shared.extend(["--resume-checkpoint", str(args.resume_checkpoint.resolve())])
spec = {"schema": 1, "project_root": str(PROJECT), "run_dir": str(run_dir), "gpu": 3,
"manifest_path": str(manifest), "readiness_file": str(manifest.parent / "acceptance.json"),
"argv": [sys.executable, *shared, "--worker"],
"preflight_command": [sys.executable, *shared, "--check-only"]}
spec_path = manifest.parent / f"{run_dir.name}.launch.json"
if spec_path.exists() and json.loads(spec_path.read_text()) != spec:
raise ValueError("Refusing to overwrite a different launch specification")
spec_path.write_text(json.dumps(spec, indent=2) + "\n")
return subprocess.call([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"),
"start", "--spec", str(spec_path)])
if os.environ.get("CUDA_VISIBLE_DEVICES") != "3" or os.environ.get("HFD_NPROC_PER_NODE") != "1":
raise RuntimeError("Use --launch; the worker requires the single-GPU managed environment")
if int(os.environ.get("WORLD_SIZE", "1")) != 1:
raise RuntimeError("This accepted pilot entry supports exactly one GPU")
# Explicit local/offline setup; never source the owner's .env or export credentials.
os.environ.update({"HFD_SKIP_DOTENV": "1", "PSI_SKIP_VLM_MODEL": "0",
"HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1",
"PSI_FLOW_SIGMA_MIN": "0.05", "PSI_FLOW_SIGMA_MAX": "1.0",
"PSI_WORKER_TORCH_THREADS": "1", "PSI_DATALOADER_PIN_MEMORY": "1",
"PSI_FOREACH_GRAD_NORM": "1", "OMP_NUM_THREADS": "4"})
os.environ.pop("_ENV_INFO_JSON", None)
os.environ["HFD_CONTINUOUS_DATA_ROOT"] = config.data.continuous_audio_root
os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = config.data.background_assignment_path
os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str(
Path(config.data.continuous_audio_root) / "reports/production_complete.json")
run_dir.mkdir(parents=True, exist_ok=True)
(run_dir / "input_identity.json").write_text(json.dumps({
"manifest_sha256": report["manifest_sha256"], "phase": args.phase,
"qwen_initialization": "checkpoint" if args.resume_checkpoint else "pretrained",
"hfd_initialization": "checkpoint" if args.resume_checkpoint else "fresh", "train_count": 300,
"temporary_storage": temporary_storage,
"resume_checkpoint": str(args.resume_checkpoint.resolve()) if args.resume_checkpoint else None,
}, indent=2) + "\n")
from scripts.third_party_resume import code_identity
(run_dir / "code_identity.json").write_text(json.dumps(code_identity(), indent=2) + "\n")
if args.resume_checkpoint:
(run_dir / "resume_identity.json").write_text(json.dumps(args.resume_record, indent=2) + "\n")
from scripts.train import train
train(config)
return 0
if __name__ == "__main__":
raise SystemExit(main())
|