code-HFD_lzk / scripts /train_third_party.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
55e0599 verified
Raw History Blame Contribute Delete
13.8 kB
#!/usr/bin/env python3
"""Validate, launch, or execute the fixed third-party-only small experiment."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
PROJECT = Path(__file__).resolve().parents[1]
for directory in (PROJECT, PROJECT / "src"):
if str(directory) not in sys.path:
sys.path.insert(0, str(directory))
PRODUCTION = PROJECT / "data/hfd_third_party_multi_v5/production"
DEFAULT_MANIFEST = PROJECT / "data/hfd_third_party_multi_v5/training_pilot_300/selection.json"
BACKBONE = Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/Qwen2.5-Omni-3B")
def contained_output(path):
value = Path(path).absolute().resolve()
if not value.is_relative_to(PROJECT / "runs") or value == PROJECT / "runs":
raise ValueError("Experiment output must be a child directory of HFD_lzk/runs")
return value
def build_config(manifest, run_dir, phase):
from psi.config.train.posttrain_third_party_config import DynamicLaunchConfig
return DynamicLaunchConfig.model_validate({
"exp": "third_party", "seed": 26100401, "auto_tag_run": False,
"timestamp": run_dir.name, "eval": False, "debug": False,
"log": {"report_to": None, "log_freq": 20},
"wandb": {"project": "hfd-lzk-third-party", "resume": "never"},
"train": {
"name": "posttrain_third_party", "experiment_phase": phase,
"probe_count": 2 if phase == "smoke" else 12, "coverage_epochs": 3,
"output_dir": str(run_dir), "data_parallel": "ddp", "mixed_precision": "bf16",
"train_batch_size": 2, "target_effective_sources_per_gpu": 2,
"val_batch_size": 1, "num_workers": 2, "val_num_workers": 0,
"source_length_lookahead_batches": 1, "dynamic_source_refill": False,
"gradient_accumulation_steps": 1, "source_accumulation_groups": 1,
"resume_from_checkpoint": None, "resume_load_full_state": False,
"max_training_steps": 50, "checkpointing_steps": 50,
"validation_steps": 0, "val_num_batches": 0, "max_checkpoints_to_keep": 2,
"learning_rate": 8e-5, "warmup_steps": 1, "warmup_ratio": None,
"cosine_min_lr_rate": 0.05, "max_grad_norm": 1.0,
"lr_scheduler_kwargs": {"betas": [0.9, 0.999], "weight_decay": 0.0, "eps": 1e-8},
"full_duplex_semantics_version": 24, "qwen_kv_gradient_horizon_windows": 1,
"dense_window_accumulation_steps": 1, "full_source_gradient_checkpointing": False,
"control_loss_weight": 1.0, "control_head_lr_multiplier": 0.25,
"joint_velocity_loss_weight": 0.1, "joint_acceleration_loss_weight": 0.05,
"root_continuity_loss_weight": 0.1, "stationary_root_height_loss_weight": 1.0,
"motion_duration_loss_weight": 0.2, "latest_instruction_loss_weight": 0.1,
"instruction_memory_loss_weight": 0.05,
"loss_group_weight": {"joint_abs": 1.0, "root_delta_xy": 3.0,
"root_rot5_delta": 10.0, "root_height": 15.0, "foot_contact": 0.1},
},
"data": {
"frozen_manifest": str(manifest), "production_root": str(PRODUCTION),
"online_audio": True, "multi_turn_sessions": True, "causal_listen_timing": True,
"stationary_heading_alignment": True, "safe_stationary_anchors": True,
"stationary_idle_residual_retargeting": True, "dynamic_task_entry_blend": True,
"initial_idle_max_steps": 7, "turn_transition_frames": 50,
"max_speech_onset_steps": 3, "listen_reaction_steps": 1,
"max_stream_steps": 1919, "long_session_drop_ratio": 0.0,
"filter_known_tts_translation_mismatches": True,
"session_voice_variants": 1, "interruption_sessions": False,
"independent_task_pools": False, "safe_hold_session_variants": 0,
"self_repair_implicit_ratio": 0.0,
"transform": {
"repack": {"action_chunk_size": 30, "rtc_prefix_frames": 10,
"textop_actions": True, "pad_action_dim": 39, "pad_state_dim": 39},
"field": {"action_norm_type": "bounds_q99", "normalize_state": True,
"pad_action_dim": 39, "pad_state_dim": 39,
"stat_path": str(PROJECT / "assets/stats/seed_all_g1_motion39_textop_stats.json")},
"model": {"audio_only": True, "text_only": True, "video_only": False, "img_aug": False},
},
},
"model": {
"model_name_or_path": str(BACKBONE), "tune_vlm": True, "gradient_checkpointing": False,
"lang_backbone_lr": 2e-6, "streaming_qwen_enabled": True,
"streaming_qwen_trainable_last_layers": 2, "streaming_qwen_train_final_norm": True,
"instruction_recency_slots": 3, "state_aware_serial_control": False,
"predict_motion_duration": True, "motion_duration_distribution_bins": 48,
"action_dim": 39, "odim": 39, "action_chunk_size": 30, "action_exec_horizon": 20,
"observation_horizon": 1, "view_feature_dim": 2048,
"use_future_progress_embedding": True, "future_progress_dim": 5, "rtc": True, "max_delay": 11,
},
})
def check_smoke(smoke_run, manifest):
if smoke_run is None:
raise ValueError("Formal training requires --smoke-run with a successful short-run directory")
smoke_run = contained_output(smoke_run)
events = [json.loads(line) for line in (smoke_run / "events.jsonl").read_text().splitlines() if line]
if not any(event.get("event") == "smoke_checkpoint_reload_passed" for event in events):
raise ValueError("Smoke checkpoint save/full-state reload has not passed")
if not any(event.get("event") == "checkpoint_probe_complete" and event.get("step", 0) > 0 for event in events):
raise ValueError("Smoke checkpoint prediction/video check has not passed")
config = json.loads((smoke_run / "run_config.json").read_text())
if Path(config["data"]["frozen_manifest"]).resolve() != manifest:
raise ValueError("Smoke run did not use this accepted manifest")
identity = json.loads((smoke_run / "input_identity.json").read_text())
if identity["manifest_sha256"] != hashlib.sha256(manifest.read_bytes()).hexdigest():
raise ValueError("Manifest changed since smoke training")
state = json.loads((smoke_run / "run_status.json").read_text())
if state.get("training_returncode") != 0 or state.get("phase") != "completed":
raise ValueError("Smoke training has not exited successfully")
for name, expected in json.loads((smoke_run / "code_identity.json").read_text()).items():
if hashlib.sha256((PROJECT / name).read_bytes()).hexdigest() != expected:
raise ValueError(f"Training implementation changed since smoke validation: {name}")
def validate(args):
from scripts.data.prepare_third_party_training import validate_acceptance
manifest = args.manifest.resolve()
run_dir = contained_output(args.run_dir)
report = validate_acceptance(
selection_path=manifest, production_root=PRODUCTION,
acceptance_path=manifest.parent / "acceptance.json", deep_payload=True,
)
if not BACKBONE.is_dir():
raise FileNotFoundError(BACKBONE)
config = build_config(manifest, run_dir, args.phase)
from scripts.infer.third_party_checkpoint_probe import check_renderer
check_renderer()
if getattr(args, "resume_checkpoint", None):
if args.phase != "train" or args.smoke_run:
raise ValueError("Use --phase train --resume-checkpoint without --smoke-run")
from scripts.third_party_resume import prepare_resume
config, args.resume_record = prepare_resume(
config, args.resume_checkpoint, manifest=manifest, run_dir=run_dir,
)
elif args.phase == "train":
check_smoke(args.smoke_run, manifest)
return manifest, run_dir, report, config
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--phase", choices=("smoke", "train"), default="smoke")
parser.add_argument("--smoke-run", type=Path)
parser.add_argument("--resume-checkpoint", type=Path,
help="Resume full state into a separate new run directory")
mode = parser.add_mutually_exclusive_group()
mode.add_argument("--check-only", action="store_true")
mode.add_argument("--launch", action="store_true")
mode.add_argument("--worker", action="store_true", help=argparse.SUPPRESS)
args = parser.parse_args()
os.chdir(PROJECT)
# This entry is intentionally single-process, even if invoked from a shell
# that previously ran torchrun/Accelerate/SLURM. Do this before psi imports.
for key in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE",
"MASTER_ADDR", "MASTER_PORT", "ACCELERATE_USE_DEEPSPEED",
"ACCELERATE_USE_FSDP", "ACCELERATE_USE_MEGATRON_LM",
"ACCELERATE_USE_CPU", "SLURM_PROCID", "SLURM_LOCALID", "SLURM_NTASKS"):
os.environ.pop(key, None)
os.environ["OMP_NUM_THREADS"] = "4"
if args.worker:
cache_root = PROJECT / "cache/third_party_training"
for key, name in (("TORCHINDUCTOR_CACHE_DIR", "inductor"),
("TRITON_CACHE_DIR", "triton"),
("CUDA_CACHE_PATH", "cuda")):
directory = cache_root / name
directory.mkdir(parents=True, exist_ok=True)
os.environ[key] = str(directory)
from scripts.third_party_temp import configure_project_temp
temporary_storage = configure_project_temp(PROJECT)
manifest, run_dir, report, config = validate(args)
if not args.launch and not args.worker:
print(json.dumps({"passed": True, "phase": args.phase, "train_count": 300,
"manifest_sha256": report["manifest_sha256"], "run_dir": str(run_dir),
"resume": getattr(args, "resume_record", None),
"visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0}, indent=2))
return 0
if args.launch:
shared = [str(PROJECT / "scripts/train_third_party.py"), "--manifest", str(manifest),
"--run-dir", str(run_dir), "--phase", args.phase]
if args.smoke_run:
shared.extend(["--smoke-run", str(args.smoke_run.resolve())])
if args.resume_checkpoint:
shared.extend(["--resume-checkpoint", str(args.resume_checkpoint.resolve())])
spec = {"schema": 1, "project_root": str(PROJECT), "run_dir": str(run_dir), "gpu": 3,
"manifest_path": str(manifest), "readiness_file": str(manifest.parent / "acceptance.json"),
"argv": [sys.executable, *shared, "--worker"],
"preflight_command": [sys.executable, *shared, "--check-only"]}
spec_path = manifest.parent / f"{run_dir.name}.launch.json"
if spec_path.exists() and json.loads(spec_path.read_text()) != spec:
raise ValueError("Refusing to overwrite a different launch specification")
spec_path.write_text(json.dumps(spec, indent=2) + "\n")
return subprocess.call([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"),
"start", "--spec", str(spec_path)])
if os.environ.get("CUDA_VISIBLE_DEVICES") != "3" or os.environ.get("HFD_NPROC_PER_NODE") != "1":
raise RuntimeError("Use --launch; the worker requires the single-GPU managed environment")
if int(os.environ.get("WORLD_SIZE", "1")) != 1:
raise RuntimeError("This accepted pilot entry supports exactly one GPU")
# Explicit local/offline setup; never source the owner's .env or export credentials.
os.environ.update({"HFD_SKIP_DOTENV": "1", "PSI_SKIP_VLM_MODEL": "0",
"HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1",
"PSI_FLOW_SIGMA_MIN": "0.05", "PSI_FLOW_SIGMA_MAX": "1.0",
"PSI_WORKER_TORCH_THREADS": "1", "PSI_DATALOADER_PIN_MEMORY": "1",
"PSI_FOREACH_GRAD_NORM": "1", "OMP_NUM_THREADS": "4"})
os.environ.pop("_ENV_INFO_JSON", None)
os.environ["HFD_CONTINUOUS_DATA_ROOT"] = config.data.continuous_audio_root
os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = config.data.background_assignment_path
os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str(
Path(config.data.continuous_audio_root) / "reports/production_complete.json")
run_dir.mkdir(parents=True, exist_ok=True)
(run_dir / "input_identity.json").write_text(json.dumps({
"manifest_sha256": report["manifest_sha256"], "phase": args.phase,
"qwen_initialization": "checkpoint" if args.resume_checkpoint else "pretrained",
"hfd_initialization": "checkpoint" if args.resume_checkpoint else "fresh", "train_count": 300,
"temporary_storage": temporary_storage,
"resume_checkpoint": str(args.resume_checkpoint.resolve()) if args.resume_checkpoint else None,
}, indent=2) + "\n")
from scripts.third_party_resume import code_identity
(run_dir / "code_identity.json").write_text(json.dumps(code_identity(), indent=2) + "\n")
if args.resume_checkpoint:
(run_dir / "resume_identity.json").write_text(json.dumps(args.resume_record, indent=2) + "\n")
from scripts.train import train
train(config)
return 0
if __name__ == "__main__":
raise SystemExit(main())