#!/usr/bin/env python3 """Validate, launch, or execute the fixed third-party-only small experiment.""" from __future__ import annotations import argparse import hashlib import json import os from pathlib import Path import subprocess import sys PROJECT = Path(__file__).resolve().parents[1] for directory in (PROJECT, PROJECT / "src"): if str(directory) not in sys.path: sys.path.insert(0, str(directory)) PRODUCTION = PROJECT / "data/hfd_third_party_multi_v5/production" DEFAULT_MANIFEST = PROJECT / "data/hfd_third_party_multi_v5/training_pilot_300/selection.json" BACKBONE = Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/Qwen2.5-Omni-3B") def contained_output(path): value = Path(path).absolute().resolve() if not value.is_relative_to(PROJECT / "runs") or value == PROJECT / "runs": raise ValueError("Experiment output must be a child directory of HFD_lzk/runs") return value def build_config(manifest, run_dir, phase): from psi.config.train.posttrain_third_party_config import DynamicLaunchConfig return DynamicLaunchConfig.model_validate({ "exp": "third_party", "seed": 26100401, "auto_tag_run": False, "timestamp": run_dir.name, "eval": False, "debug": False, "log": {"report_to": None, "log_freq": 20}, "wandb": {"project": "hfd-lzk-third-party", "resume": "never"}, "train": { "name": "posttrain_third_party", "experiment_phase": phase, "probe_count": 2 if phase == "smoke" else 12, "coverage_epochs": 3, "output_dir": str(run_dir), "data_parallel": "ddp", "mixed_precision": "bf16", "train_batch_size": 2, "target_effective_sources_per_gpu": 2, "val_batch_size": 1, "num_workers": 2, "val_num_workers": 0, "source_length_lookahead_batches": 1, "dynamic_source_refill": False, "gradient_accumulation_steps": 1, "source_accumulation_groups": 1, "resume_from_checkpoint": None, "resume_load_full_state": False, "max_training_steps": 50, "checkpointing_steps": 50, "validation_steps": 0, "val_num_batches": 0, "max_checkpoints_to_keep": 2, "learning_rate": 8e-5, "warmup_steps": 1, "warmup_ratio": None, "cosine_min_lr_rate": 0.05, "max_grad_norm": 1.0, "lr_scheduler_kwargs": {"betas": [0.9, 0.999], "weight_decay": 0.0, "eps": 1e-8}, "full_duplex_semantics_version": 24, "qwen_kv_gradient_horizon_windows": 1, "dense_window_accumulation_steps": 1, "full_source_gradient_checkpointing": False, "control_loss_weight": 1.0, "control_head_lr_multiplier": 0.25, "joint_velocity_loss_weight": 0.1, "joint_acceleration_loss_weight": 0.05, "root_continuity_loss_weight": 0.1, "stationary_root_height_loss_weight": 1.0, "motion_duration_loss_weight": 0.2, "latest_instruction_loss_weight": 0.1, "instruction_memory_loss_weight": 0.05, "loss_group_weight": {"joint_abs": 1.0, "root_delta_xy": 3.0, "root_rot5_delta": 10.0, "root_height": 15.0, "foot_contact": 0.1}, }, "data": { "frozen_manifest": str(manifest), "production_root": str(PRODUCTION), "online_audio": True, "multi_turn_sessions": True, "causal_listen_timing": True, "stationary_heading_alignment": True, "safe_stationary_anchors": True, "stationary_idle_residual_retargeting": True, "dynamic_task_entry_blend": True, "initial_idle_max_steps": 7, "turn_transition_frames": 50, "max_speech_onset_steps": 3, "listen_reaction_steps": 1, "max_stream_steps": 1919, "long_session_drop_ratio": 0.0, "filter_known_tts_translation_mismatches": True, "session_voice_variants": 1, "interruption_sessions": False, "independent_task_pools": False, "safe_hold_session_variants": 0, "self_repair_implicit_ratio": 0.0, "transform": { "repack": {"action_chunk_size": 30, "rtc_prefix_frames": 10, "textop_actions": True, "pad_action_dim": 39, "pad_state_dim": 39}, "field": {"action_norm_type": "bounds_q99", "normalize_state": True, "pad_action_dim": 39, "pad_state_dim": 39, "stat_path": str(PROJECT / "assets/stats/seed_all_g1_motion39_textop_stats.json")}, "model": {"audio_only": True, "text_only": True, "video_only": False, "img_aug": False}, }, }, "model": { "model_name_or_path": str(BACKBONE), "tune_vlm": True, "gradient_checkpointing": False, "lang_backbone_lr": 2e-6, "streaming_qwen_enabled": True, "streaming_qwen_trainable_last_layers": 2, "streaming_qwen_train_final_norm": True, "instruction_recency_slots": 3, "state_aware_serial_control": False, "predict_motion_duration": True, "motion_duration_distribution_bins": 48, "action_dim": 39, "odim": 39, "action_chunk_size": 30, "action_exec_horizon": 20, "observation_horizon": 1, "view_feature_dim": 2048, "use_future_progress_embedding": True, "future_progress_dim": 5, "rtc": True, "max_delay": 11, }, }) def check_smoke(smoke_run, manifest): if smoke_run is None: raise ValueError("Formal training requires --smoke-run with a successful short-run directory") smoke_run = contained_output(smoke_run) events = [json.loads(line) for line in (smoke_run / "events.jsonl").read_text().splitlines() if line] if not any(event.get("event") == "smoke_checkpoint_reload_passed" for event in events): raise ValueError("Smoke checkpoint save/full-state reload has not passed") if not any(event.get("event") == "checkpoint_probe_complete" and event.get("step", 0) > 0 for event in events): raise ValueError("Smoke checkpoint prediction/video check has not passed") config = json.loads((smoke_run / "run_config.json").read_text()) if Path(config["data"]["frozen_manifest"]).resolve() != manifest: raise ValueError("Smoke run did not use this accepted manifest") identity = json.loads((smoke_run / "input_identity.json").read_text()) if identity["manifest_sha256"] != hashlib.sha256(manifest.read_bytes()).hexdigest(): raise ValueError("Manifest changed since smoke training") state = json.loads((smoke_run / "run_status.json").read_text()) if state.get("training_returncode") != 0 or state.get("phase") != "completed": raise ValueError("Smoke training has not exited successfully") for name, expected in json.loads((smoke_run / "code_identity.json").read_text()).items(): if hashlib.sha256((PROJECT / name).read_bytes()).hexdigest() != expected: raise ValueError(f"Training implementation changed since smoke validation: {name}") def validate(args): from scripts.data.prepare_third_party_training import validate_acceptance manifest = args.manifest.resolve() run_dir = contained_output(args.run_dir) report = validate_acceptance( selection_path=manifest, production_root=PRODUCTION, acceptance_path=manifest.parent / "acceptance.json", deep_payload=True, ) if not BACKBONE.is_dir(): raise FileNotFoundError(BACKBONE) config = build_config(manifest, run_dir, args.phase) from scripts.infer.third_party_checkpoint_probe import check_renderer check_renderer() if getattr(args, "resume_checkpoint", None): if args.phase != "train" or args.smoke_run: raise ValueError("Use --phase train --resume-checkpoint without --smoke-run") from scripts.third_party_resume import prepare_resume config, args.resume_record = prepare_resume( config, args.resume_checkpoint, manifest=manifest, run_dir=run_dir, ) elif args.phase == "train": check_smoke(args.smoke_run, manifest) return manifest, run_dir, report, config def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) parser.add_argument("--run-dir", type=Path, required=True) parser.add_argument("--phase", choices=("smoke", "train"), default="smoke") parser.add_argument("--smoke-run", type=Path) parser.add_argument("--resume-checkpoint", type=Path, help="Resume full state into a separate new run directory") mode = parser.add_mutually_exclusive_group() mode.add_argument("--check-only", action="store_true") mode.add_argument("--launch", action="store_true") mode.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) args = parser.parse_args() os.chdir(PROJECT) # This entry is intentionally single-process, even if invoked from a shell # that previously ran torchrun/Accelerate/SLURM. Do this before psi imports. for key in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT", "ACCELERATE_USE_DEEPSPEED", "ACCELERATE_USE_FSDP", "ACCELERATE_USE_MEGATRON_LM", "ACCELERATE_USE_CPU", "SLURM_PROCID", "SLURM_LOCALID", "SLURM_NTASKS"): os.environ.pop(key, None) os.environ["OMP_NUM_THREADS"] = "4" if args.worker: cache_root = PROJECT / "cache/third_party_training" for key, name in (("TORCHINDUCTOR_CACHE_DIR", "inductor"), ("TRITON_CACHE_DIR", "triton"), ("CUDA_CACHE_PATH", "cuda")): directory = cache_root / name directory.mkdir(parents=True, exist_ok=True) os.environ[key] = str(directory) from scripts.third_party_temp import configure_project_temp temporary_storage = configure_project_temp(PROJECT) manifest, run_dir, report, config = validate(args) if not args.launch and not args.worker: print(json.dumps({"passed": True, "phase": args.phase, "train_count": 300, "manifest_sha256": report["manifest_sha256"], "run_dir": str(run_dir), "resume": getattr(args, "resume_record", None), "visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0}, indent=2)) return 0 if args.launch: shared = [str(PROJECT / "scripts/train_third_party.py"), "--manifest", str(manifest), "--run-dir", str(run_dir), "--phase", args.phase] if args.smoke_run: shared.extend(["--smoke-run", str(args.smoke_run.resolve())]) if args.resume_checkpoint: shared.extend(["--resume-checkpoint", str(args.resume_checkpoint.resolve())]) spec = {"schema": 1, "project_root": str(PROJECT), "run_dir": str(run_dir), "gpu": 3, "manifest_path": str(manifest), "readiness_file": str(manifest.parent / "acceptance.json"), "argv": [sys.executable, *shared, "--worker"], "preflight_command": [sys.executable, *shared, "--check-only"]} spec_path = manifest.parent / f"{run_dir.name}.launch.json" if spec_path.exists() and json.loads(spec_path.read_text()) != spec: raise ValueError("Refusing to overwrite a different launch specification") spec_path.write_text(json.dumps(spec, indent=2) + "\n") return subprocess.call([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(spec_path)]) if os.environ.get("CUDA_VISIBLE_DEVICES") != "3" or os.environ.get("HFD_NPROC_PER_NODE") != "1": raise RuntimeError("Use --launch; the worker requires the single-GPU managed environment") if int(os.environ.get("WORLD_SIZE", "1")) != 1: raise RuntimeError("This accepted pilot entry supports exactly one GPU") # Explicit local/offline setup; never source the owner's .env or export credentials. os.environ.update({"HFD_SKIP_DOTENV": "1", "PSI_SKIP_VLM_MODEL": "0", "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", "PSI_FLOW_SIGMA_MIN": "0.05", "PSI_FLOW_SIGMA_MAX": "1.0", "PSI_WORKER_TORCH_THREADS": "1", "PSI_DATALOADER_PIN_MEMORY": "1", "PSI_FOREACH_GRAD_NORM": "1", "OMP_NUM_THREADS": "4"}) os.environ.pop("_ENV_INFO_JSON", None) os.environ["HFD_CONTINUOUS_DATA_ROOT"] = config.data.continuous_audio_root os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = config.data.background_assignment_path os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str( Path(config.data.continuous_audio_root) / "reports/production_complete.json") run_dir.mkdir(parents=True, exist_ok=True) (run_dir / "input_identity.json").write_text(json.dumps({ "manifest_sha256": report["manifest_sha256"], "phase": args.phase, "qwen_initialization": "checkpoint" if args.resume_checkpoint else "pretrained", "hfd_initialization": "checkpoint" if args.resume_checkpoint else "fresh", "train_count": 300, "temporary_storage": temporary_storage, "resume_checkpoint": str(args.resume_checkpoint.resolve()) if args.resume_checkpoint else None, }, indent=2) + "\n") from scripts.third_party_resume import code_identity (run_dir / "code_identity.json").write_text(json.dumps(code_identity(), indent=2) + "\n") if args.resume_checkpoint: (run_dir / "resume_identity.json").write_text(json.dumps(args.resume_record, indent=2) + "\n") from scripts.train import train train(config) return 0 if __name__ == "__main__": raise SystemExit(main())