Download scripts/train_third_party.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 13.8 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/train_third_party.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/train_third_party.py
-
curl -L -o train_third_party.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/train_third_party.py
13.8 kB
| #!/usr/bin/env python3 | |
| """Validate, launch, or execute the fixed third-party-only small experiment.""" | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import subprocess | |
| import sys | |
| PROJECT = Path(__file__).resolve().parents[1] | |
| for directory in (PROJECT, PROJECT / "src"): | |
| if str(directory) not in sys.path: | |
| sys.path.insert(0, str(directory)) | |
| PRODUCTION = PROJECT / "data/hfd_third_party_multi_v5/production" | |
| DEFAULT_MANIFEST = PROJECT / "data/hfd_third_party_multi_v5/training_pilot_300/selection.json" | |
| BACKBONE = Path("/cpfs01/thscc/sharestorage/AIGC-06-01/Hithink_VideoGen_Group/team_workspace/public_ckpt/Qwen/Qwen2.5-Omni-3B") | |
| def contained_output(path): | |
| value = Path(path).absolute().resolve() | |
| if not value.is_relative_to(PROJECT / "runs") or value == PROJECT / "runs": | |
| raise ValueError("Experiment output must be a child directory of HFD_lzk/runs") | |
| return value | |
| def build_config(manifest, run_dir, phase): | |
| from psi.config.train.posttrain_third_party_config import DynamicLaunchConfig | |
| return DynamicLaunchConfig.model_validate({ | |
| "exp": "third_party", "seed": 26100401, "auto_tag_run": False, | |
| "timestamp": run_dir.name, "eval": False, "debug": False, | |
| "log": {"report_to": None, "log_freq": 20}, | |
| "wandb": {"project": "hfd-lzk-third-party", "resume": "never"}, | |
| "train": { | |
| "name": "posttrain_third_party", "experiment_phase": phase, | |
| "probe_count": 2 if phase == "smoke" else 12, "coverage_epochs": 3, | |
| "output_dir": str(run_dir), "data_parallel": "ddp", "mixed_precision": "bf16", | |
| "train_batch_size": 2, "target_effective_sources_per_gpu": 2, | |
| "val_batch_size": 1, "num_workers": 2, "val_num_workers": 0, | |
| "source_length_lookahead_batches": 1, "dynamic_source_refill": False, | |
| "gradient_accumulation_steps": 1, "source_accumulation_groups": 1, | |
| "resume_from_checkpoint": None, "resume_load_full_state": False, | |
| "max_training_steps": 50, "checkpointing_steps": 50, | |
| "validation_steps": 0, "val_num_batches": 0, "max_checkpoints_to_keep": 2, | |
| "learning_rate": 8e-5, "warmup_steps": 1, "warmup_ratio": None, | |
| "cosine_min_lr_rate": 0.05, "max_grad_norm": 1.0, | |
| "lr_scheduler_kwargs": {"betas": [0.9, 0.999], "weight_decay": 0.0, "eps": 1e-8}, | |
| "full_duplex_semantics_version": 24, "qwen_kv_gradient_horizon_windows": 1, | |
| "dense_window_accumulation_steps": 1, "full_source_gradient_checkpointing": False, | |
| "control_loss_weight": 1.0, "control_head_lr_multiplier": 0.25, | |
| "joint_velocity_loss_weight": 0.1, "joint_acceleration_loss_weight": 0.05, | |
| "root_continuity_loss_weight": 0.1, "stationary_root_height_loss_weight": 1.0, | |
| "motion_duration_loss_weight": 0.2, "latest_instruction_loss_weight": 0.1, | |
| "instruction_memory_loss_weight": 0.05, | |
| "loss_group_weight": {"joint_abs": 1.0, "root_delta_xy": 3.0, | |
| "root_rot5_delta": 10.0, "root_height": 15.0, "foot_contact": 0.1}, | |
| }, | |
| "data": { | |
| "frozen_manifest": str(manifest), "production_root": str(PRODUCTION), | |
| "online_audio": True, "multi_turn_sessions": True, "causal_listen_timing": True, | |
| "stationary_heading_alignment": True, "safe_stationary_anchors": True, | |
| "stationary_idle_residual_retargeting": True, "dynamic_task_entry_blend": True, | |
| "initial_idle_max_steps": 7, "turn_transition_frames": 50, | |
| "max_speech_onset_steps": 3, "listen_reaction_steps": 1, | |
| "max_stream_steps": 1919, "long_session_drop_ratio": 0.0, | |
| "filter_known_tts_translation_mismatches": True, | |
| "session_voice_variants": 1, "interruption_sessions": False, | |
| "independent_task_pools": False, "safe_hold_session_variants": 0, | |
| "self_repair_implicit_ratio": 0.0, | |
| "transform": { | |
| "repack": {"action_chunk_size": 30, "rtc_prefix_frames": 10, | |
| "textop_actions": True, "pad_action_dim": 39, "pad_state_dim": 39}, | |
| "field": {"action_norm_type": "bounds_q99", "normalize_state": True, | |
| "pad_action_dim": 39, "pad_state_dim": 39, | |
| "stat_path": str(PROJECT / "assets/stats/seed_all_g1_motion39_textop_stats.json")}, | |
| "model": {"audio_only": True, "text_only": True, "video_only": False, "img_aug": False}, | |
| }, | |
| }, | |
| "model": { | |
| "model_name_or_path": str(BACKBONE), "tune_vlm": True, "gradient_checkpointing": False, | |
| "lang_backbone_lr": 2e-6, "streaming_qwen_enabled": True, | |
| "streaming_qwen_trainable_last_layers": 2, "streaming_qwen_train_final_norm": True, | |
| "instruction_recency_slots": 3, "state_aware_serial_control": False, | |
| "predict_motion_duration": True, "motion_duration_distribution_bins": 48, | |
| "action_dim": 39, "odim": 39, "action_chunk_size": 30, "action_exec_horizon": 20, | |
| "observation_horizon": 1, "view_feature_dim": 2048, | |
| "use_future_progress_embedding": True, "future_progress_dim": 5, "rtc": True, "max_delay": 11, | |
| }, | |
| }) | |
| def check_smoke(smoke_run, manifest): | |
| if smoke_run is None: | |
| raise ValueError("Formal training requires --smoke-run with a successful short-run directory") | |
| smoke_run = contained_output(smoke_run) | |
| events = [json.loads(line) for line in (smoke_run / "events.jsonl").read_text().splitlines() if line] | |
| if not any(event.get("event") == "smoke_checkpoint_reload_passed" for event in events): | |
| raise ValueError("Smoke checkpoint save/full-state reload has not passed") | |
| if not any(event.get("event") == "checkpoint_probe_complete" and event.get("step", 0) > 0 for event in events): | |
| raise ValueError("Smoke checkpoint prediction/video check has not passed") | |
| config = json.loads((smoke_run / "run_config.json").read_text()) | |
| if Path(config["data"]["frozen_manifest"]).resolve() != manifest: | |
| raise ValueError("Smoke run did not use this accepted manifest") | |
| identity = json.loads((smoke_run / "input_identity.json").read_text()) | |
| if identity["manifest_sha256"] != hashlib.sha256(manifest.read_bytes()).hexdigest(): | |
| raise ValueError("Manifest changed since smoke training") | |
| state = json.loads((smoke_run / "run_status.json").read_text()) | |
| if state.get("training_returncode") != 0 or state.get("phase") != "completed": | |
| raise ValueError("Smoke training has not exited successfully") | |
| for name, expected in json.loads((smoke_run / "code_identity.json").read_text()).items(): | |
| if hashlib.sha256((PROJECT / name).read_bytes()).hexdigest() != expected: | |
| raise ValueError(f"Training implementation changed since smoke validation: {name}") | |
| def validate(args): | |
| from scripts.data.prepare_third_party_training import validate_acceptance | |
| manifest = args.manifest.resolve() | |
| run_dir = contained_output(args.run_dir) | |
| report = validate_acceptance( | |
| selection_path=manifest, production_root=PRODUCTION, | |
| acceptance_path=manifest.parent / "acceptance.json", deep_payload=True, | |
| ) | |
| if not BACKBONE.is_dir(): | |
| raise FileNotFoundError(BACKBONE) | |
| config = build_config(manifest, run_dir, args.phase) | |
| from scripts.infer.third_party_checkpoint_probe import check_renderer | |
| check_renderer() | |
| if getattr(args, "resume_checkpoint", None): | |
| if args.phase != "train" or args.smoke_run: | |
| raise ValueError("Use --phase train --resume-checkpoint without --smoke-run") | |
| from scripts.third_party_resume import prepare_resume | |
| config, args.resume_record = prepare_resume( | |
| config, args.resume_checkpoint, manifest=manifest, run_dir=run_dir, | |
| ) | |
| elif args.phase == "train": | |
| check_smoke(args.smoke_run, manifest) | |
| return manifest, run_dir, report, config | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) | |
| parser.add_argument("--run-dir", type=Path, required=True) | |
| parser.add_argument("--phase", choices=("smoke", "train"), default="smoke") | |
| parser.add_argument("--smoke-run", type=Path) | |
| parser.add_argument("--resume-checkpoint", type=Path, | |
| help="Resume full state into a separate new run directory") | |
| mode = parser.add_mutually_exclusive_group() | |
| mode.add_argument("--check-only", action="store_true") | |
| mode.add_argument("--launch", action="store_true") | |
| mode.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) | |
| args = parser.parse_args() | |
| os.chdir(PROJECT) | |
| # This entry is intentionally single-process, even if invoked from a shell | |
| # that previously ran torchrun/Accelerate/SLURM. Do this before psi imports. | |
| for key in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE", | |
| "MASTER_ADDR", "MASTER_PORT", "ACCELERATE_USE_DEEPSPEED", | |
| "ACCELERATE_USE_FSDP", "ACCELERATE_USE_MEGATRON_LM", | |
| "ACCELERATE_USE_CPU", "SLURM_PROCID", "SLURM_LOCALID", "SLURM_NTASKS"): | |
| os.environ.pop(key, None) | |
| os.environ["OMP_NUM_THREADS"] = "4" | |
| if args.worker: | |
| cache_root = PROJECT / "cache/third_party_training" | |
| for key, name in (("TORCHINDUCTOR_CACHE_DIR", "inductor"), | |
| ("TRITON_CACHE_DIR", "triton"), | |
| ("CUDA_CACHE_PATH", "cuda")): | |
| directory = cache_root / name | |
| directory.mkdir(parents=True, exist_ok=True) | |
| os.environ[key] = str(directory) | |
| from scripts.third_party_temp import configure_project_temp | |
| temporary_storage = configure_project_temp(PROJECT) | |
| manifest, run_dir, report, config = validate(args) | |
| if not args.launch and not args.worker: | |
| print(json.dumps({"passed": True, "phase": args.phase, "train_count": 300, | |
| "manifest_sha256": report["manifest_sha256"], "run_dir": str(run_dir), | |
| "resume": getattr(args, "resume_record", None), | |
| "visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0}, indent=2)) | |
| return 0 | |
| if args.launch: | |
| shared = [str(PROJECT / "scripts/train_third_party.py"), "--manifest", str(manifest), | |
| "--run-dir", str(run_dir), "--phase", args.phase] | |
| if args.smoke_run: | |
| shared.extend(["--smoke-run", str(args.smoke_run.resolve())]) | |
| if args.resume_checkpoint: | |
| shared.extend(["--resume-checkpoint", str(args.resume_checkpoint.resolve())]) | |
| spec = {"schema": 1, "project_root": str(PROJECT), "run_dir": str(run_dir), "gpu": 3, | |
| "manifest_path": str(manifest), "readiness_file": str(manifest.parent / "acceptance.json"), | |
| "argv": [sys.executable, *shared, "--worker"], | |
| "preflight_command": [sys.executable, *shared, "--check-only"]} | |
| spec_path = manifest.parent / f"{run_dir.name}.launch.json" | |
| if spec_path.exists() and json.loads(spec_path.read_text()) != spec: | |
| raise ValueError("Refusing to overwrite a different launch specification") | |
| spec_path.write_text(json.dumps(spec, indent=2) + "\n") | |
| return subprocess.call([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), | |
| "start", "--spec", str(spec_path)]) | |
| if os.environ.get("CUDA_VISIBLE_DEVICES") != "3" or os.environ.get("HFD_NPROC_PER_NODE") != "1": | |
| raise RuntimeError("Use --launch; the worker requires the single-GPU managed environment") | |
| if int(os.environ.get("WORLD_SIZE", "1")) != 1: | |
| raise RuntimeError("This accepted pilot entry supports exactly one GPU") | |
| # Explicit local/offline setup; never source the owner's .env or export credentials. | |
| os.environ.update({"HFD_SKIP_DOTENV": "1", "PSI_SKIP_VLM_MODEL": "0", | |
| "HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", | |
| "PSI_FLOW_SIGMA_MIN": "0.05", "PSI_FLOW_SIGMA_MAX": "1.0", | |
| "PSI_WORKER_TORCH_THREADS": "1", "PSI_DATALOADER_PIN_MEMORY": "1", | |
| "PSI_FOREACH_GRAD_NORM": "1", "OMP_NUM_THREADS": "4"}) | |
| os.environ.pop("_ENV_INFO_JSON", None) | |
| os.environ["HFD_CONTINUOUS_DATA_ROOT"] = config.data.continuous_audio_root | |
| os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = config.data.background_assignment_path | |
| os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str( | |
| Path(config.data.continuous_audio_root) / "reports/production_complete.json") | |
| run_dir.mkdir(parents=True, exist_ok=True) | |
| (run_dir / "input_identity.json").write_text(json.dumps({ | |
| "manifest_sha256": report["manifest_sha256"], "phase": args.phase, | |
| "qwen_initialization": "checkpoint" if args.resume_checkpoint else "pretrained", | |
| "hfd_initialization": "checkpoint" if args.resume_checkpoint else "fresh", "train_count": 300, | |
| "temporary_storage": temporary_storage, | |
| "resume_checkpoint": str(args.resume_checkpoint.resolve()) if args.resume_checkpoint else None, | |
| }, indent=2) + "\n") | |
| from scripts.third_party_resume import code_identity | |
| (run_dir / "code_identity.json").write_text(json.dumps(code_identity(), indent=2) + "\n") | |
| if args.resume_checkpoint: | |
| (run_dir / "resume_identity.json").write_text(json.dumps(args.resume_record, indent=2) + "\n") | |
| from scripts.train import train | |
| train(config) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |