File size: 4,986 Bytes
9fc9873 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 | #!/usr/bin/env python3
"""Current joint launcher. --print-config is CPU-only and creates no run."""
import json
import os
from pathlib import Path
import sys
from psi.hfd_training_launcher import resolve, runtime_environment, to_args
def main():
root = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(root))
os.chdir(root) # Match the old shell: relative paths are repo-relative.
extra = sys.argv[1:]
prepare_data = "--prepare-data" in extra
config_only = os.environ.get("HFD_CONFIG_ONLY", "0") in {"1", "json"} or "--print-config" in extra
extra = [arg for arg in extra if arg not in {"--print-config", "--prepare-data"}]
options, mode = resolve(root, os.environ, extra)
env = runtime_environment(options, os.environ)
args = to_args(options)
if config_only:
# No dataset scan, network request, monitor, CUDA import or output dir.
print(json.dumps(options, ensure_ascii=False, indent=2))
return
# The same parser as train.py checks every option before starting workers.
import tyro
from psi.config.train.posttrain_full_duplex_audio_command_psi0_config import DynamicLaunchConfig
parsed = tyro.cli(DynamicLaunchConfig, config=(tyro.conf.ConsolidateSubcommandArgs,), args=args[1:])
if prepare_data:
# Exactly the training configuration, no models, CUDA, WandB or new run.
os.environ.update(env)
os.environ["CUDA_VISIBLE_DEVICES"] = ""
from scripts.data.prepare_full_duplex_indexes import warm_indexes
warm_indexes(parsed.data)
return
from psi.hfd_training_preflight import validate_inputs, validate_wandb, validate_output_storage
if mode != 'contract':
validate_output_storage(options['train.output-dir'],env)
validate_inputs(root, options, env, mode)
if mode == "contract":
print("[ready] Data contracts verified; contract mode does not start training.")
return
validate_wandb(options, env)
import torch
count = torch.cuda.device_count()
requested = int(env.get("HFD_NPROC_PER_NODE", count))
if not 1 <= requested <= count:
raise ValueError(f"HFD_NPROC_PER_NODE={requested}; allocated CUDA device_count={count}")
o = options
print("[HFD] action input MLP precision=" +
("FP32" if o.get("model.action-input-projection-fp32", False) else "legacy AMP (saved run)"),
flush=True)
print(f"[HFD] architecture={o.get('model.interaction-architecture', 'legacy')} "
"(a1_c_deferred: shared frozen layers 0-27 + private tail8/cross; "
"anchored state token prepended to next AV packet)", flush=True)
ratios = [o[f"data.{name}-task-session-ratio"] for name in ("interruption", "control", "backchannel")]
print(f"[HFD] joint semantics=24 | GPUs={requested} B/GPU={o['train.train-batch-size']} "
f"steps={o['train.max-training-steps']} warmup={o['train.warmup-steps']}", flush=True)
print(f"[HFD] audio session mix: turn-taking={1-sum(ratios):.1%} interruption={ratios[0]:.1%} "
f"stop={ratios[1]:.1%} backchannel={ratios[2]:.1%}; hold=off", flush=True)
print(f"[HFD] audio window shares: hesitation={o['data.hesitation-task-window-ratio']:.1%} "
f"pause={o['data.pause-task-window-ratio']:.1%}; AV={mode}", flush=True)
if o.get('data.third-party-root'):
print('[HFD] third-party: append approved split selections once; preserve original pools; '
'current listen-hold/root40 + shared visual overlay; total ratio reported by dataset.', flush=True)
if o.get('data.multimodal-overlay-manifests'):
print(f"[HFD] all-task overlay: QA={o.get('data.multimodal-overlay-qa-ratio',.75):.1%}, "
f"visual-only={o.get('data.multimodal-overlay-visual-only-ratio',.10):.1%}; "
"remaining audio-only; same voice per session, no independent SWA task. "
"These are selection probabilities; unsafe insertion falls back to audio-only.",flush=True)
print(f"[HFD] save={o['train.checkpointing-steps']} eval={o['train.validation-steps']} "
f"eval B={o['train.val-batch-size']}; action/Qwen strides=1/20; RTC=30/20/10", flush=True)
print(f"[HFD] timestamp={o['timestamp']} resume={o['train.resume-from-checkpoint']} "
f"full_state={o['train.resume-load-full-state']} wandb={o['wandb.resume']}", flush=True)
command = [sys.executable, "-m", "torch.distributed.run", "--standalone",
"--local_addr=" + env.get("HFD_TORCHRUN_LOCAL_ADDR", "127.0.0.1"),
f"--nproc_per_node={requested}", "scripts/train.py", *args]
if env.get("HFD_MANAGE_GPU_MONITOR", "0") == "1":
command = ["bash", str(root / "scripts/train_with_gpu_monitor.sh"), *command]
os.chdir(root)
os.execvpe(command[0], command, env)
if __name__ == "__main__":
try:
main()
except (ValueError, FileNotFoundError) as exc:
raise SystemExit(f"[error] {exc}") from None
|