File size: 4,986 Bytes
9fc9873
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
#!/usr/bin/env python3
"""Current joint launcher. --print-config is CPU-only and creates no run."""
import json
import os
from pathlib import Path
import sys

from psi.hfd_training_launcher import resolve, runtime_environment, to_args


def main():
    root = Path(__file__).resolve().parents[1]
    sys.path.insert(0, str(root))
    os.chdir(root)  # Match the old shell: relative paths are repo-relative.
    extra = sys.argv[1:]
    prepare_data = "--prepare-data" in extra
    config_only = os.environ.get("HFD_CONFIG_ONLY", "0") in {"1", "json"} or "--print-config" in extra
    extra = [arg for arg in extra if arg not in {"--print-config", "--prepare-data"}]
    options, mode = resolve(root, os.environ, extra)
    env = runtime_environment(options, os.environ)
    args = to_args(options)
    if config_only:
        # No dataset scan, network request, monitor, CUDA import or output dir.
        print(json.dumps(options, ensure_ascii=False, indent=2))
        return

    # The same parser as train.py checks every option before starting workers.
    import tyro
    from psi.config.train.posttrain_full_duplex_audio_command_psi0_config import DynamicLaunchConfig
    parsed = tyro.cli(DynamicLaunchConfig, config=(tyro.conf.ConsolidateSubcommandArgs,), args=args[1:])
    if prepare_data:
        # Exactly the training configuration, no models, CUDA, WandB or new run.
        os.environ.update(env)
        os.environ["CUDA_VISIBLE_DEVICES"] = ""
        from scripts.data.prepare_full_duplex_indexes import warm_indexes
        warm_indexes(parsed.data)
        return
    from psi.hfd_training_preflight import validate_inputs, validate_wandb, validate_output_storage
    if mode != 'contract':
        validate_output_storage(options['train.output-dir'],env)
    validate_inputs(root, options, env, mode)
    if mode == "contract":
        print("[ready] Data contracts verified; contract mode does not start training.")
        return
    validate_wandb(options, env)
    import torch
    count = torch.cuda.device_count()
    requested = int(env.get("HFD_NPROC_PER_NODE", count))
    if not 1 <= requested <= count:
        raise ValueError(f"HFD_NPROC_PER_NODE={requested}; allocated CUDA device_count={count}")
    o = options
    print("[HFD] action input MLP precision=" +
          ("FP32" if o.get("model.action-input-projection-fp32", False) else "legacy AMP (saved run)"),
          flush=True)
    print(f"[HFD] architecture={o.get('model.interaction-architecture', 'legacy')} "
          "(a1_c_deferred: shared frozen layers 0-27 + private tail8/cross; "
          "anchored state token prepended to next AV packet)", flush=True)
    ratios = [o[f"data.{name}-task-session-ratio"] for name in ("interruption", "control", "backchannel")]
    print(f"[HFD] joint semantics=24 | GPUs={requested} B/GPU={o['train.train-batch-size']} "
          f"steps={o['train.max-training-steps']} warmup={o['train.warmup-steps']}", flush=True)
    print(f"[HFD] audio session mix: turn-taking={1-sum(ratios):.1%} interruption={ratios[0]:.1%} "
          f"stop={ratios[1]:.1%} backchannel={ratios[2]:.1%}; hold=off", flush=True)
    print(f"[HFD] audio window shares: hesitation={o['data.hesitation-task-window-ratio']:.1%} "
          f"pause={o['data.pause-task-window-ratio']:.1%}; AV={mode}", flush=True)
    if o.get('data.third-party-root'):
        print('[HFD] third-party: append approved split selections once; preserve original pools; '
              'current listen-hold/root40 + shared visual overlay; total ratio reported by dataset.', flush=True)
    if o.get('data.multimodal-overlay-manifests'):
        print(f"[HFD] all-task overlay: QA={o.get('data.multimodal-overlay-qa-ratio',.75):.1%}, "
              f"visual-only={o.get('data.multimodal-overlay-visual-only-ratio',.10):.1%}; "
              "remaining audio-only; same voice per session, no independent SWA task. "
              "These are selection probabilities; unsafe insertion falls back to audio-only.",flush=True)
    print(f"[HFD] save={o['train.checkpointing-steps']} eval={o['train.validation-steps']} "
          f"eval B={o['train.val-batch-size']}; action/Qwen strides=1/20; RTC=30/20/10", flush=True)
    print(f"[HFD] timestamp={o['timestamp']} resume={o['train.resume-from-checkpoint']} "
          f"full_state={o['train.resume-load-full-state']} wandb={o['wandb.resume']}", flush=True)
    command = [sys.executable, "-m", "torch.distributed.run", "--standalone",
               "--local_addr=" + env.get("HFD_TORCHRUN_LOCAL_ADDR", "127.0.0.1"),
               f"--nproc_per_node={requested}", "scripts/train.py", *args]
    if env.get("HFD_MANAGE_GPU_MONITOR", "0") == "1":
        command = ["bash", str(root / "scripts/train_with_gpu_monitor.sh"), *command]
    os.chdir(root)
    os.execvpe(command[0], command, env)


if __name__ == "__main__":
    try:
        main()
    except (ValueError, FileNotFoundError) as exc:
        raise SystemExit(f"[error] {exc}") from None