File size: 30,809 Bytes
55e0599
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
#!/usr/bin/env python3
"""Managed GPU0 full eligible-data stage, with held-out metrics and final videos."""
from __future__ import annotations

import argparse
import hashlib
import importlib.util
import json
import math
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time

PROJECT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
from scripts.third_party_strategy_plan import write_json
from scripts.third_party_scale_plan import coverage_plan

ORIGINAL_PILOT_CONFIG = PROJECT / "runs/third_party_train_20261004_183932/run_config.json"
CODE_FILES = ("scripts/train_third_party_scale.py", "scripts/third_party_scale_plan.py",
    "scripts/train_third_party.py", "scripts/train.py", "scripts/third_party_joint_init.py",
    "scripts/data/prepare_third_party_scale.py", "scripts/infer/third_party_scale_metrics.py",
    "scripts/infer/render_third_party_scale.py", "scripts/train/third_party_run.py",
    "src/psi/trainers/posttrain_full_duplex.py", "src/psi/trainers/posttrain_v1.py",
    "src/psi/data/seed/qwen_streaming_audio.py",
    "src/psi/data/seed/third_party_frozen_dataset.py")


def read(path):
    return json.loads(Path(path).read_text())


def digest(path):
    h = hashlib.sha256()
    with Path(path).open("rb") as f:
        for data in iter(lambda: f.read(4 * 1024 * 1024), b""):
            h.update(data)
    return h.hexdigest()


def validate_original_definition(current, original):
    """Compare configuration, never read the old experiment's learned tensors."""
    fields = {k for k in original["train"] if "loss" in k} | {
        "learning_rate", "control_head_lr_multiplier", "cosine_min_lr_rate", "max_grad_norm",
        "lr_scheduler_kwargs", "full_duplex_semantics_version", "qwen_kv_gradient_horizon_windows"}
    if (current["model"] != original["model"] or current["seed"] != original["seed"]
            or any(current["train"].get(k) != original["train"].get(k) for k in fields)
            or current["train"].get("resume_from_checkpoint") is not None
            or current["train"].get("resume_load_full_state") is not False
            or current["train"].get("resume_checkpoint_step", 0) != 0
            or current["model"].get("pretrained_action_header_path")
            or current["model"].get("vlm_ckpt_step")):
        raise ValueError("Fresh scale initialization/model/loss must match the original fresh pilot")


def configure(args):
    from scripts.data.prepare_third_party_scale import validate
    from scripts.train_third_party import contained_output, build_config
    from scripts.third_party_joint_init import _contract
    args.run_dir = contained_output(args.run_dir)
    args.manifest = args.manifest.resolve()
    selection, validation, readiness = validate(args.manifest)
    plan = coverage_plan(selection, epochs=args.epochs)
    if getattr(args, "parent", None) is not None:
        raise ValueError("Fresh scale training must not specify any parent weights")
    cfg = build_config(args.manifest, args.run_dir, "train")
    cfg.train.train_batch_size = cfg.train.target_effective_sources_per_gpu = 16
    cfg.train.num_workers = 8
    cfg.train.coverage_epochs = args.epochs
    cfg.train.max_training_steps = plan["max_optimizer_updates"]
    cfg.train.warmup_steps = 500  # Original fresh train300, not the later warm-start stage.
    cfg.train.checkpointing_steps = 20000
    cfg.train.max_checkpoints_to_keep = 3
    original = read(ORIGINAL_PILOT_CONFIG)
    validate_original_definition(cfg.model_dump(mode="json"), original)
    contract = _contract(cfg)
    identity = {"schema": 2, "initialization_policy": "original_pilot_fresh_v1",
        "parent": None, "parent_step": None, "qwen_initialization": "pretrained", "hfd_initialization": "fresh",
        "reference_pilot_config": str(ORIGINAL_PILOT_CONFIG), "reference_pilot_config_sha256": digest(ORIGINAL_PILOT_CONFIG),
        "train_infer_contract": contract, "initialization_seed": cfg.seed,
        "manifest_sha256": readiness["manifest_sha256"], "validation_sha256": readiness["validation_sha256"],
        "code_hashes": {name: digest(PROJECT / name) for name in CODE_FILES},
        "initialization": "original pretrained Qwen plus fresh HFD modules; no pilot checkpoint or adapter",
        "train_count": len(selection["session_ids"]), "validation_count": len(validation["session_ids"]),
        "optimizer_heldout_inputs": 0, "visual_inputs": 0, "other_task_inputs": 0,
        "gpu": 0, "base_step_not_added_to_new_stage": True}
    resume = None
    if args.resume_checkpoint:
        checkpoint = args.resume_checkpoint.resolve(); old_run = checkpoint.parent.parent
        if (not checkpoint.is_relative_to(PROJECT / "runs") or old_run == args.run_dir
                or old_run.is_relative_to(args.run_dir) or args.run_dir.is_relative_to(old_run)
                or not (checkpoint / "optimizer.bin").is_file()):
            raise ValueError("Use a complete scale checkpoint and a fresh output run")
        old_identity = read(old_run / "scale_identity.json")
        if old_identity != identity or read(old_run / "coverage_plan.json") != plan:
            raise ValueError("Scale resume cannot change inputs, implementation, initialization or budget")
        module_spec = importlib.util.spec_from_file_location("scale_resume_runner", PROJECT / "scripts/train/third_party_run.py")
        manager = importlib.util.module_from_spec(module_spec)
        sys.modules[module_spec.name] = manager
        module_spec.loader.exec_module(manager)
        process_identity = manager.identity
        import socket
        previous = read(old_run / "run_status.json")
        if previous["host"] != socket.gethostname() and not args.allow_host_change:
            raise ValueError("Host changed: review GPU allocation and use --allow-host-change")
        process = process_identity(previous["launcher_pid"])
        if process and previous["host"] == socket.gethostname() and process.start_ticks == previous["launcher_start_ticks"]:
            raise ValueError("Original scale launcher is still alive")
        cursor = read(checkpoint / "scale_cursor.json")
        step = int(checkpoint.name.removeprefix("ckpt_"))
        epoch = plan["epochs"][cursor["epoch"] - 1]
        expected = sum(e["optimizer_updates"] for e in plan["epochs"][:cursor["epoch"] - 1]) + sum(epoch["group_steps"][:cursor["next_group"]])
        if cursor["step"] != step or step != expected or not 0 < cursor["next_group"] <= len(epoch["source_indices"]):
            raise ValueError("Scale checkpoint coverage cursor mismatch")
        marker = read(checkpoint / "full_duplex_source_boundary.json")
        if marker.get("global_step") != step or not marker.get("source_group_complete"):
            raise ValueError("Scale resume is not at a complete source boundary")
        resume = {"checkpoint": str(checkpoint), "old_run": str(old_run), "cursor": cursor}
    return cfg, selection, validation, plan, identity, resume


def publish(run):
    from scripts.third_party_dashboard import render_curves
    state = read(run / "workflow_state.json")
    identity, plan = read(run / "scale_identity.json"), read(run / "coverage_plan.json")
    evaluations = read(run / "evaluations.json") if (run / "evaluations.json").exists() else []
    if (run / "metrics.jsonl").exists():
        points = [json.loads(l) for l in (run / "metrics.jsonl").read_text().splitlines() if l.strip()]
        (run / "curves.svg").write_text(render_curves(points))
    lines = ["# Third-party 全量训练与验证", "",
        f"GPU 0;训练 {identity['train_count']} 条/独立验证 {identity['validation_count']} 条;只输入语音,不混入其他任务。",
        f"当前阶段:`{state['phase']}`;新阶段 step {state.get('step',0)}/{plan['max_optimizer_updates']};最多 {plan['max_epochs']} 遍。",
        "与最初300条实验相同:原版Qwen预训练+新初始化HFD模块,不加载890620或控制头adapter。新优化器/学习率,step从0计;预算完成不代表收敛。", "",
        "![训练 loss 曲线](curves.svg)", "",
        "[实时状态](workflow_state.json) · [原始 loss](metrics.jsonl) · [覆盖游标](coverage_state.json) · [GPU/占卡状态](run_status.json)", "",
        "中途固定训练/验证面板按类型与语言抽取,不能把均衡面板均值当作整个验证集分布。最终另测全部可用验证样本。", "",
        "| 新阶段 step | 范围 | 条数 | 控制准确率 | 背景误动率 | 漏动作率 | 关节 MAE | 报告 |",
        "|---:|---|---:|---:|---:|---:|---:|---|"]
    for item in evaluations:
        for split in ("train", "validation"):
            if split not in item:
                continue
            r = item[split]; a = r["aggregate"]
            label = "训练" if split == "train" else "独立验证"
            if r.get("panel") == "all_eligible": label += "全量"
            def fmt(key):
                return "不适用" if a.get(key) is None else f"{a[key]:.2%}"
            link = os.path.relpath(r["report_path"], run)
            lines.append(f"| {item['step']} | {label} | {r['count']} | {fmt('control_accuracy')} | "
                f"{fmt('background_only_false_motion_rate')} | {fmt('missed_motion_rate')} | {a['joint_mae_radians']:.5f} | [逐条与分组]({link}) |")
    lines += ["", "训练结束后自动从保存的预测生成审核视频,不把 GT 当作模型效果。",
              "[最终视频索引](prediction_videos/README.md) · [最终结果](result.json)", ""]
    (run / "review.md").write_text("\n".join(lines))


def verify_fresh_initialization(trainer, identity):
    """Read-only audit of the original initializer; never install old weights."""
    from safetensors import safe_open
    import torch
    from scripts.third_party_joint_init import _trainability
    if identity.get("initialization_policy") != "original_pilot_fresh_v1" or identity.get("parent") is not None:
        raise ValueError("Expected fresh initialization without a learned parent")
    if trainer.global_step != 0 or trainer.cfg.train.resume_from_checkpoint is not None:
        raise ValueError("Fresh initialization must precede every optimizer update/load")
    model = trainer.model
    parameters, qwen, expected = _trainability(model, trainer.cfg)
    root = Path(trainer.cfg.model.model_name_or_path)
    weights = read(root / "model.safetensors.index.json")["weight_map"]
    state = model.state_dict()
    for name in sorted(qwen):
        key = name.removeprefix("vlm_model.")
        if key not in weights:
            key = "thinker." + key
        with safe_open(str(root / weights[key]), framework="pt", device="cpu") as tensors:
            original = tensors.get_tensor(key).to(dtype=state[name].dtype)
        if not torch.equal(state[name].detach().cpu(), original):
            raise ValueError(f"Qwen trainable tail is not the original pretrained tensor: {name}")
    if trainer._hfdv2_checkpoint_contract() != identity["train_infer_contract"]:
        raise ValueError("Fresh runtime checkpoint contract differs")
    return {"initialization_policy": identity["initialization_policy"], "loaded_hfd_checkpoints": [],
        "loaded_control_adapters": [], "qwen_tail_verified_against_pretrained": len(qwen),
        "compact_tensor_count": len(expected),
        "trainable_parameters": sum(p.numel() for p in parameters.values() if p.requires_grad),
        "new_step": 0, "optimizer_reset": True, "scheduler_reset": True, "control_head_replaced": False}


def validation_regressed(candidate, reference):
    # An untrained all-idle predictor is not a trained quality target. Start
    # this safety comparison at the first actual trained fixed-panel result.
    if reference is None:
        return False
    return (candidate["control_accuracy"] < reference["control_accuracy"] - .03
        or candidate["missed_motion_rate"] > reference["missed_motion_rate"] + .03
        or candidate["joint_mae_radians"] > reference["joint_mae_radians"] * 1.20)


def worker(args, prepared):
    import torch
    from accelerate.utils import set_seed
    from torch.utils.data import DataLoader
    from scripts.train import _initialize_accelerator
    from scripts.train_third_party_strategy import source_batches
    from scripts.third_party_convergence_plan import EpochLossAccumulator
    from scripts.infer.third_party_scale_metrics import evaluate
    from psi.trainers.posttrain_third_party import PosttrainThirdPartyTrainer, append_record
    from psi.trainers.posttrain_full_duplex import _source_collator
    from psi.utils import flatten, seed_everything
    cfg, selection, validation, plan, identity, resume = prepared
    os.environ["HFD_CONTINUOUS_DATA_ROOT"] = cfg.data.continuous_audio_root
    os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = cfg.data.background_assignment_path
    os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str(Path(cfg.data.continuous_audio_root) / "reports/production_complete.json")
    run = args.run_dir
    if (run / "scale_identity.json").exists():
        raise ValueError("Refusing to overwrite an existing scale stage")
    run.mkdir(parents=True, exist_ok=True)
    for name, value in (("scale_identity.json", identity), ("coverage_plan.json", plan),
                        ("input_identity.json", identity), ("evaluations.json", [])):
        write_json(run / name, value)
    (run / "run_config.json").write_text(cfg.model_dump_json(indent=2))
    evaluations = []
    step = 0
    def phase(name, **fields):
        write_json(run / "workflow_state.json", {"phase": name, "step": step, "time": time.time(), **fields})
    def evaluated(source, checkpoint_step, label, stage_step, final=False):
        record = {"step": stage_step, "source_step": checkpoint_step, "label": label}
        for split, manifest in (("validation", args.manifest.parent / "validation.json"), ("train", args.manifest)):
            phase(f"{label}_{split}_metrics")
            output = run / "evaluations" / label / split
            result = evaluate(source, checkpoint_step, manifest, output,
                full=final and split == "validation", include_videos=final, workers=4)
            record[split] = {"aggregate": result["aggregate"], "count": result["count"],
                "report_path": str(output / "report.json"), "panel": result["panel"],
                "fixed_panel_comparison": result["fixed_panel_comparison"]}
            write_json(run / "current_evaluation.json", record)
        evaluations.append(record); write_json(run / "evaluations.json", evaluations)
        publish(run)
        return record
    phase("initializing_full_training"); publish(run)
    if resume:
        old = Path(resume["old_run"])
        step = resume["cursor"]["step"]
        evaluations = [r for r in read(old / "evaluations.json") if r["step"] <= step]
        write_json(run / "evaluations.json", evaluations)
        write_json(run / "resume_identity.json", resume)
        # Loss records are references, not regenerated observations.
        (run / "previous_run.txt").write_text(str(old) + "\n")
    class ScaleTrainer(PosttrainThirdPartyTrainer):
        @property
        def num_steps_per_epoch(self):
            return plan["epochs"][0]["optimizer_updates"]

        def create_datasets(self):
            self.train_dataset = self.data_cfg(split="train", transform_kwargs={"vlm_processor": self.vlm_processor})
            if len(self.train_dataset) != len(selection["sessions"]):
                raise ValueError("Full frozen coverage count changed")
            self.val_dataset = None
            return self.train_dataset, None

        def _checkpoint_probe(self, global_step):
            return None

        def log(self, metrics, start_time=None):
            self._dashboard_step = self.global_step  # publish full-scale review separately
            super().log(metrics, start_time)
    seed_everything(cfg.seed); torch.cuda.set_device(0)
    trainer = ScaleTrainer(cfg, 0); trainer.init_models(); trainer.global_step = 0
    if not resume:
        write_json(run / "initialization_verified.json", verify_fresh_initialization(trainer, identity))
    accelerator = _initialize_accelerator(trainer)
    trainer.create_datasets()
    trainer.train_dataloader = DataLoader(trainer.train_dataset, batch_size=1, shuffle=False,
        num_workers=0, collate_fn=_source_collator)
    trainer.val_dataloader = None
    trainer.create_optimizer_and_scheduler(plan["max_optimizer_updates"])
    trainer.prepare(accelerator)
    set_seed(cfg.seed, device_specific=True)
    if resume:
        accelerator.load_state(resume["checkpoint"])
        if trainer.lr_scheduler.state_dict()["last_epoch"] != step or not trainer.optimizer.state_dict()["state"]:
            raise ValueError("Full-scale optimizer/schedule restore failed")
        write_json(run / "full_state_resume_verified.json", {"step": step, "passed": True})
        # Publish a local complete checkpoint even if resuming at the last
        # boundary solely to finish interrupted final evaluations.
        trainer.global_step = step
        trainer._at_source_group_boundary = True
        trainer._last_source_group_complete = True
        trainer._last_boundary_reason = "restored_complete_source_boundary"
        restored = Path(trainer.save_checkpoint(step))
        write_json(restored / "scale_cursor.json", resume["cursor"])
    elif trainer.optimizer.state_dict()["state"]:
        raise ValueError("New scale stage inherited optimizer moments")
    else:
        if trainer.lr_scheduler.state_dict()["last_epoch"] != 0:
            raise ValueError("Fresh learning-rate schedule is not at step zero")
        # No source/KV has been opened: this is a genuine cache-free boundary.
        trainer._at_source_group_boundary = True
        trainer._last_source_group_complete = True
        trainer._last_boundary_reason = "fresh_before_first_source"
        checkpoint = Path(trainer.save_checkpoint(0))
        write_json(run / "initial_checkpoint.json", {"step": 0, "checkpoint": str(checkpoint),
            "weights_sha256": digest(checkpoint / "model.safetensors"),
            "optimizer_state_entries": 0, "training_updates": 0, "not_a_pilot_checkpoint": True})
        torch.cuda.empty_cache()
        evaluated(run, 0, "baseline", 0)
    trainer.global_step = step
    (run / "run_config.json").write_text(cfg.model_dump_json(indent=2))
    append_record(run / "events.jsonl", {"event": "scale_training_initialized", "step": step,
        "train_count": len(selection["sessions"]), "batch": 16, "time": time.time(), "full_state_resume": bool(resume)})
    last_saved, reload_passed = step, bool(resume)
    best_bg = float("inf"); regressions = 0
    reference = None
    for record in evaluations:
        if record["step"] <= 0:
            continue
        candidate = record["validation"]["fixed_panel_comparison"]["aggregate"]
        bad = validation_regressed(candidate, reference)
        regressions = regressions + 1 if bad else 0
        if reference is None:
            reference = candidate
        if not bad and candidate["background_only_false_motion_rate"] is not None:
            best_bg = min(best_bg, candidate["background_only_false_motion_rate"])
    decision = {"action": "stop_budget", "converged": False, "reason": "bounded complete-coverage stage"}
    try:
        for epoch_plan in plan["epochs"]:
            epoch = epoch_plan["epoch"]
            if resume and epoch < resume["cursor"]["epoch"]:
                continue
            offset = resume["cursor"]["next_group"] if resume and epoch == resume["cursor"]["epoch"] else 0
            groups = epoch_plan["source_indices"]
            acc = EpochLossAccumulator()
            if resume and epoch == resume["cursor"]["epoch"]:
                c = resume["cursor"]["accumulator"]
                for attr, key in (("action_sum", "action_loss_sum"), ("action_samples", "action_samples"),
                    ("control_sum", "control_ce_sum"), ("control_samples", "control_samples"),
                    ("optimizer_updates", "optimizer_updates"), ("control_updates", "control_updates")):
                    setattr(acc, attr, c[key])
            completed = sum(len(g) for g in groups[:offset])
            interval = max(1, math.ceil(len(groups) / 10))
            trainer.set_train()
            for group_index, (indices, batch) in enumerate(zip(groups[offset:], source_batches(trainer, groups[offset:], workers=8)), offset):
                phase("training", epoch=epoch, completed_sources=completed, train_count=len(selection["sessions"]))
                cfg.train.train_batch_size = cfg.train.target_effective_sources_per_gpu = len(indices)
                before, boundary = step, False
                for _, values, boundary in trainer.iter_optimizer_steps(batch, global_step_start=step,
                    local_step=group_index, max_optimizer_events=epoch_plan["group_steps"][group_index]):
                    step += 1; trainer.global_step = step
                    acc.add(action_loss=values["loss_action"], action_samples=values["effective_sources_per_gpu"] * values["dense_windows_per_optimizer_step"],
                            control_ce=values.get("loss_control"), control_samples=values["control_update_samples"])
                    trainer.log(flatten({**values, "epoch": epoch, "completed_sources": completed}, parent_key="train"))
                    if step % 1000 == 0:
                        print(f"[scale] epoch={epoch}/{args.epochs} step={step}/{plan['max_optimizer_updates']} sources={completed}/{len(selection['sessions'])} action={values['loss_action']:.5f}", flush=True)
                        phase("training", epoch=epoch, completed_sources=completed, train_count=len(selection["sessions"]))
                    if step % 50000 == 0:
                        publish(run)
                    if torch.cuda.max_memory_reserved() / 2**30 > 120:
                        raise RuntimeError("GPU reserved memory exceeded 120 GiB safety ceiling")
                if not boundary or step - before != epoch_plan["group_steps"][group_index]:
                    raise RuntimeError("Incomplete scale source group")
                completed += len(indices)
                cfg.train.train_batch_size = cfg.train.target_effective_sources_per_gpu = 16
                del batch
                cursor = {"epoch": epoch, "step": step, "next_group": group_index + 1,
                          "completed_sources": completed, "accumulator": acc.summary()}
                write_json(run / "coverage_state.json", cursor)
                last_group = group_index + 1 == len(groups)
                needs_eval = (group_index + 1) % interval == 0 or last_group
                if not reload_passed or step - last_saved >= 20000 or needs_eval:
                    phase("saving_checkpoint", epoch=epoch, completed_sources=completed)
                    checkpoint = Path(trainer.save_checkpoint(step)); last_saved = step
                    write_json(checkpoint / "scale_cursor.json", cursor)
                    if not reload_passed:
                        accelerator.load_state(str(checkpoint))
                        if trainer.lr_scheduler.state_dict()["last_epoch"] != step or not trainer.optimizer.state_dict()["state"]:
                            raise RuntimeError("First real full-scale checkpoint did not reload")
                        reload_passed = True
                        write_json(run / "first_checkpoint_verified.json", {"passed": True, "step": step, "checkpoint": str(checkpoint)})
                    publish(run)
                if needs_eval:
                    torch.cuda.synchronize(); torch.cuda.empty_cache()
                    final = last_group and epoch == args.epochs
                    record = evaluated(run, step, "final" if final else f"step_{step}", step, final=final)
                    candidate = record["validation"]["fixed_panel_comparison"]["aggregate"]
                    bad = validation_regressed(candidate, reference)
                    if reference is None:
                        reference = candidate
                    regressions = regressions + 1 if bad else 0
                    bg = candidate["background_only_false_motion_rate"]
                    if not bad and bg is not None and bg < best_bg:
                        dest = run / "best_model"; dest.mkdir(exist_ok=True)
                        for name in ("model.safetensors", "hfd_fused_delta_manifest.json"):
                            temp = dest / (name + ".tmp"); shutil.copy2(checkpoint / name, temp); temp.replace(dest / name)
                        write_json(dest / "selection.json", {"step": step, "validation_fixed_panel": candidate,
                            "policy": "lowest BG error passing first-trained-panel accuracy/miss/joint protection; weights only"})
                        best_bg = bg
                    if regressions >= 2:
                        decision = {"action": "stop_regression", "converged": False, "reason": "two fixed validation checks materially regressed"}
                        break
            if completed == len(selection["sessions"]):
                summary = acc.summary()
                if summary["action_samples"] != epoch_plan["action_samples"] or summary["control_samples"] != epoch_plan["control_samples"]:
                    raise RuntimeError("Complete epoch lost or duplicated supervision")
                append_record(run / "epoch_metrics.jsonl", {"epoch": epoch, "step": step, **summary})
            if decision["action"] == "stop_regression":
                break
        if not (run / "evaluations/final/validation/report.json").exists():
            evaluated(run, step, "final", step, final=True)
        write_json(run / "result.json", {"status": "complete", "step": step, "decision": decision,
            "train_count": len(selection["sessions"]), "validation_count": len(validation["sessions"]),
            "evaluation_file": str(run / "evaluations.json"), "final_checkpoint": "last, not necessarily best",
            "original_data_and_parent_weights_modified": False, "video_status": "see prediction_videos/status.json"})
        phase("completed", decision=decision)
    except Exception as error:
        phase("failed", error=repr(error)); raise
    finally:
        trainer.finalize()
        publish(run)
    with (run / "video_render.log").open("x") as log:
        child = subprocess.Popen([sys.executable, str(PROJECT / "scripts/infer/render_third_party_scale.py"),
            "--run-dir", str(run), "--manifest", str(args.manifest)], cwd=PROJECT,
            env={**os.environ, "CUDA_VISIBLE_DEVICES": "", "OMP_NUM_THREADS": "1", "MKL_NUM_THREADS": "1"},
            stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
    write_json(run / "video_render_job.json", {"pid": child.pid, "gpu_used": False, "inference_repeated": False})


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--manifest", type=Path, default=PROJECT / "data/hfd_third_party_multi_v5/training_scale_all_commands/selection.json")
    parser.add_argument("--run-dir", type=Path, required=True)
    parser.add_argument("--epochs", type=int, default=3)
    parser.add_argument("--resume-checkpoint", type=Path)
    parser.add_argument("--allow-host-change", action="store_true")
    mode = parser.add_mutually_exclusive_group(required=True)
    for name in ("check-only", "launch", "worker"):
        mode.add_argument(f"--{name}", action="store_true")
    args = parser.parse_args(); os.chdir(PROJECT)
    for key in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT",
        "ACCELERATE_USE_DEEPSPEED", "ACCELERATE_USE_FSDP", "ACCELERATE_USE_CPU", "SLURM_PROCID", "SLURM_LOCALID", "SLURM_NTASKS"):
        os.environ.pop(key, None)
    os.environ.update(HFD_SKIP_DOTENV="1", HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", PSI_SKIP_VLM_MODEL="0",
        PSI_FLOW_SIGMA_MIN="0.05", PSI_FLOW_SIGMA_MAX="1.0", PSI_WORKER_TORCH_THREADS="1",
        PSI_DATALOADER_PIN_MEMORY="1", PSI_FOREACH_GRAD_NORM="1", OMP_NUM_THREADS="4", PYTHONDONTWRITEBYTECODE="1")
    if args.worker:
        if os.environ.get("CUDA_VISIBLE_DEVICES") != "0" or os.environ.get("HFD_NPROC_PER_NODE") != "1":
            raise RuntimeError("Use --launch for GPU0 management")
        from scripts.third_party_temp import configure_project_temp
        configure_project_temp(PROJECT)
        for key, name in (("TORCHINDUCTOR_CACHE_DIR", "inductor"), ("TRITON_CACHE_DIR", "triton"), ("CUDA_CACHE_PATH", "cuda")):
            path = PROJECT / "cache/third_party_scale" / name; path.mkdir(parents=True, exist_ok=True); os.environ[key] = str(path)
    prepared = configure(args)
    if args.check_only:
        print(json.dumps({"passed": True, "gpu": 0, "train_count": len(prepared[1]["sessions"]),
            "validation_count": len(prepared[2]["sessions"]), "max_steps": prepared[3]["max_optimizer_updates"],
            "epochs": args.epochs, "initialization": prepared[4]["initialization_policy"],
            "parent": prepared[4]["parent"], "full_state_resume": bool(prepared[5])})); return
    if args.worker:
        try:
            worker(args, prepared)
        except Exception as error:
            state_path = args.run_dir / "workflow_state.json"
            if state_path.exists():
                state = read(state_path)
                write_json(state_path, {**state, "phase": "failed", "error": repr(error), "time": time.time()})
                publish(args.run_dir)
            raise
        return
    argv = [sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest),
        "--run-dir", str(args.run_dir), "--epochs", str(args.epochs)]
    if args.resume_checkpoint:
        argv += ["--resume-checkpoint", str(args.resume_checkpoint.resolve())]
    if args.allow_host_change:
        argv += ["--allow-host-change"]
    spec = {"schema": 1, "gpu": 0, "scope": "full_third_party", "project_root": str(PROJECT),
        "run_dir": str(args.run_dir), "manifest_path": str(args.manifest),
        "readiness_file": str(args.manifest.parent / "acceptance.json"),
        "argv": [*argv, "--worker"], "preflight_command": [*argv, "--check-only"]}
    path = args.manifest.parent / f"{args.run_dir.name}.launch.json"
    if path.exists():
        raise ValueError("Use a fresh launch specification")
    write_json(path, spec)
    raise SystemExit(subprocess.call([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(path)]))


if __name__ == "__main__":
    main()