File size: 25,424 Bytes
55e0599
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
#!/usr/bin/env python3
"""Measured batch migration, historical train-only metrics, then continuation.

The original model/loss/optimizer implementation is reused unchanged. The new
driver changes source grouping and explicitly maps the remaining LR schedule.
"""
from __future__ import annotations

import argparse
import hashlib
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import time

PROJECT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
from scripts.third_party_strategy_plan import (
    evaluation_ids, remaining_plan, mapped_scheduler_step, choose_batch, write_json,
)


def setup_environment():
    os.environ.update(HFD_SKIP_DOTENV="1", HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1",
                      PSI_SKIP_VLM_MODEL="0", PSI_FLOW_SIGMA_MIN="0.05", PSI_FLOW_SIGMA_MAX="1.0",
                      PSI_WORKER_TORCH_THREADS="1", PSI_DATALOADER_PIN_MEMORY="1",
                      PSI_FOREACH_GRAD_NORM="1", OMP_NUM_THREADS="4", OPENBLAS_NUM_THREADS="1")
    os.environ.pop("_ENV_INFO_JSON", None)
    for variable, name in (("TORCHINDUCTOR_CACHE_DIR", "inductor"),
                           ("TRITON_CACHE_DIR", "triton"), ("CUDA_CACHE_PATH", "cuda")):
        directory = PROJECT / "cache" / "third_party_training" / name
        directory.mkdir(parents=True, exist_ok=True)
        os.environ[variable] = str(directory)
    from scripts.third_party_temp import configure_project_temp
    configure_project_temp(PROJECT)


def prepare(args, output=None):
    from scripts.train_third_party import build_config, contained_output, PRODUCTION
    from scripts.third_party_resume import prepare_resume, frozen_resume_cursor
    from scripts.data.prepare_third_party_training import validate_acceptance
    from psi.trainers.posttrain_full_duplex import SourceLengthGroupedSampler
    manifest = args.manifest.resolve()
    run = contained_output(output or args.run_dir)
    validate_acceptance(manifest, PRODUCTION, deep_payload=False)
    cfg, identity = prepare_resume(build_config(manifest, run, "train"), args.resume_checkpoint,
                                   manifest=manifest, run_dir=run)
    selection = json.loads(manifest.read_text())
    rows = selection["sessions"]

    class Lengths:
        def __len__(self):
            return len(rows)

        def estimated_stream_steps(self, i):
            return rows[i]["dense_steps"]

    sampler = SourceLengthGroupedSampler(Lengths(), world_size=1, batch_size=2, seed=cfg.seed)
    cursor = identity["source_cursor"]
    if cfg.train.train_batch_size != 2 or cursor["epoch"] != 2:
        raise ValueError("This switch requires an original batch-2 third-epoch checkpoint")
    plans = {str(b): remaining_plan(selection, cursor, sampler._ordered_indices(2), b)
             for b in (2, 4, 8, 16)}
    return cfg, identity, selection, plans


def restore_trainer(args, output):
    import torch
    from accelerate.utils import set_seed
    from scripts.train import _initialize_accelerator
    from scripts.third_party_resume import code_identity
    from psi.trainers.posttrain_third_party import PosttrainThirdPartyTrainer, append_record
    from psi.trainers.posttrain_full_duplex import PosttrainFullDuplexTrainer

    cfg, identity, selection, plans = prepare(args, output)
    os.environ["HFD_CONTINUOUS_DATA_ROOT"] = cfg.data.continuous_audio_root
    os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = cfg.data.background_assignment_path
    os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str(
        Path(cfg.data.continuous_audio_root) / "reports/production_complete.json")
    output.mkdir(parents=True, exist_ok=True)
    implementation = ("scripts/train_third_party_strategy.py", "scripts/third_party_strategy_plan.py",
                      "scripts/infer/third_party_metrics.py")
    write_json(output / "strategy_runtime_code_identity.json", {
        "time": time.time(), "files": {p: hashlib.sha256((PROJECT / p).read_bytes()).hexdigest() for p in implementation}})
    write_json(output / "resume_identity.json", identity)
    write_json(output / "code_identity.json", code_identity())
    write_json(output / "input_identity.json", {"manifest_sha256": identity["manifest_sha256"],
                "train_count": 300, "visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0})

    class StrategyTrainer(PosttrainThirdPartyTrainer):
        def _checkpoint_probe(self, global_step):
            # Metrics are explicitly invoked by this driver after a complete
            # checkpoint; never run the inherited 12-video pause.
            return None

    set_seed(int(cfg.seed))
    torch.cuda.set_device(0)
    trainer = StrategyTrainer(cfg, 0)
    trainer.init_models()
    accelerator = _initialize_accelerator(trainer)
    trainer.create_datasets()
    trainer.create_dataloaders(trainer.train_dataset, None)
    trainer.create_optimizer_and_scheduler(trainer.max_training_steps)
    trainer.prepare(accelerator)
    trainer.resume_from_checkpoint()
    assert trainer.global_step == identity["step"]
    return trainer, identity, selection, plans


def source_batches(trainer, groups, workers=8):
    import torch
    from torch.utils.data import DataLoader
    from psi.trainers.posttrain_full_duplex import _source_collator
    from psi.trainers.trainer import worker_init_fn
    from psi.utils import batch_str_to_tensor
    flattened = [i for group in groups for i in group]
    loader = DataLoader(trainer.train_dataset, batch_size=1, sampler=flattened,
                        num_workers=min(workers, len(flattened)), prefetch_factor=2, persistent_workers=False,
                        pin_memory=True, collate_fn=_source_collator, worker_init_fn=worker_init_fn,
                        generator=torch.Generator().manual_seed(26100601))
    iterator = iter(loader)
    for group in groups:
        sources = [next(iterator)["sources"][0] for _ in group]
        yield batch_str_to_tensor({"sources": sources})


def configure_training_batch(trainer, size):
    trainer.train_cfg.train_batch_size = size
    trainer.train_cfg.target_effective_sources_per_gpu = size
    # Full-state restore must use the old contract; subsequent saves must use
    # the new batch. The base trainer caches this metadata at initialization.
    trainer._hfdv2_contract_cache = None
    trainer._hfdv2_checkpoint_contract()


def benchmark(args):
    import torch
    trainer, identity, selection, plans = restore_trainer(args, args.run_dir)
    size = args.batch_size
    # Same long-source pool for all candidates. Benchmark weights are discarded.
    pending = [i for g in plans["2"]["source_indices"] for i in g]
    pending.sort(key=lambda i: selection["sessions"][i]["dense_steps"], reverse=True)
    group = pending[:size]
    configure_training_batch(trainer, size)
    trainer.set_train()
    batch = next(source_batches(trainer, [group]))
    torch.cuda.reset_peak_memory_stats()
    torch.cuda.synchronize()
    measured_start = None
    windows = 0
    losses = []
    started = time.monotonic()
    for i, (_, values, _) in enumerate(trainer.iter_optimizer_steps(
            batch, global_step_start=identity["step"], local_step=0, max_optimizer_events=240)):
        loss = float(values["loss"])
        if not math.isfinite(loss):
            raise FloatingPointError("Non-finite benchmark loss")
        losses.append(loss)
        if i == 39:
            torch.cuda.synchronize()
            measured_start = time.monotonic()
        elif i >= 40:
            windows += size
    torch.cuda.synchronize()
    duration = time.monotonic() - measured_start
    result = {"status": "passed", "batch_size": size, "updates": len(losses),
              "measured_updates": len(losses) - 40, "active_windows_per_second": windows / duration,
              "measured_seconds": duration, "total_compute_seconds": time.monotonic() - started,
              "peak_allocated_gib": torch.cuda.max_memory_allocated() / 2**30,
              "peak_reserved_gib": torch.cuda.max_memory_reserved() / 2**30,
              "last_loss": losses[-1], "saved_weights": False}
    write_json(args.run_dir / "benchmark.json", result)
    print(json.dumps(result), flush=True)


def continue_training(args):
    import torch
    from scripts.infer.third_party_metrics import evaluate_checkpoint
    from psi.trainers.posttrain_third_party import append_record
    from psi.utils import flatten
    from scripts.third_party_dashboard import update_dashboard
    trainer, identity, selection, plans = restore_trainer(args, args.run_dir)
    plan = plans[str(args.batch_size)]
    write_json(args.run_dir / "strategy_plan.json", plan)
    base = identity["step"]
    cfg = trainer.train_cfg
    configure_training_batch(trainer, args.batch_size)
    cfg.max_training_steps = plan["final_optimizer_step"]
    cfg.max_checkpoints_to_keep = 8
    scheduler = getattr(trainer.lr_scheduler, "scheduler", trainer.lr_scheduler)
    old_lambdas = list(scheduler.lr_lambdas)
    scheduler.lr_lambdas = [
        (lambda step, fn=fn: fn(mapped_scheduler_step(
            step, base, plan["remaining_updates"], plan["original_final_step"])))
        for fn in old_lambdas]
    for group, fn in zip(trainer.optimizer.param_groups, scheduler.lr_lambdas):
        expected = group["initial_lr"] * fn(base)
        if not math.isclose(group["lr"], expected, rel_tol=1e-8, abs_tol=1e-12):
            raise RuntimeError("LR mapping would jump at the migration boundary")
    (args.run_dir / "run_config.json").write_text(trainer.cfg.model_dump_json(indent=2))
    append_record(args.run_dir / "events.jsonl", {"event": "batch_migration_started", "time": time.time(),
        "step": base, "batch_size": args.batch_size, "remaining_sources": plan["source_count"],
        "remaining_updates": plan["remaining_updates"], "final_optimizer_step": plan["final_optimizer_step"],
        "original_final_step": plan["original_final_step"], "learning_rates": scheduler.get_last_lr()})
    trainer.set_train()
    step = base
    completed_sources = []
    halfway_saved = False
    for group_index, (indices, batch) in enumerate(zip(plan["source_indices"], source_batches(trainer, plan["source_indices"]))):
        # An incomplete final source group is real, smaller training—not a
        # dropped tail or padded duplicate. Checkpoints describe the max batch.
        cfg.train_batch_size = len(indices)
        cfg.target_effective_sources_per_gpu = len(indices)
        group_start = step
        boundary = False
        for _, values, boundary in trainer.iter_optimizer_steps(
                batch, global_step_start=step, local_step=group_index,
                max_optimizer_events=plan["group_steps"][group_index]):
            step += 1
            trainer.global_step = step
            metrics = flatten({**values, "epoch": 2}, parent_key="train")
            metrics.update({"strategy/completed_sources": len(completed_sources),
                            "strategy/target_sources": plan["source_count"],
                            "strategy/source_batch_size": len(indices)})
            trainer.log(metrics)
            if step % 100 == 0:
                print(f"[strategy_train] step={step}/{cfg.max_training_steps} loss={float(values['loss']):.5g} batch={len(indices)}", flush=True)
        if not boundary or step - group_start != plan["group_steps"][group_index]:
            raise RuntimeError("Source group ended outside the planned natural boundary")
        completed_sources.extend(indices)
        cfg.train_batch_size = args.batch_size
        cfg.target_effective_sources_per_gpu = args.batch_size
        state = {"step": step, "completed_group_count": group_index + 1,
                 "completed_source_indices": completed_sources,
                 "remaining_source_indices": [i for g in plan["source_indices"][group_index + 1:] for i in g],
                 "parent_checkpoint": identity["checkpoint"], "plan": plan}
        write_json(args.run_dir / "strategy_state.json", state)
        final = group_index == len(plan["source_indices"]) - 1
        halfway = not halfway_saved and len(completed_sources) >= plan["source_count"] / 2
        if halfway or final:
            checkpoint = Path(trainer.save_checkpoint(step))
            write_json(checkpoint / "strategy_state.json", state)
            append_record(args.run_dir / "events.jsonl", {"event": "metrics_evaluation_start", "step": step,
                                                          "count": 300 if final else 60, "time": time.time()})
            torch.cuda.empty_cache()
            result = evaluate_checkpoint(args.run_dir, step, args.manifest,
                        args.run_dir / "metrics_evaluations" / f"step_{step}",
                        workers=args.eval_workers, count=300 if final else 60, reuse=False)
            append_record(args.run_dir / "events.jsonl", {"event": "metrics_evaluation_complete", "step": step,
                          "count": result["count"], "aggregate": result["aggregate"], "time": time.time()})
            halfway_saved = True
            update_strategy_review(args.run_dir)
    if step != plan["final_optimizer_step"] or len(set(completed_sources)) != plan["source_count"]:
        raise RuntimeError("Remaining coverage incomplete")
    append_record(args.run_dir / "events.jsonl", {"event": "strategy_training_complete", "step": step,
                                                "remaining_sources_completed": len(completed_sources), "time": time.time()})
    trainer.finalize()
    update_strategy_review(args.run_dir)


def update_strategy_review(run):
    lines = ["# 并行策略训练与训练集复查", "", "仅使用固定 300 条训练数据;中途无视频渲染。", "",
             "| Checkpoint | 样例数 | 控制准确率 ↑ | 背景误触发率 ↓(逐条/逐帧) | 关节误差 ↓(弧度) | 指标 |",
             "|---|---:|---:|---:|---:|---|"]
    reports = list(run.glob("**/report.json"))
    origin = run / "strategy_origin.json"
    if origin.is_file():
        reports.extend(Path(json.loads(origin.read_text())["previous_run"]).glob("history/*/report.json"))
    for path in sorted(reports):
        if "samples" in path.parts:
            continue
        result = json.loads(path.read_text())
        if result.get("evaluation_split") != "train_only" or result.get("status") != "complete":
            continue
        if "inference_benchmark" in str(path):
            continue
        views = [(str(result["count"]), result["aggregate"])]
        if result["count"] == 300 and "fixed_sixty_comparison" in result:
            views.append(("其中固定 60", result["fixed_sixty_comparison"]["aggregate"]))
        for label, a in views:
            def percent(value):
                return "无有效时段" if value is None else f"{value:.1%}"
            bg = percent(a["background_only_false_motion_rate"])
            weighted = percent(a["background_frame_weighted_false_motion_rate"])
            lines.append(f"| {result['step']} | {label} | {a['control_accuracy']:.1%} | "
                         f"{bg}/{weighted} | {a['joint_mae_radians']:.3f} | [报告]({os.path.relpath(path, run)}) |")
    lines.extend(["", "305990、459330 的权重已在本次工作前被轮换删除,不能补做;原 12 条报告保留。",
                  "背景误触发仅统计有背景且 GT 不要求动作的时段;分母见报告。控制准确率提高不等于背景误触发已降低。",
                  "最终 300 条同时单列相同随机种子的固定 60 条;不同样本数的整体均值不直接比较。",
                  "数值与视频分离;全量指标不代表测试集泛化。改变 batch 后更新步数不与旧步数等价,覆盖以 strategy_plan.json 为准。"])
    (run / "strategy_review.md").write_text("\n".join(lines) + "\n")


def verified_prior_strategy(args, identity, selection, plans):
    prior = args.strategy_from.resolve()
    if not prior.is_relative_to(PROJECT / "runs") or prior == args.run_dir:
        raise ValueError("Use a distinct existing local strategy run")
    status = json.loads((prior / "run_status.json").read_text())
    if status["phase"] not in ("failed", "completed") or status.get("keeper_restore_verified") is not True:
        raise ValueError("Previous strategy manager has not safely exited")
    chosen = json.loads((prior / "chosen_strategy.json").read_text())
    measured = choose_batch(json.loads((prior / "training_benchmarks.json").read_text()))
    if (chosen["training"] != measured or chosen["eval_workers"] not in (1, 2, 4)
            or chosen["plan"] != plans[str(measured["batch_size"])]):
        raise ValueError("Measured strategy differs from this continuation plan")
    for step in (153530, 611980, identity["step"]):
        directory = prior / "history" / f"step_{step}"
        report = json.loads((directory / "report.json").read_text())
        previous_identity = json.loads((directory / "identity.json").read_text())
        if (report["status"] != "complete" or report["count"] != 60
                or report["evaluation_split"] != "train_only"
                or [row["session_id"] for row in report["samples"]] != evaluation_ids(selection)
                or previous_identity["manifest_sha256"] != identity["manifest_sha256"]):
            raise ValueError("Historical fixed-sixty evaluation incomplete or incompatible")
    return chosen


def run_continuation(args, plans, selected, eval_workers):
    command = [sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest),
               "--run-dir", str(args.run_dir), "--resume-checkpoint", str(args.resume_checkpoint),
               "--continue-training", "--batch-size", str(selected["batch_size"]),
               "--eval-workers", str(eval_workers)]
    write_json(args.run_dir / "workflow_state.json", {"phase": "continuing_training", "batch_size": selected["batch_size"],
               "remaining_sources": plans[str(selected["batch_size"])]["source_count"], "time": time.time()})
    subprocess.run(command, check=True)
    write_json(args.run_dir / "workflow_state.json", {"phase": "completed", "time": time.time()})


def workflow(args):
    from scripts.infer.third_party_metrics import evaluate_checkpoint
    cfg, identity, selection, plans = prepare(args)
    write_json(args.run_dir / "evaluation_selection.json", {"ids": evaluation_ids(selection),
               "training_manifest": str(args.manifest), "seed": 26100601, "split": "train_only"})
    if args.strategy_from:
        chosen = verified_prior_strategy(args, identity, selection, plans)
        for name in ("chosen_strategy.json", "training_benchmarks.json", "inference_benchmarks.json"):
            write_json(args.run_dir / name, json.loads((args.strategy_from / name).read_text()))
        write_json(args.run_dir / "strategy_origin.json", {"previous_run": str(args.strategy_from),
                   "reuse": "completed benchmarks and three fixed-sixty evaluations; no benchmark weights", "time": time.time()})
        update_strategy_review(args.run_dir)
        run_continuation(args, plans, chosen["training"], chosen["eval_workers"])
        return
    snapshots = [(Path(identity["source_run"]).parent / "third_party_train_20261004_183932", 153530),
                 (Path(identity["source_run"]), 611980), (Path(identity["source_run"]), 770084)]
    # Short real timing sweep on the same missing sessions, independent outputs.
    eval_ids = evaluation_ids(selection)[12:16]
    inference_results = []
    for workers in (1, 2, 4):
        write_json(args.run_dir / "workflow_state.json", {"phase": "benchmark_inference", "workers": workers, "time": time.time()})
        result = evaluate_checkpoint(snapshots[-1][0], 770084, args.manifest,
                    args.run_dir / "inference_benchmark" / str(workers), workers=workers,
                    subset=eval_ids, reuse=False)
        inference_results.append({"workers": workers, "seconds": result["elapsed_seconds"]})
        write_json(args.run_dir / "inference_benchmarks.json", inference_results)
    fastest = min(r["seconds"] for r in inference_results)
    eval_workers = min(r["workers"] for r in inference_results if r["seconds"] <= fastest * 1.05)
    training_results = []
    for batch in (2, 4, 8, 16):
        write_json(args.run_dir / "workflow_state.json", {"phase": "benchmark_training", "batch_size": batch, "time": time.time()})
        directory = args.run_dir / "benchmarks" / f"batch_{batch}"
        directory.mkdir(parents=True)
        with (directory / "train.log").open("w") as log:
            result = subprocess.run([sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest),
                "--run-dir", str(directory), "--resume-checkpoint", str(args.resume_checkpoint),
                "--benchmark", "--batch-size", str(batch)], stdout=log, stderr=subprocess.STDOUT, timeout=1800)
        if result.returncode:
            training_results.append({"batch_size": batch, "status": "failed", "returncode": result.returncode})
        else:
            training_results.append(json.loads((directory / "benchmark.json").read_text()))
        write_json(args.run_dir / "training_benchmarks.json", training_results)
    selected = choose_batch(training_results)
    write_json(args.run_dir / "chosen_strategy.json", {"training": selected, "eval_workers": eval_workers,
                                                      "plan": plans[str(selected["batch_size"])]})
    for source, step in snapshots:
        write_json(args.run_dir / "workflow_state.json", {"phase": "historical_metrics", "checkpoint_step": step,
                   "count": 60, "workers": eval_workers, "time": time.time()})
        evaluate_checkpoint(source, step, args.manifest, args.run_dir / "history" / f"step_{step}",
                            workers=eval_workers, reuse=True)
        update_strategy_review(args.run_dir)
    # Fresh full-state restoration discards every benchmark parameter update.
    run_continuation(args, plans, selected, eval_workers)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--manifest", type=Path, required=True)
    parser.add_argument("--run-dir", type=Path, required=True)
    parser.add_argument("--resume-checkpoint", type=Path, required=True)
    mode = parser.add_mutually_exclusive_group(required=True)
    for name in ("check-only", "launch", "worker", "benchmark", "continue-training"):
        mode.add_argument(f"--{name}", action="store_true")
    parser.add_argument("--batch-size", type=int, choices=(2, 4, 8, 16), default=2)
    parser.add_argument("--eval-workers", type=int, choices=(1, 2, 4), default=2)
    parser.add_argument("--strategy-from", type=Path,
                        help="Reuse verified completed benchmarks and historical metrics after a safe stop")
    args = parser.parse_args()
    args.manifest, args.run_dir, args.resume_checkpoint = (
        args.manifest.resolve(), args.run_dir.resolve(), args.resume_checkpoint.resolve())
    setup_environment()
    _, identity, selection, plans = prepare(args)
    if args.strategy_from:
        args.strategy_from = args.strategy_from.resolve()
        verified_prior_strategy(args, identity, selection, plans)
    if args.check_only:
        print(json.dumps({"passed": True, "parent_step": identity["step"],
              "remaining_sources": plans['2']['source_count'], "planned_eval_count": len(evaluation_ids(selection)),
              "candidate_remaining_updates": {k:v['remaining_updates'] for k,v in plans.items()}}))
        return
    if args.launch:
        shared = [str(Path(__file__).resolve()), "--manifest", str(args.manifest), "--run-dir", str(args.run_dir),
                  "--resume-checkpoint", str(args.resume_checkpoint)]
        if args.strategy_from:
            shared.extend(["--strategy-from", str(args.strategy_from)])
        spec = {"schema": 1, "project_root": str(PROJECT), "run_dir": str(args.run_dir), "gpu": 3,
                "manifest_path": str(args.manifest), "readiness_file": str(args.manifest.parent / "acceptance.json"),
                "argv": [sys.executable, *shared, "--worker"],
                "preflight_command": [sys.executable, *shared, "--check-only"]}
        spec_path = args.manifest.parent / f"{args.run_dir.name}.launch.json"
        if spec_path.exists():
            raise FileExistsError(spec_path)
        write_json(spec_path, spec)
        subprocess.run([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(spec_path)], check=True)
        return
    if os.environ.get("CUDA_VISIBLE_DEVICES") != "3" or os.environ.get("HFD_NPROC_PER_NODE") != "1":
        raise RuntimeError("Only use --launch for GPU work")
    args.run_dir.mkdir(parents=True, exist_ok=True)
    if args.benchmark:
        benchmark(args)
    elif args.continue_training:
        continue_training(args)
    else:
        paths = ("scripts/train_third_party_strategy.py", "scripts/third_party_strategy_plan.py", "scripts/infer/third_party_metrics.py")
        write_json(args.run_dir / "strategy_code_identity.json", {p: hashlib.sha256((PROJECT/p).read_bytes()).hexdigest() for p in paths})
        workflow(args)


if __name__ == "__main__":
    main()