#!/usr/bin/env python3 """Measured batch migration, historical train-only metrics, then continuation. The original model/loss/optimizer implementation is reused unchanged. The new driver changes source grouping and explicitly maps the remaining LR schedule. """ from __future__ import annotations import argparse import hashlib import json import math import os from pathlib import Path import subprocess import sys import time PROJECT = Path(__file__).resolve().parents[1] sys.path[:0] = [str(PROJECT), str(PROJECT / "src")] from scripts.third_party_strategy_plan import ( evaluation_ids, remaining_plan, mapped_scheduler_step, choose_batch, write_json, ) def setup_environment(): os.environ.update(HFD_SKIP_DOTENV="1", HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", PSI_SKIP_VLM_MODEL="0", PSI_FLOW_SIGMA_MIN="0.05", PSI_FLOW_SIGMA_MAX="1.0", PSI_WORKER_TORCH_THREADS="1", PSI_DATALOADER_PIN_MEMORY="1", PSI_FOREACH_GRAD_NORM="1", OMP_NUM_THREADS="4", OPENBLAS_NUM_THREADS="1") os.environ.pop("_ENV_INFO_JSON", None) for variable, name in (("TORCHINDUCTOR_CACHE_DIR", "inductor"), ("TRITON_CACHE_DIR", "triton"), ("CUDA_CACHE_PATH", "cuda")): directory = PROJECT / "cache" / "third_party_training" / name directory.mkdir(parents=True, exist_ok=True) os.environ[variable] = str(directory) from scripts.third_party_temp import configure_project_temp configure_project_temp(PROJECT) def prepare(args, output=None): from scripts.train_third_party import build_config, contained_output, PRODUCTION from scripts.third_party_resume import prepare_resume, frozen_resume_cursor from scripts.data.prepare_third_party_training import validate_acceptance from psi.trainers.posttrain_full_duplex import SourceLengthGroupedSampler manifest = args.manifest.resolve() run = contained_output(output or args.run_dir) validate_acceptance(manifest, PRODUCTION, deep_payload=False) cfg, identity = prepare_resume(build_config(manifest, run, "train"), args.resume_checkpoint, manifest=manifest, run_dir=run) selection = json.loads(manifest.read_text()) rows = selection["sessions"] class Lengths: def __len__(self): return len(rows) def estimated_stream_steps(self, i): return rows[i]["dense_steps"] sampler = SourceLengthGroupedSampler(Lengths(), world_size=1, batch_size=2, seed=cfg.seed) cursor = identity["source_cursor"] if cfg.train.train_batch_size != 2 or cursor["epoch"] != 2: raise ValueError("This switch requires an original batch-2 third-epoch checkpoint") plans = {str(b): remaining_plan(selection, cursor, sampler._ordered_indices(2), b) for b in (2, 4, 8, 16)} return cfg, identity, selection, plans def restore_trainer(args, output): import torch from accelerate.utils import set_seed from scripts.train import _initialize_accelerator from scripts.third_party_resume import code_identity from psi.trainers.posttrain_third_party import PosttrainThirdPartyTrainer, append_record from psi.trainers.posttrain_full_duplex import PosttrainFullDuplexTrainer cfg, identity, selection, plans = prepare(args, output) os.environ["HFD_CONTINUOUS_DATA_ROOT"] = cfg.data.continuous_audio_root os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = cfg.data.background_assignment_path os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str( Path(cfg.data.continuous_audio_root) / "reports/production_complete.json") output.mkdir(parents=True, exist_ok=True) implementation = ("scripts/train_third_party_strategy.py", "scripts/third_party_strategy_plan.py", "scripts/infer/third_party_metrics.py") write_json(output / "strategy_runtime_code_identity.json", { "time": time.time(), "files": {p: hashlib.sha256((PROJECT / p).read_bytes()).hexdigest() for p in implementation}}) write_json(output / "resume_identity.json", identity) write_json(output / "code_identity.json", code_identity()) write_json(output / "input_identity.json", {"manifest_sha256": identity["manifest_sha256"], "train_count": 300, "visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0}) class StrategyTrainer(PosttrainThirdPartyTrainer): def _checkpoint_probe(self, global_step): # Metrics are explicitly invoked by this driver after a complete # checkpoint; never run the inherited 12-video pause. return None set_seed(int(cfg.seed)) torch.cuda.set_device(0) trainer = StrategyTrainer(cfg, 0) trainer.init_models() accelerator = _initialize_accelerator(trainer) trainer.create_datasets() trainer.create_dataloaders(trainer.train_dataset, None) trainer.create_optimizer_and_scheduler(trainer.max_training_steps) trainer.prepare(accelerator) trainer.resume_from_checkpoint() assert trainer.global_step == identity["step"] return trainer, identity, selection, plans def source_batches(trainer, groups, workers=8): import torch from torch.utils.data import DataLoader from psi.trainers.posttrain_full_duplex import _source_collator from psi.trainers.trainer import worker_init_fn from psi.utils import batch_str_to_tensor flattened = [i for group in groups for i in group] loader = DataLoader(trainer.train_dataset, batch_size=1, sampler=flattened, num_workers=min(workers, len(flattened)), prefetch_factor=2, persistent_workers=False, pin_memory=True, collate_fn=_source_collator, worker_init_fn=worker_init_fn, generator=torch.Generator().manual_seed(26100601)) iterator = iter(loader) for group in groups: sources = [next(iterator)["sources"][0] for _ in group] yield batch_str_to_tensor({"sources": sources}) def configure_training_batch(trainer, size): trainer.train_cfg.train_batch_size = size trainer.train_cfg.target_effective_sources_per_gpu = size # Full-state restore must use the old contract; subsequent saves must use # the new batch. The base trainer caches this metadata at initialization. trainer._hfdv2_contract_cache = None trainer._hfdv2_checkpoint_contract() def benchmark(args): import torch trainer, identity, selection, plans = restore_trainer(args, args.run_dir) size = args.batch_size # Same long-source pool for all candidates. Benchmark weights are discarded. pending = [i for g in plans["2"]["source_indices"] for i in g] pending.sort(key=lambda i: selection["sessions"][i]["dense_steps"], reverse=True) group = pending[:size] configure_training_batch(trainer, size) trainer.set_train() batch = next(source_batches(trainer, [group])) torch.cuda.reset_peak_memory_stats() torch.cuda.synchronize() measured_start = None windows = 0 losses = [] started = time.monotonic() for i, (_, values, _) in enumerate(trainer.iter_optimizer_steps( batch, global_step_start=identity["step"], local_step=0, max_optimizer_events=240)): loss = float(values["loss"]) if not math.isfinite(loss): raise FloatingPointError("Non-finite benchmark loss") losses.append(loss) if i == 39: torch.cuda.synchronize() measured_start = time.monotonic() elif i >= 40: windows += size torch.cuda.synchronize() duration = time.monotonic() - measured_start result = {"status": "passed", "batch_size": size, "updates": len(losses), "measured_updates": len(losses) - 40, "active_windows_per_second": windows / duration, "measured_seconds": duration, "total_compute_seconds": time.monotonic() - started, "peak_allocated_gib": torch.cuda.max_memory_allocated() / 2**30, "peak_reserved_gib": torch.cuda.max_memory_reserved() / 2**30, "last_loss": losses[-1], "saved_weights": False} write_json(args.run_dir / "benchmark.json", result) print(json.dumps(result), flush=True) def continue_training(args): import torch from scripts.infer.third_party_metrics import evaluate_checkpoint from psi.trainers.posttrain_third_party import append_record from psi.utils import flatten from scripts.third_party_dashboard import update_dashboard trainer, identity, selection, plans = restore_trainer(args, args.run_dir) plan = plans[str(args.batch_size)] write_json(args.run_dir / "strategy_plan.json", plan) base = identity["step"] cfg = trainer.train_cfg configure_training_batch(trainer, args.batch_size) cfg.max_training_steps = plan["final_optimizer_step"] cfg.max_checkpoints_to_keep = 8 scheduler = getattr(trainer.lr_scheduler, "scheduler", trainer.lr_scheduler) old_lambdas = list(scheduler.lr_lambdas) scheduler.lr_lambdas = [ (lambda step, fn=fn: fn(mapped_scheduler_step( step, base, plan["remaining_updates"], plan["original_final_step"]))) for fn in old_lambdas] for group, fn in zip(trainer.optimizer.param_groups, scheduler.lr_lambdas): expected = group["initial_lr"] * fn(base) if not math.isclose(group["lr"], expected, rel_tol=1e-8, abs_tol=1e-12): raise RuntimeError("LR mapping would jump at the migration boundary") (args.run_dir / "run_config.json").write_text(trainer.cfg.model_dump_json(indent=2)) append_record(args.run_dir / "events.jsonl", {"event": "batch_migration_started", "time": time.time(), "step": base, "batch_size": args.batch_size, "remaining_sources": plan["source_count"], "remaining_updates": plan["remaining_updates"], "final_optimizer_step": plan["final_optimizer_step"], "original_final_step": plan["original_final_step"], "learning_rates": scheduler.get_last_lr()}) trainer.set_train() step = base completed_sources = [] halfway_saved = False for group_index, (indices, batch) in enumerate(zip(plan["source_indices"], source_batches(trainer, plan["source_indices"]))): # An incomplete final source group is real, smaller training—not a # dropped tail or padded duplicate. Checkpoints describe the max batch. cfg.train_batch_size = len(indices) cfg.target_effective_sources_per_gpu = len(indices) group_start = step boundary = False for _, values, boundary in trainer.iter_optimizer_steps( batch, global_step_start=step, local_step=group_index, max_optimizer_events=plan["group_steps"][group_index]): step += 1 trainer.global_step = step metrics = flatten({**values, "epoch": 2}, parent_key="train") metrics.update({"strategy/completed_sources": len(completed_sources), "strategy/target_sources": plan["source_count"], "strategy/source_batch_size": len(indices)}) trainer.log(metrics) if step % 100 == 0: print(f"[strategy_train] step={step}/{cfg.max_training_steps} loss={float(values['loss']):.5g} batch={len(indices)}", flush=True) if not boundary or step - group_start != plan["group_steps"][group_index]: raise RuntimeError("Source group ended outside the planned natural boundary") completed_sources.extend(indices) cfg.train_batch_size = args.batch_size cfg.target_effective_sources_per_gpu = args.batch_size state = {"step": step, "completed_group_count": group_index + 1, "completed_source_indices": completed_sources, "remaining_source_indices": [i for g in plan["source_indices"][group_index + 1:] for i in g], "parent_checkpoint": identity["checkpoint"], "plan": plan} write_json(args.run_dir / "strategy_state.json", state) final = group_index == len(plan["source_indices"]) - 1 halfway = not halfway_saved and len(completed_sources) >= plan["source_count"] / 2 if halfway or final: checkpoint = Path(trainer.save_checkpoint(step)) write_json(checkpoint / "strategy_state.json", state) append_record(args.run_dir / "events.jsonl", {"event": "metrics_evaluation_start", "step": step, "count": 300 if final else 60, "time": time.time()}) torch.cuda.empty_cache() result = evaluate_checkpoint(args.run_dir, step, args.manifest, args.run_dir / "metrics_evaluations" / f"step_{step}", workers=args.eval_workers, count=300 if final else 60, reuse=False) append_record(args.run_dir / "events.jsonl", {"event": "metrics_evaluation_complete", "step": step, "count": result["count"], "aggregate": result["aggregate"], "time": time.time()}) halfway_saved = True update_strategy_review(args.run_dir) if step != plan["final_optimizer_step"] or len(set(completed_sources)) != plan["source_count"]: raise RuntimeError("Remaining coverage incomplete") append_record(args.run_dir / "events.jsonl", {"event": "strategy_training_complete", "step": step, "remaining_sources_completed": len(completed_sources), "time": time.time()}) trainer.finalize() update_strategy_review(args.run_dir) def update_strategy_review(run): lines = ["# 并行策略训练与训练集复查", "", "仅使用固定 300 条训练数据;中途无视频渲染。", "", "| Checkpoint | 样例数 | 控制准确率 ↑ | 背景误触发率 ↓(逐条/逐帧) | 关节误差 ↓(弧度) | 指标 |", "|---|---:|---:|---:|---:|---|"] reports = list(run.glob("**/report.json")) origin = run / "strategy_origin.json" if origin.is_file(): reports.extend(Path(json.loads(origin.read_text())["previous_run"]).glob("history/*/report.json")) for path in sorted(reports): if "samples" in path.parts: continue result = json.loads(path.read_text()) if result.get("evaluation_split") != "train_only" or result.get("status") != "complete": continue if "inference_benchmark" in str(path): continue views = [(str(result["count"]), result["aggregate"])] if result["count"] == 300 and "fixed_sixty_comparison" in result: views.append(("其中固定 60", result["fixed_sixty_comparison"]["aggregate"])) for label, a in views: def percent(value): return "无有效时段" if value is None else f"{value:.1%}" bg = percent(a["background_only_false_motion_rate"]) weighted = percent(a["background_frame_weighted_false_motion_rate"]) lines.append(f"| {result['step']} | {label} | {a['control_accuracy']:.1%} | " f"{bg}/{weighted} | {a['joint_mae_radians']:.3f} | [报告]({os.path.relpath(path, run)}) |") lines.extend(["", "305990、459330 的权重已在本次工作前被轮换删除,不能补做;原 12 条报告保留。", "背景误触发仅统计有背景且 GT 不要求动作的时段;分母见报告。控制准确率提高不等于背景误触发已降低。", "最终 300 条同时单列相同随机种子的固定 60 条;不同样本数的整体均值不直接比较。", "数值与视频分离;全量指标不代表测试集泛化。改变 batch 后更新步数不与旧步数等价,覆盖以 strategy_plan.json 为准。"]) (run / "strategy_review.md").write_text("\n".join(lines) + "\n") def verified_prior_strategy(args, identity, selection, plans): prior = args.strategy_from.resolve() if not prior.is_relative_to(PROJECT / "runs") or prior == args.run_dir: raise ValueError("Use a distinct existing local strategy run") status = json.loads((prior / "run_status.json").read_text()) if status["phase"] not in ("failed", "completed") or status.get("keeper_restore_verified") is not True: raise ValueError("Previous strategy manager has not safely exited") chosen = json.loads((prior / "chosen_strategy.json").read_text()) measured = choose_batch(json.loads((prior / "training_benchmarks.json").read_text())) if (chosen["training"] != measured or chosen["eval_workers"] not in (1, 2, 4) or chosen["plan"] != plans[str(measured["batch_size"])]): raise ValueError("Measured strategy differs from this continuation plan") for step in (153530, 611980, identity["step"]): directory = prior / "history" / f"step_{step}" report = json.loads((directory / "report.json").read_text()) previous_identity = json.loads((directory / "identity.json").read_text()) if (report["status"] != "complete" or report["count"] != 60 or report["evaluation_split"] != "train_only" or [row["session_id"] for row in report["samples"]] != evaluation_ids(selection) or previous_identity["manifest_sha256"] != identity["manifest_sha256"]): raise ValueError("Historical fixed-sixty evaluation incomplete or incompatible") return chosen def run_continuation(args, plans, selected, eval_workers): command = [sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest), "--run-dir", str(args.run_dir), "--resume-checkpoint", str(args.resume_checkpoint), "--continue-training", "--batch-size", str(selected["batch_size"]), "--eval-workers", str(eval_workers)] write_json(args.run_dir / "workflow_state.json", {"phase": "continuing_training", "batch_size": selected["batch_size"], "remaining_sources": plans[str(selected["batch_size"])]["source_count"], "time": time.time()}) subprocess.run(command, check=True) write_json(args.run_dir / "workflow_state.json", {"phase": "completed", "time": time.time()}) def workflow(args): from scripts.infer.third_party_metrics import evaluate_checkpoint cfg, identity, selection, plans = prepare(args) write_json(args.run_dir / "evaluation_selection.json", {"ids": evaluation_ids(selection), "training_manifest": str(args.manifest), "seed": 26100601, "split": "train_only"}) if args.strategy_from: chosen = verified_prior_strategy(args, identity, selection, plans) for name in ("chosen_strategy.json", "training_benchmarks.json", "inference_benchmarks.json"): write_json(args.run_dir / name, json.loads((args.strategy_from / name).read_text())) write_json(args.run_dir / "strategy_origin.json", {"previous_run": str(args.strategy_from), "reuse": "completed benchmarks and three fixed-sixty evaluations; no benchmark weights", "time": time.time()}) update_strategy_review(args.run_dir) run_continuation(args, plans, chosen["training"], chosen["eval_workers"]) return snapshots = [(Path(identity["source_run"]).parent / "third_party_train_20261004_183932", 153530), (Path(identity["source_run"]), 611980), (Path(identity["source_run"]), 770084)] # Short real timing sweep on the same missing sessions, independent outputs. eval_ids = evaluation_ids(selection)[12:16] inference_results = [] for workers in (1, 2, 4): write_json(args.run_dir / "workflow_state.json", {"phase": "benchmark_inference", "workers": workers, "time": time.time()}) result = evaluate_checkpoint(snapshots[-1][0], 770084, args.manifest, args.run_dir / "inference_benchmark" / str(workers), workers=workers, subset=eval_ids, reuse=False) inference_results.append({"workers": workers, "seconds": result["elapsed_seconds"]}) write_json(args.run_dir / "inference_benchmarks.json", inference_results) fastest = min(r["seconds"] for r in inference_results) eval_workers = min(r["workers"] for r in inference_results if r["seconds"] <= fastest * 1.05) training_results = [] for batch in (2, 4, 8, 16): write_json(args.run_dir / "workflow_state.json", {"phase": "benchmark_training", "batch_size": batch, "time": time.time()}) directory = args.run_dir / "benchmarks" / f"batch_{batch}" directory.mkdir(parents=True) with (directory / "train.log").open("w") as log: result = subprocess.run([sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest), "--run-dir", str(directory), "--resume-checkpoint", str(args.resume_checkpoint), "--benchmark", "--batch-size", str(batch)], stdout=log, stderr=subprocess.STDOUT, timeout=1800) if result.returncode: training_results.append({"batch_size": batch, "status": "failed", "returncode": result.returncode}) else: training_results.append(json.loads((directory / "benchmark.json").read_text())) write_json(args.run_dir / "training_benchmarks.json", training_results) selected = choose_batch(training_results) write_json(args.run_dir / "chosen_strategy.json", {"training": selected, "eval_workers": eval_workers, "plan": plans[str(selected["batch_size"])]}) for source, step in snapshots: write_json(args.run_dir / "workflow_state.json", {"phase": "historical_metrics", "checkpoint_step": step, "count": 60, "workers": eval_workers, "time": time.time()}) evaluate_checkpoint(source, step, args.manifest, args.run_dir / "history" / f"step_{step}", workers=eval_workers, reuse=True) update_strategy_review(args.run_dir) # Fresh full-state restoration discards every benchmark parameter update. run_continuation(args, plans, selected, eval_workers) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--run-dir", type=Path, required=True) parser.add_argument("--resume-checkpoint", type=Path, required=True) mode = parser.add_mutually_exclusive_group(required=True) for name in ("check-only", "launch", "worker", "benchmark", "continue-training"): mode.add_argument(f"--{name}", action="store_true") parser.add_argument("--batch-size", type=int, choices=(2, 4, 8, 16), default=2) parser.add_argument("--eval-workers", type=int, choices=(1, 2, 4), default=2) parser.add_argument("--strategy-from", type=Path, help="Reuse verified completed benchmarks and historical metrics after a safe stop") args = parser.parse_args() args.manifest, args.run_dir, args.resume_checkpoint = ( args.manifest.resolve(), args.run_dir.resolve(), args.resume_checkpoint.resolve()) setup_environment() _, identity, selection, plans = prepare(args) if args.strategy_from: args.strategy_from = args.strategy_from.resolve() verified_prior_strategy(args, identity, selection, plans) if args.check_only: print(json.dumps({"passed": True, "parent_step": identity["step"], "remaining_sources": plans['2']['source_count'], "planned_eval_count": len(evaluation_ids(selection)), "candidate_remaining_updates": {k:v['remaining_updates'] for k,v in plans.items()}})) return if args.launch: shared = [str(Path(__file__).resolve()), "--manifest", str(args.manifest), "--run-dir", str(args.run_dir), "--resume-checkpoint", str(args.resume_checkpoint)] if args.strategy_from: shared.extend(["--strategy-from", str(args.strategy_from)]) spec = {"schema": 1, "project_root": str(PROJECT), "run_dir": str(args.run_dir), "gpu": 3, "manifest_path": str(args.manifest), "readiness_file": str(args.manifest.parent / "acceptance.json"), "argv": [sys.executable, *shared, "--worker"], "preflight_command": [sys.executable, *shared, "--check-only"]} spec_path = args.manifest.parent / f"{args.run_dir.name}.launch.json" if spec_path.exists(): raise FileExistsError(spec_path) write_json(spec_path, spec) subprocess.run([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(spec_path)], check=True) return if os.environ.get("CUDA_VISIBLE_DEVICES") != "3" or os.environ.get("HFD_NPROC_PER_NODE") != "1": raise RuntimeError("Only use --launch for GPU work") args.run_dir.mkdir(parents=True, exist_ok=True) if args.benchmark: benchmark(args) elif args.continue_training: continue_training(args) else: paths = ("scripts/train_third_party_strategy.py", "scripts/third_party_strategy_plan.py", "scripts/infer/third_party_metrics.py") write_json(args.run_dir / "strategy_code_identity.json", {p: hashlib.sha256((PROJECT/p).read_bytes()).hexdigest() for p in paths}) workflow(args) if __name__ == "__main__": main()