#!/usr/bin/env python3 """Bounded, coverage-balanced joint fitting of the existing train300. Fresh optimizer/schedule; immutable trained backbone/action + verified head. No heldout inputs, no extra tasks, no random cuts through recurrent histories. """ from __future__ import annotations import argparse import hashlib import json import math import os from pathlib import Path import shutil import subprocess import sys import time from types import SimpleNamespace PROJECT = Path(__file__).resolve().parents[1] sys.path[:0] = [str(PROJECT), str(PROJECT / "src")] from scripts.third_party_strategy_plan import evaluation_ids, write_json def read(path): return json.loads(Path(path).read_text()) def configure(args): from scripts.train_third_party import build_config, contained_output, PRODUCTION from scripts.data.prepare_third_party_training import validate_acceptance from scripts.third_party_convergence_plan import plan_epochs from scripts.third_party_joint_init import prepare_joint_initialization args.run_dir = contained_output(args.run_dir) args.manifest = args.manifest.resolve() validate_acceptance(args.manifest, PRODUCTION, deep_payload=False) selection = read(args.manifest) plan = plan_epochs(selection, batch_size=16, seed=26100701, min_epochs=6, max_epochs=20) cfg = build_config(args.manifest, args.run_dir, "train") cfg.train.train_batch_size = 16 cfg.train.target_effective_sources_per_gpu = 16 cfg.train.coverage_epochs = 20 cfg.train.max_training_steps = plan["max_optimizer_updates"] cfg.train.learning_rate = 2e-5 cfg.model.lang_backbone_lr = 5e-7 cfg.train.warmup_steps = 500 cfg.train.cosine_min_lr_rate = .1 cfg.train.max_checkpoints_to_keep = 3 cfg.train.num_workers = 8 # Revalidate all existing model/data/safety contracts after overrides. cfg = type(cfg).model_validate(cfg.model_dump()) identity = prepare_joint_initialization(cfg, base_checkpoint=args.base_checkpoint, adapter=args.adapter, verification_run=args.verification_run, manifest=args.manifest, run_dir=args.run_dir) verified = read(args.verification_run / "result.json") panel = read(args.adapter.parent.parent / "selection.json")["paired_twentyfour_ids"] if len(panel) != 24 or len(set(panel)) != 24 or not set(panel) <= set(evaluation_ids(selection)): raise ValueError("Clean diagnostics must be the original paired subset of fixed train60") return cfg, identity, selection, plan, panel, verified def build_launch_spec(args): command = [sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest), "--run-dir", str(args.run_dir), "--base-checkpoint", str(args.base_checkpoint), "--adapter", str(args.adapter), "--verification-run", str(args.verification_run)] return {"schema": 1, "project_root": str(PROJECT), "run_dir": str(args.run_dir), "gpu": 3, "manifest_path": str(args.manifest), "readiness_file": str(args.manifest.parent / "acceptance.json"), "argv": [*command, "--worker"], "preflight_command": [*command, "--check-only"]} def clean_safe(candidate, baseline): checks = {"accuracy": candidate["control_accuracy"] >= baseline["control_accuracy"] - .02, "missed_motion": candidate["missed_motion_rate"] <= baseline["missed_motion_rate"] + .02, "joint_mae": candidate["joint_mae_radians"] <= baseline["joint_mae_radians"] * 1.10} return all(checks.values()), checks def publish(run, records, state): from scripts.third_party_dashboard import update_dashboard, render_curves update_dashboard(run) points = [{"step": r["epoch"], "train/loss_action": r["action_loss"], "train/loss_control": r["control_ce"]} for r in records] svg = render_curves(points).replace("Optimizer step on x axis", "Completed epoch on x axis") (run / "epoch_losses.svg").write_text(svg) write_json(run / "convergence.json", {"scope": "train_only", "state": state, "epochs": records}) content = (run / "review.md").read_text() extra = ["", "## 完整遍数与收敛检查", "", "只训原 300 条;每遍每条恰好一次,保留尾批 12 条。新阶段从 0 计步,不与旧 793374 直接相加。", "最少新增 6 遍、最多 20 遍;连续三次控制/动作 loss 改善均小于 1% 且无闭环退化才称平台。", "预算耗尽、指标退化、异常退出不叫收敛;全部指标来自训练集,不代表泛化。", "", "![逐轮分项 loss](epoch_losses.svg)", "", "[收敛记录](convergence.json) · [轨迹覆盖计划](coverage_plan.json) · [当前阶段](workflow_state.json)", "", "| 新增遍数 | 动作 loss | 控制 CE | 固定60准确率 | 背景误动率 | 关节 MAE |", "|---:|---:|---:|---:|---:|---:|"] for r in records: p = r["probe"] extra.append(f"| {r['epoch']} | {r['action_loss']:.5f} | {r['control_ce']:.5f} | " f"{p['control_accuracy']:.2%} | {p['background_only_false_motion_rate']:.2%} | {p['joint_mae_radians']:.5f} |") (run / "review.md").write_text(content + "\n".join(extra) + "\n") def preserve_best(run, checkpoint, record): """One extra weights-only best snapshot; full resume states rotate separately.""" destination = run / "best_model" destination.mkdir(exist_ok=True) for name in ("model.safetensors", "hfd_fused_delta_manifest.json"): temporary = destination / (name + ".tmp") shutil.copy2(checkpoint / name, temporary) temporary.replace(destination / name) write_json(destination / "selection.json", {"step": record["step"], "epoch": record["epoch"], "probe": record["probe"], "source_checkpoint": str(checkpoint), "weights_only_not_full_state_resume": True, "policy": "lowest joint MAE among new epochs passing mixed/clean guardrails; not automatically deployed"}) def run_training(args, prepared): import torch from torch.utils.data import DataLoader from accelerate.utils import set_seed from scripts.train import _initialize_accelerator from scripts.train_third_party_strategy import source_batches from scripts.third_party_joint_init import initialize_joint_weights from scripts.third_party_convergence_plan import EpochLossAccumulator, assess_progress from scripts.third_party_resume import code_identity from scripts.infer.third_party_metrics import evaluate_checkpoint, aggregate from scripts import train_third_party_control_repair as repair from psi.trainers.posttrain_third_party import PosttrainThirdPartyTrainer, append_record from psi.trainers.posttrain_full_duplex import _source_collator from psi.utils import flatten cfg, identity, selection, plan, panel, verified = prepared run = args.run_dir for name in ("metrics.jsonl", "stage_identity.json", "checkpoints"): if (run / name).exists(): raise FileExistsError(f"Refusing to overwrite a prior stage: {run / name}") run.mkdir(parents=True, exist_ok=True) def phase(name, **fields): write_json(run / "workflow_state.json", {"phase": name, "time": time.time(), **fields}) phase("initializing_joint_model") write_json(run / "coverage_plan.json", plan) write_json(run / "stage_identity.json", identity) write_json(run / "code_identity.json", code_identity()) paths = ("scripts/train_third_party_convergence.py", "scripts/third_party_convergence_plan.py", "scripts/third_party_joint_init.py") write_json(run / "stage_code_identity.json", {p: hashlib.sha256((PROJECT / p).read_bytes()).hexdigest() for p in paths}) write_json(run / "input_identity.json", {"manifest_sha256": hashlib.sha256(args.manifest.read_bytes()).hexdigest(), "train_count": 300, "visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0, "initialization": "793374 trained model plus verified uniform control adapter; fresh optimizer/schedule"}) os.environ["HFD_CONTINUOUS_DATA_ROOT"] = cfg.data.continuous_audio_root os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = cfg.data.background_assignment_path os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str(Path(cfg.data.continuous_audio_root) / "reports/production_complete.json") class JointTrainer(PosttrainThirdPartyTrainer): @property def num_steps_per_epoch(self): return plan["epochs"][0]["optimizer_updates"] def _checkpoint_probe(self, global_step): return None # explicit epoch metrics, no intermediate rendering set_seed(int(cfg.seed)) torch.cuda.set_device(0) trainer = JointTrainer(cfg, 0) trainer.init_models() trainer.global_step = 0 loaded = initialize_joint_weights(trainer, identity) write_json(run / "initialization_verified.json", loaded) accelerator = _initialize_accelerator(trainer) trainer.create_datasets() # prepare() needs a loader, but source_batches below owns all sampling. # Do not use the old divisible-batch sampler: it would drop the last 12. trainer.train_dataloader = DataLoader(trainer.train_dataset, batch_size=1, shuffle=False, num_workers=0, collate_fn=_source_collator) trainer.val_dataloader = None trainer.create_optimizer_and_scheduler(cfg.train.max_training_steps) trainer.prepare(accelerator) if trainer.optimizer.state_dict()["state"]: raise RuntimeError("New phase unexpectedly inherited optimizer moments") (run / "run_config.json").write_text(cfg.model_dump_json(indent=2)) append_record(run / "events.jsonl", {"event": "joint_stage_initialized", "time": time.time(), "step": 0, "optimizer_reset": True, "scheduler_reset": True, "parent_step": 793374, "max_epochs": 20, "min_epochs": 6, "batch_size": 16}) records, step, reload_passed, best_mae = [], 0, False, float("inf") decision = {"action": "continue", "reason": "stage_started"} publish(run, records, decision) try: for epoch_plan in plan["epochs"]: epoch = epoch_plan["epoch"] groups = epoch_plan["source_indices"] accumulator = EpochLossAccumulator() completed = [] halfway_saved = False trainer.set_train() phase("joint_training", epoch=epoch, completed_sources=0, step=step, max_epochs=20) for group_index, (indices, batch) in enumerate(zip(groups, source_batches(trainer, groups))): cfg.train.train_batch_size = len(indices) cfg.train.target_effective_sources_per_gpu = len(indices) before = step boundary = False for _, values, boundary in trainer.iter_optimizer_steps(batch, global_step_start=step, local_step=group_index, max_optimizer_events=epoch_plan["group_steps"][group_index]): step += 1 trainer.global_step = step active = values["effective_sources_per_gpu"] * values["dense_windows_per_optimizer_step"] accumulator.add(action_loss=values["loss_action"], action_samples=active, control_ce=values.get("loss_control"), control_samples=values["control_update_samples"]) trainer.log(flatten({**values, "epoch": epoch, "completed_sources": len(completed)}, parent_key="train")) if step % 100 == 0: print(f"[joint_train] epoch={epoch}/20 step={step} sources={len(completed)}/300 " f"loss={values['loss']:.5g} action={values['loss_action']:.5g}", flush=True) if step % 1000 == 0: phase("joint_training", epoch=epoch, completed_sources=len(completed), step=step, max_epochs=20) if not boundary or step - before != epoch_plan["group_steps"][group_index]: raise RuntimeError("Source group was cut before its natural end") completed.extend(indices) cfg.train.train_batch_size = 16 cfg.train.target_effective_sources_per_gpu = 16 state = {"epoch": epoch, "step": step, "completed_source_indices": completed, "next_group": group_index + 1, "accumulator": accumulator.summary(), "plan": str(run / "coverage_plan.json"), "fresh_stage_not_legacy_resume": True} write_json(run / "coverage_state.json", state) final_group = group_index == len(groups) - 1 halfway = not halfway_saved and len(completed) >= 150 if not reload_passed or halfway or final_group: checkpoint = Path(trainer.save_checkpoint(step)) write_json(checkpoint / "coverage_state.json", state) if not reload_passed: # Real full-state readback at the first complete group, # before spending the multi-epoch training budget. accelerator.load_state(str(checkpoint)) scheduler = trainer.lr_scheduler.state_dict() if scheduler["last_epoch"] != step or not trainer.optimizer.state_dict()["state"]: raise RuntimeError("Joint stage checkpoint failed full-state reload") reload_passed = True append_record(run / "events.jsonl", {"event": "joint_stage_reload_passed", "step": step, "time": time.time(), "scheduler_last_epoch": scheduler["last_epoch"]}) halfway_saved |= halfway phase("joint_training", epoch=epoch, completed_sources=len(completed), step=step, max_epochs=20) publish(run, records, decision) if len(completed) != 300 or len(set(completed)) != 300: raise RuntimeError("Epoch source coverage is incomplete or duplicated") summary = accumulator.summary() if (summary["action_samples"] != epoch_plan["action_samples"] or summary["control_samples"] != epoch_plan["control_samples"]): raise RuntimeError(f"Epoch did not score its complete frame/window budget: {summary}") phase("epoch_train60_metrics", epoch=epoch, step=step) torch.cuda.synchronize() torch.cuda.empty_cache() output = run / "metrics_evaluations" / f"step_{step}" mixed = evaluate_checkpoint(run, step, args.manifest, output, workers=4, count=60, reuse=False) phase("epoch_paired24_metrics", epoch=epoch, step=step) clean_rows = repair.parallel(SimpleNamespace(base_run=run, step=step, manifest=args.manifest, run_dir=output), {}, panel, "clean", view="clean", adapter=None) clean = aggregate(clean_rows) clean_ok, clean_checks = clean_safe(clean, verified["candidate_clean"]) diagnostics = {} if "diagnostic_summary" in verified: # CPU-only reuse of saved predictions: report termination and # genuine new-trigger events separately from BG frame errors. diagnosed = repair.baseline_rows(args, mixed, evaluation_ids(selection)) diagnostics = {"mixed": repair.summarize_diagnostics(diagnosed), "clean": repair.summarize_diagnostics(clean_rows)} write_json(output / "control_diagnostics.json", diagnostics) record = {"epoch": epoch, "step": step, "complete": True, "source_count": 300, **summary, "probe": mixed["aggregate"], "clean_probe": clean, "control_diagnostics": diagnostics, "clean_guardrails_passed": clean_ok, "clean_checks": clean_checks, "mixed_report": str(output / "report.json"), "clean_report": str(output / "clean/report.json")} records.append(record) decision = assess_progress(records, baseline_probe=verified["candidate_mixed"]) if len(records) >= 2 and not clean_ok and not records[-2]["clean_guardrails_passed"]: decision = {"action": "stop_regression", "reason": "paired clean regression in two consecutive epochs", "converged": False} elif not clean_ok and decision["action"] == "stop_plateau": decision = {"action": "continue", "reason": "loss plateau but clean guardrails failed", "converged": False} p, b = mixed["aggregate"], verified["candidate_mixed"] mixed_ok = (p["control_accuracy"] >= b["control_accuracy"] - .02 and p["background_only_false_motion_rate"] <= b["background_only_false_motion_rate"] + .05 and p["missed_motion_rate"] <= b["missed_motion_rate"] + .02 and p["joint_mae_radians"] <= b["joint_mae_radians"] * 1.10) if clean_ok and mixed_ok and p["joint_mae_radians"] < best_mae: preserve_best(run, checkpoint, record) best_mae = p["joint_mae_radians"] append_record(run / "epoch_metrics.jsonl", record) append_record(run / "events.jsonl", {"event": "epoch_convergence_check", "step": step, "time": time.time(), "epoch": epoch, "decision": decision}) publish(run, records, decision) if decision["action"] != "continue": break # Evaluations use child processes; main RNG and live model remain intact. if decision["action"] == "continue": decision = {"action": "stop_budget", "reason": "20 additional epochs reached", "converged": False} if decision["action"] in ("stop_plateau", "stop_budget"): phase("final_train300_metrics", epoch=records[-1]["epoch"], step=step) evaluate_checkpoint(run, step, args.manifest, run / "final_train300", workers=4, count=300, reuse=False) write_json(run / "result.json", {"status": "complete", "decision": decision, "step": step, "completed_additional_epochs": len(records), "original_epochs": 3, "training_sessions": 300, "old_weights_modified": False, "automatic_scale_up": False, "scope": "training fit only", "best_new_model_available": math.isfinite(best_mae), "final_evaluated_checkpoint": "last, not necessarily best"}) phase("completed", decision=decision, completed_epochs=len(records), step=step) except Exception as error: phase("failed", step=step, error=repr(error)) append_record(run / "events.jsonl", {"event": "joint_training_failed", "step": step, "time": time.time(), "error": repr(error)}) raise finally: trainer.finalize() publish(run, records, decision) def main(): parser = argparse.ArgumentParser(description=__doc__) for name in ("manifest", "run-dir", "base-checkpoint", "adapter", "verification-run"): parser.add_argument(f"--{name}", type=Path, required=True) mode = parser.add_mutually_exclusive_group(required=True) for name in ("check-only", "launch", "worker"): mode.add_argument(f"--{name}", action="store_true") args = parser.parse_args() for name in ("manifest", "run_dir", "base_checkpoint", "adapter", "verification_run"): setattr(args, name, getattr(args, name).resolve()) os.chdir(PROJECT) for key in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT", "ACCELERATE_USE_DEEPSPEED", "ACCELERATE_USE_FSDP", "ACCELERATE_USE_MEGATRON_LM", "ACCELERATE_USE_CPU", "SLURM_PROCID", "SLURM_LOCALID", "SLURM_NTASKS"): os.environ.pop(key, None) from scripts.train_third_party_strategy import setup_environment setup_environment() prepared = configure(args) if args.check_only: print(json.dumps({"passed": True, "train_count": 300, "batch": 16, "min_additional_epochs": 6, "max_additional_epochs": 20, "max_new_optimizer_steps": prepared[3]["max_optimizer_updates"], "initialization": "793374 plus verified uniform head; new optimizer/schedule"})) elif args.launch: if args.run_dir.exists(): raise FileExistsError("Use a fresh run directory") spec = build_launch_spec(args) path = PROJECT / "tmp" / f"{args.run_dir.name}.launch.json" if path.exists(): raise FileExistsError(path) write_json(path, spec) subprocess.run([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(path)], check=True) else: if os.environ.get("CUDA_VISIBLE_DEVICES") != "3" or os.environ.get("HFD_NPROC_PER_NODE") != "1": raise RuntimeError("Only use the managed --launch on GPU 3") try: run_training(args, prepared) except Exception as error: state_path = args.run_dir / "workflow_state.json" if state_path.exists(): previous = read(state_path) write_json(state_path, {**previous, "phase": "failed", "previous_phase": previous["phase"], "error": repr(error), "time": time.time()}) raise if __name__ == "__main__": main()