Download scripts/train_third_party_convergence.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 21.5 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/train_third_party_convergence.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/train_third_party_convergence.py
-
curl -L -o train_third_party_convergence.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/train_third_party_convergence.py
21.5 kB
| #!/usr/bin/env python3 | |
| """Bounded, coverage-balanced joint fitting of the existing train300. | |
| Fresh optimizer/schedule; immutable trained backbone/action + verified head. | |
| No heldout inputs, no extra tasks, no random cuts through recurrent histories. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import shutil | |
| import subprocess | |
| import sys | |
| import time | |
| from types import SimpleNamespace | |
| PROJECT = Path(__file__).resolve().parents[1] | |
| sys.path[:0] = [str(PROJECT), str(PROJECT / "src")] | |
| from scripts.third_party_strategy_plan import evaluation_ids, write_json | |
| def read(path): | |
| return json.loads(Path(path).read_text()) | |
| def configure(args): | |
| from scripts.train_third_party import build_config, contained_output, PRODUCTION | |
| from scripts.data.prepare_third_party_training import validate_acceptance | |
| from scripts.third_party_convergence_plan import plan_epochs | |
| from scripts.third_party_joint_init import prepare_joint_initialization | |
| args.run_dir = contained_output(args.run_dir) | |
| args.manifest = args.manifest.resolve() | |
| validate_acceptance(args.manifest, PRODUCTION, deep_payload=False) | |
| selection = read(args.manifest) | |
| plan = plan_epochs(selection, batch_size=16, seed=26100701, | |
| min_epochs=6, max_epochs=20) | |
| cfg = build_config(args.manifest, args.run_dir, "train") | |
| cfg.train.train_batch_size = 16 | |
| cfg.train.target_effective_sources_per_gpu = 16 | |
| cfg.train.coverage_epochs = 20 | |
| cfg.train.max_training_steps = plan["max_optimizer_updates"] | |
| cfg.train.learning_rate = 2e-5 | |
| cfg.model.lang_backbone_lr = 5e-7 | |
| cfg.train.warmup_steps = 500 | |
| cfg.train.cosine_min_lr_rate = .1 | |
| cfg.train.max_checkpoints_to_keep = 3 | |
| cfg.train.num_workers = 8 | |
| # Revalidate all existing model/data/safety contracts after overrides. | |
| cfg = type(cfg).model_validate(cfg.model_dump()) | |
| identity = prepare_joint_initialization(cfg, base_checkpoint=args.base_checkpoint, | |
| adapter=args.adapter, verification_run=args.verification_run, | |
| manifest=args.manifest, run_dir=args.run_dir) | |
| verified = read(args.verification_run / "result.json") | |
| panel = read(args.adapter.parent.parent / "selection.json")["paired_twentyfour_ids"] | |
| if len(panel) != 24 or len(set(panel)) != 24 or not set(panel) <= set(evaluation_ids(selection)): | |
| raise ValueError("Clean diagnostics must be the original paired subset of fixed train60") | |
| return cfg, identity, selection, plan, panel, verified | |
| def build_launch_spec(args): | |
| command = [sys.executable, str(Path(__file__).resolve()), "--manifest", str(args.manifest), | |
| "--run-dir", str(args.run_dir), "--base-checkpoint", str(args.base_checkpoint), | |
| "--adapter", str(args.adapter), "--verification-run", str(args.verification_run)] | |
| return {"schema": 1, "project_root": str(PROJECT), "run_dir": str(args.run_dir), "gpu": 3, | |
| "manifest_path": str(args.manifest), "readiness_file": str(args.manifest.parent / "acceptance.json"), | |
| "argv": [*command, "--worker"], "preflight_command": [*command, "--check-only"]} | |
| def clean_safe(candidate, baseline): | |
| checks = {"accuracy": candidate["control_accuracy"] >= baseline["control_accuracy"] - .02, | |
| "missed_motion": candidate["missed_motion_rate"] <= baseline["missed_motion_rate"] + .02, | |
| "joint_mae": candidate["joint_mae_radians"] <= baseline["joint_mae_radians"] * 1.10} | |
| return all(checks.values()), checks | |
| def publish(run, records, state): | |
| from scripts.third_party_dashboard import update_dashboard, render_curves | |
| update_dashboard(run) | |
| points = [{"step": r["epoch"], "train/loss_action": r["action_loss"], | |
| "train/loss_control": r["control_ce"]} for r in records] | |
| svg = render_curves(points).replace("Optimizer step on x axis", "Completed epoch on x axis") | |
| (run / "epoch_losses.svg").write_text(svg) | |
| write_json(run / "convergence.json", {"scope": "train_only", "state": state, "epochs": records}) | |
| content = (run / "review.md").read_text() | |
| extra = ["", "## 完整遍数与收敛检查", "", | |
| "只训原 300 条;每遍每条恰好一次,保留尾批 12 条。新阶段从 0 计步,不与旧 793374 直接相加。", | |
| "最少新增 6 遍、最多 20 遍;连续三次控制/动作 loss 改善均小于 1% 且无闭环退化才称平台。", | |
| "预算耗尽、指标退化、异常退出不叫收敛;全部指标来自训练集,不代表泛化。", "", | |
| "", "", | |
| "[收敛记录](convergence.json) · [轨迹覆盖计划](coverage_plan.json) · [当前阶段](workflow_state.json)", "", | |
| "| 新增遍数 | 动作 loss | 控制 CE | 固定60准确率 | 背景误动率 | 关节 MAE |", "|---:|---:|---:|---:|---:|---:|"] | |
| for r in records: | |
| p = r["probe"] | |
| extra.append(f"| {r['epoch']} | {r['action_loss']:.5f} | {r['control_ce']:.5f} | " | |
| f"{p['control_accuracy']:.2%} | {p['background_only_false_motion_rate']:.2%} | {p['joint_mae_radians']:.5f} |") | |
| (run / "review.md").write_text(content + "\n".join(extra) + "\n") | |
| def preserve_best(run, checkpoint, record): | |
| """One extra weights-only best snapshot; full resume states rotate separately.""" | |
| destination = run / "best_model" | |
| destination.mkdir(exist_ok=True) | |
| for name in ("model.safetensors", "hfd_fused_delta_manifest.json"): | |
| temporary = destination / (name + ".tmp") | |
| shutil.copy2(checkpoint / name, temporary) | |
| temporary.replace(destination / name) | |
| write_json(destination / "selection.json", {"step": record["step"], "epoch": record["epoch"], | |
| "probe": record["probe"], "source_checkpoint": str(checkpoint), | |
| "weights_only_not_full_state_resume": True, | |
| "policy": "lowest joint MAE among new epochs passing mixed/clean guardrails; not automatically deployed"}) | |
| def run_training(args, prepared): | |
| import torch | |
| from torch.utils.data import DataLoader | |
| from accelerate.utils import set_seed | |
| from scripts.train import _initialize_accelerator | |
| from scripts.train_third_party_strategy import source_batches | |
| from scripts.third_party_joint_init import initialize_joint_weights | |
| from scripts.third_party_convergence_plan import EpochLossAccumulator, assess_progress | |
| from scripts.third_party_resume import code_identity | |
| from scripts.infer.third_party_metrics import evaluate_checkpoint, aggregate | |
| from scripts import train_third_party_control_repair as repair | |
| from psi.trainers.posttrain_third_party import PosttrainThirdPartyTrainer, append_record | |
| from psi.trainers.posttrain_full_duplex import _source_collator | |
| from psi.utils import flatten | |
| cfg, identity, selection, plan, panel, verified = prepared | |
| run = args.run_dir | |
| for name in ("metrics.jsonl", "stage_identity.json", "checkpoints"): | |
| if (run / name).exists(): | |
| raise FileExistsError(f"Refusing to overwrite a prior stage: {run / name}") | |
| run.mkdir(parents=True, exist_ok=True) | |
| def phase(name, **fields): | |
| write_json(run / "workflow_state.json", {"phase": name, "time": time.time(), **fields}) | |
| phase("initializing_joint_model") | |
| write_json(run / "coverage_plan.json", plan) | |
| write_json(run / "stage_identity.json", identity) | |
| write_json(run / "code_identity.json", code_identity()) | |
| paths = ("scripts/train_third_party_convergence.py", "scripts/third_party_convergence_plan.py", | |
| "scripts/third_party_joint_init.py") | |
| write_json(run / "stage_code_identity.json", {p: hashlib.sha256((PROJECT / p).read_bytes()).hexdigest() for p in paths}) | |
| write_json(run / "input_identity.json", {"manifest_sha256": hashlib.sha256(args.manifest.read_bytes()).hexdigest(), | |
| "train_count": 300, "visual_inputs": 0, "other_task_inputs": 0, "heldout_inputs": 0, | |
| "initialization": "793374 trained model plus verified uniform control adapter; fresh optimizer/schedule"}) | |
| os.environ["HFD_CONTINUOUS_DATA_ROOT"] = cfg.data.continuous_audio_root | |
| os.environ["HFD_BACKGROUND_ASSIGNMENT_PATH"] = cfg.data.background_assignment_path | |
| os.environ["HFD_CONTINUOUS_DATA_CONTRACT_PATH"] = str(Path(cfg.data.continuous_audio_root) / "reports/production_complete.json") | |
| class JointTrainer(PosttrainThirdPartyTrainer): | |
| def num_steps_per_epoch(self): | |
| return plan["epochs"][0]["optimizer_updates"] | |
| def _checkpoint_probe(self, global_step): | |
| return None # explicit epoch metrics, no intermediate rendering | |
| set_seed(int(cfg.seed)) | |
| torch.cuda.set_device(0) | |
| trainer = JointTrainer(cfg, 0) | |
| trainer.init_models() | |
| trainer.global_step = 0 | |
| loaded = initialize_joint_weights(trainer, identity) | |
| write_json(run / "initialization_verified.json", loaded) | |
| accelerator = _initialize_accelerator(trainer) | |
| trainer.create_datasets() | |
| # prepare() needs a loader, but source_batches below owns all sampling. | |
| # Do not use the old divisible-batch sampler: it would drop the last 12. | |
| trainer.train_dataloader = DataLoader(trainer.train_dataset, batch_size=1, | |
| shuffle=False, num_workers=0, collate_fn=_source_collator) | |
| trainer.val_dataloader = None | |
| trainer.create_optimizer_and_scheduler(cfg.train.max_training_steps) | |
| trainer.prepare(accelerator) | |
| if trainer.optimizer.state_dict()["state"]: | |
| raise RuntimeError("New phase unexpectedly inherited optimizer moments") | |
| (run / "run_config.json").write_text(cfg.model_dump_json(indent=2)) | |
| append_record(run / "events.jsonl", {"event": "joint_stage_initialized", "time": time.time(), | |
| "step": 0, "optimizer_reset": True, "scheduler_reset": True, "parent_step": 793374, | |
| "max_epochs": 20, "min_epochs": 6, "batch_size": 16}) | |
| records, step, reload_passed, best_mae = [], 0, False, float("inf") | |
| decision = {"action": "continue", "reason": "stage_started"} | |
| publish(run, records, decision) | |
| try: | |
| for epoch_plan in plan["epochs"]: | |
| epoch = epoch_plan["epoch"] | |
| groups = epoch_plan["source_indices"] | |
| accumulator = EpochLossAccumulator() | |
| completed = [] | |
| halfway_saved = False | |
| trainer.set_train() | |
| phase("joint_training", epoch=epoch, completed_sources=0, step=step, max_epochs=20) | |
| for group_index, (indices, batch) in enumerate(zip(groups, source_batches(trainer, groups))): | |
| cfg.train.train_batch_size = len(indices) | |
| cfg.train.target_effective_sources_per_gpu = len(indices) | |
| before = step | |
| boundary = False | |
| for _, values, boundary in trainer.iter_optimizer_steps(batch, global_step_start=step, | |
| local_step=group_index, max_optimizer_events=epoch_plan["group_steps"][group_index]): | |
| step += 1 | |
| trainer.global_step = step | |
| active = values["effective_sources_per_gpu"] * values["dense_windows_per_optimizer_step"] | |
| accumulator.add(action_loss=values["loss_action"], action_samples=active, | |
| control_ce=values.get("loss_control"), control_samples=values["control_update_samples"]) | |
| trainer.log(flatten({**values, "epoch": epoch, "completed_sources": len(completed)}, parent_key="train")) | |
| if step % 100 == 0: | |
| print(f"[joint_train] epoch={epoch}/20 step={step} sources={len(completed)}/300 " | |
| f"loss={values['loss']:.5g} action={values['loss_action']:.5g}", flush=True) | |
| if step % 1000 == 0: | |
| phase("joint_training", epoch=epoch, completed_sources=len(completed), step=step, max_epochs=20) | |
| if not boundary or step - before != epoch_plan["group_steps"][group_index]: | |
| raise RuntimeError("Source group was cut before its natural end") | |
| completed.extend(indices) | |
| cfg.train.train_batch_size = 16 | |
| cfg.train.target_effective_sources_per_gpu = 16 | |
| state = {"epoch": epoch, "step": step, "completed_source_indices": completed, | |
| "next_group": group_index + 1, "accumulator": accumulator.summary(), | |
| "plan": str(run / "coverage_plan.json"), "fresh_stage_not_legacy_resume": True} | |
| write_json(run / "coverage_state.json", state) | |
| final_group = group_index == len(groups) - 1 | |
| halfway = not halfway_saved and len(completed) >= 150 | |
| if not reload_passed or halfway or final_group: | |
| checkpoint = Path(trainer.save_checkpoint(step)) | |
| write_json(checkpoint / "coverage_state.json", state) | |
| if not reload_passed: | |
| # Real full-state readback at the first complete group, | |
| # before spending the multi-epoch training budget. | |
| accelerator.load_state(str(checkpoint)) | |
| scheduler = trainer.lr_scheduler.state_dict() | |
| if scheduler["last_epoch"] != step or not trainer.optimizer.state_dict()["state"]: | |
| raise RuntimeError("Joint stage checkpoint failed full-state reload") | |
| reload_passed = True | |
| append_record(run / "events.jsonl", {"event": "joint_stage_reload_passed", "step": step, | |
| "time": time.time(), "scheduler_last_epoch": scheduler["last_epoch"]}) | |
| halfway_saved |= halfway | |
| phase("joint_training", epoch=epoch, completed_sources=len(completed), step=step, max_epochs=20) | |
| publish(run, records, decision) | |
| if len(completed) != 300 or len(set(completed)) != 300: | |
| raise RuntimeError("Epoch source coverage is incomplete or duplicated") | |
| summary = accumulator.summary() | |
| if (summary["action_samples"] != epoch_plan["action_samples"] | |
| or summary["control_samples"] != epoch_plan["control_samples"]): | |
| raise RuntimeError(f"Epoch did not score its complete frame/window budget: {summary}") | |
| phase("epoch_train60_metrics", epoch=epoch, step=step) | |
| torch.cuda.synchronize() | |
| torch.cuda.empty_cache() | |
| output = run / "metrics_evaluations" / f"step_{step}" | |
| mixed = evaluate_checkpoint(run, step, args.manifest, output, workers=4, count=60, reuse=False) | |
| phase("epoch_paired24_metrics", epoch=epoch, step=step) | |
| clean_rows = repair.parallel(SimpleNamespace(base_run=run, step=step, manifest=args.manifest, | |
| run_dir=output), {}, panel, "clean", view="clean", adapter=None) | |
| clean = aggregate(clean_rows) | |
| clean_ok, clean_checks = clean_safe(clean, verified["candidate_clean"]) | |
| diagnostics = {} | |
| if "diagnostic_summary" in verified: | |
| # CPU-only reuse of saved predictions: report termination and | |
| # genuine new-trigger events separately from BG frame errors. | |
| diagnosed = repair.baseline_rows(args, mixed, evaluation_ids(selection)) | |
| diagnostics = {"mixed": repair.summarize_diagnostics(diagnosed), | |
| "clean": repair.summarize_diagnostics(clean_rows)} | |
| write_json(output / "control_diagnostics.json", diagnostics) | |
| record = {"epoch": epoch, "step": step, "complete": True, "source_count": 300, | |
| **summary, "probe": mixed["aggregate"], "clean_probe": clean, | |
| "control_diagnostics": diagnostics, | |
| "clean_guardrails_passed": clean_ok, "clean_checks": clean_checks, | |
| "mixed_report": str(output / "report.json"), "clean_report": str(output / "clean/report.json")} | |
| records.append(record) | |
| decision = assess_progress(records, baseline_probe=verified["candidate_mixed"]) | |
| if len(records) >= 2 and not clean_ok and not records[-2]["clean_guardrails_passed"]: | |
| decision = {"action": "stop_regression", "reason": "paired clean regression in two consecutive epochs", "converged": False} | |
| elif not clean_ok and decision["action"] == "stop_plateau": | |
| decision = {"action": "continue", "reason": "loss plateau but clean guardrails failed", "converged": False} | |
| p, b = mixed["aggregate"], verified["candidate_mixed"] | |
| mixed_ok = (p["control_accuracy"] >= b["control_accuracy"] - .02 | |
| and p["background_only_false_motion_rate"] <= b["background_only_false_motion_rate"] + .05 | |
| and p["missed_motion_rate"] <= b["missed_motion_rate"] + .02 | |
| and p["joint_mae_radians"] <= b["joint_mae_radians"] * 1.10) | |
| if clean_ok and mixed_ok and p["joint_mae_radians"] < best_mae: | |
| preserve_best(run, checkpoint, record) | |
| best_mae = p["joint_mae_radians"] | |
| append_record(run / "epoch_metrics.jsonl", record) | |
| append_record(run / "events.jsonl", {"event": "epoch_convergence_check", "step": step, | |
| "time": time.time(), "epoch": epoch, "decision": decision}) | |
| publish(run, records, decision) | |
| if decision["action"] != "continue": | |
| break | |
| # Evaluations use child processes; main RNG and live model remain intact. | |
| if decision["action"] == "continue": | |
| decision = {"action": "stop_budget", "reason": "20 additional epochs reached", "converged": False} | |
| if decision["action"] in ("stop_plateau", "stop_budget"): | |
| phase("final_train300_metrics", epoch=records[-1]["epoch"], step=step) | |
| evaluate_checkpoint(run, step, args.manifest, run / "final_train300", workers=4, count=300, reuse=False) | |
| write_json(run / "result.json", {"status": "complete", "decision": decision, "step": step, | |
| "completed_additional_epochs": len(records), "original_epochs": 3, "training_sessions": 300, | |
| "old_weights_modified": False, "automatic_scale_up": False, "scope": "training fit only", | |
| "best_new_model_available": math.isfinite(best_mae), "final_evaluated_checkpoint": "last, not necessarily best"}) | |
| phase("completed", decision=decision, completed_epochs=len(records), step=step) | |
| except Exception as error: | |
| phase("failed", step=step, error=repr(error)) | |
| append_record(run / "events.jsonl", {"event": "joint_training_failed", "step": step, | |
| "time": time.time(), "error": repr(error)}) | |
| raise | |
| finally: | |
| trainer.finalize() | |
| publish(run, records, decision) | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| for name in ("manifest", "run-dir", "base-checkpoint", "adapter", "verification-run"): | |
| parser.add_argument(f"--{name}", type=Path, required=True) | |
| mode = parser.add_mutually_exclusive_group(required=True) | |
| for name in ("check-only", "launch", "worker"): | |
| mode.add_argument(f"--{name}", action="store_true") | |
| args = parser.parse_args() | |
| for name in ("manifest", "run_dir", "base_checkpoint", "adapter", "verification_run"): | |
| setattr(args, name, getattr(args, name).resolve()) | |
| os.chdir(PROJECT) | |
| for key in ("RANK", "LOCAL_RANK", "WORLD_SIZE", "LOCAL_WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT", | |
| "ACCELERATE_USE_DEEPSPEED", "ACCELERATE_USE_FSDP", "ACCELERATE_USE_MEGATRON_LM", | |
| "ACCELERATE_USE_CPU", "SLURM_PROCID", "SLURM_LOCALID", "SLURM_NTASKS"): | |
| os.environ.pop(key, None) | |
| from scripts.train_third_party_strategy import setup_environment | |
| setup_environment() | |
| prepared = configure(args) | |
| if args.check_only: | |
| print(json.dumps({"passed": True, "train_count": 300, "batch": 16, | |
| "min_additional_epochs": 6, "max_additional_epochs": 20, | |
| "max_new_optimizer_steps": prepared[3]["max_optimizer_updates"], | |
| "initialization": "793374 plus verified uniform head; new optimizer/schedule"})) | |
| elif args.launch: | |
| if args.run_dir.exists(): | |
| raise FileExistsError("Use a fresh run directory") | |
| spec = build_launch_spec(args) | |
| path = PROJECT / "tmp" / f"{args.run_dir.name}.launch.json" | |
| if path.exists(): | |
| raise FileExistsError(path) | |
| write_json(path, spec) | |
| subprocess.run([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), | |
| "start", "--spec", str(path)], check=True) | |
| else: | |
| if os.environ.get("CUDA_VISIBLE_DEVICES") != "3" or os.environ.get("HFD_NPROC_PER_NODE") != "1": | |
| raise RuntimeError("Only use the managed --launch on GPU 3") | |
| try: | |
| run_training(args, prepared) | |
| except Exception as error: | |
| state_path = args.run_dir / "workflow_state.json" | |
| if state_path.exists(): | |
| previous = read(state_path) | |
| write_json(state_path, {**previous, "phase": "failed", | |
| "previous_phase": previous["phase"], "error": repr(error), "time": time.time()}) | |
| raise | |
| if __name__ == "__main__": | |
| main() | |