#!/usr/bin/env python3 """Independently verify an already fitted control adapter; never fit a model. Only completed, hash-bound repair runs may supply adapters and paired baselines. The same frozen train60/train24 receive new raw-audio closed-loop predictions. An adapter is not a full-state checkpoint and is never automatically promoted. """ from __future__ import annotations import argparse import json import math import os from pathlib import Path import subprocess import sys import time from types import SimpleNamespace PROJECT = Path(__file__).resolve().parents[1] sys.path[:0] = [str(PROJECT), str(PROJECT / "src")] from scripts import train_third_party_control_repair as repair from scripts.third_party_strategy_plan import write_json CANDIDATES = ("uniform_ce", "balanced_ce") SOURCE_CODE = ("scripts/train_third_party_control_repair.py", "scripts/third_party_control_refit.py", "scripts/third_party_control_diagnostics.py", "scripts/third_party_clean_view.py") def _read(path): return json.loads(Path(path).read_text()) def _completed(path): status = _read(path / "run_status.json") if (status.get("phase") != "completed" or status.get("training_returncode") != 0 or status.get("launcher_returncode") != 0 or status.get("keeper_restore_verified") is not True): raise ValueError("Source/base run must be completed successfully with keeper restored") def _report(source, name, ids, seeds, *, view=None, adapter=None): path = source / ("baseline_mixed.json" if name == "baseline_mixed" else f"{name}/report.json") report = _read(path) if report.get("count") != len(ids) or [r["session_id"] for r in report.get("samples", [])] != ids: raise ValueError(f"Incomplete or different fixed selection: {name}") if name != "baseline_mixed": if (report.get("status") != "complete" or report.get("split") != "train_only" or report.get("view") != view or report.get("adapter") != adapter): raise ValueError(f"Incomplete/nonmatching source report: {name}") for row in report["samples"]: sid = row["session_id"] if row.get("seed") != seeds[sid] or "metrics" not in row or "diagnostics" not in row: raise ValueError(f"Unpaired or incomplete source sample: {name}/{sid}") if name == "baseline_mixed": continue directory = repair.local(row["prediction_directory"], output=True) if directory != source / name / "samples" / sid: raise ValueError("Source prediction path differs from its report") for filename in ("motion.npy", "infer_meta.json", "session_recipe.json", "result.json"): if not (directory / filename).is_file(): raise ValueError(f"Missing completed prediction artifact: {name}/{sid}/{filename}") if _read(directory / "result.json") != row: raise ValueError("Source report differs from completed sample result") metadata = _read(directory / "infer_meta.json") if (metadata.get("session_id") != sid or metadata.get("audio_view") != view or metadata.get("control_adapter") != adapter): raise ValueError("Source prediction metadata differs from its report") return report, path def prepare_verification(args): """Read-only complete provenance gate, including current source code hashes.""" if args.candidate not in CANDIDATES: raise ValueError("Unknown fitted control candidate") source = repair.local(args.source_run, output=True) output = repair.local(args.run_dir, output=True) _completed(source) identity = _read(source / "identity.json") base = repair.local(identity["base_run"], output=True) if output.is_relative_to(source) or output.is_relative_to(base) or source.is_relative_to(output) or base.is_relative_to(output): raise ValueError("Use a fresh independent output run, not the immutable source/base") _completed(base) cfg = _read(base / "run_config.json") manifest = repair.local(cfg["data"]["frozen_manifest"]) if getattr(args, "manifest", None) is not None and repair.local(args.manifest) != manifest: raise ValueError("Explicit --manifest differs from the source frozen manifest") step = int(identity["base_step"]) checkpoint = base / "checkpoints" / f"ckpt_{step}" expected = {"base_weights_sha256": repair.digest(checkpoint / "model.safetensors"), "base_config_sha256": repair.digest(base / "run_config.json"), "manifest_sha256": repair.digest(manifest)} if any(identity.get(k) != v for k, v in expected.items()): raise ValueError("Source identity no longer matches base weights/config/manifest") if (identity.get("training_count") != 300 or any(identity.get(k) != 0 for k in ("heldout_inputs", "visual_inputs", "other_task_inputs"))): raise ValueError("Source repair scope was not the frozen audio-only train300") code = identity.get("code_sha256", {}) if set(code) != set(SOURCE_CODE) or any(repair.digest(PROJECT / name) != code[name] for name in SOURCE_CODE): raise ValueError("Source repair code changed; do not silently reuse its feature/clean policy") repair_args = SimpleNamespace(base_run=base, step=step, manifest=manifest, run_dir=output) selection, previous, sixty, panel = repair.prepare(repair_args) chosen = _read(source / "selection.json") if chosen.get("fixed_sixty_ids") != sixty or chosen.get("paired_twentyfour_ids") != panel: raise ValueError("Source fixed60/paired24 selection differs from frozen training selection") source_result = _read(source / "result.json") if (source_result.get("status") != "complete" or source_result.get("candidate") not in CANDIDATES or source_result.get("promoted") is not False or source_result.get("old_weights_modified") is not False or _read(source / "workflow_state.json").get("phase") != "completed"): raise ValueError("Source repair has not finished its complete paired candidate verification") fit = _read(source / "fit_report.json") if args.candidate not in fit.get("candidates", {}): raise ValueError("Requested adapter was not fitted by this completed repair") adapter = source / "adapters" / f"{args.candidate}.safetensors" adapter_record = _read(adapter.with_suffix(".json")) if (adapter_record.get("format") != "hfd_control_head_adapter_v1" or adapter_record.get("candidate") != args.candidate or adapter_record.get("not_full_state_resume_checkpoint") is not True or adapter_record.get("selection_scope") != "train_only" or adapter_record.get("base_checkpoint") != str(checkpoint) or adapter_record.get("code_sha256") != code or adapter_record.get("fit_config") != fit.get("config") or any(adapter_record.get(k) != identity[k] for k in ("base_run", "base_step", "training_count", "heldout_inputs", "visual_inputs", "other_task_inputs")) or any(adapter_record.get(k) != v for k, v in expected.items()) or repair.digest(adapter) != adapter_record.get("adapter_sha256")): raise ValueError("Adapter hash/provenance does not bind this immutable base") from safetensors.torch import load_file state = load_file(str(adapter), device="cpu") repair.build_head(state) if (state["1.weight"].shape != (cfg["model"]["streaming_memory_dim"], cfg["model"]["view_feature_dim"])): raise ValueError("Adapter dimensions do not match base control_head") seeds = {r["session_id"]: r["seed"] for r in previous["samples"]} selected_adapter = str(source / "adapters" / f"{source_result['candidate']}.safetensors") if source_result.get("adapter") != selected_adapter: raise ValueError("Source completed candidate identity is inconsistent") reports, paths = {}, [] for name, ids, view, used_adapter in ( ("baseline_mixed", sixty, None, None), ("baseline_clean", panel, "clean", None), ("candidate_mixed", sixty, "mixed", selected_adapter), ("candidate_clean", panel, "clean", selected_adapter)): reports[name], path = _report(source, name, ids, seeds, view=view, adapter=used_adapter) paths.append(path) paths.extend(source / name for name in ("identity.json", "selection.json", "fit_report.json", "result.json", "workflow_state.json", "run_status.json")) paths.extend((adapter, adapter.with_suffix(".json"), checkpoint / "model.safetensors", base / "run_config.json", manifest)) paths.extend(PROJECT / name for name in SOURCE_CODE) hashes = {str(path): repair.digest(path) for path in paths} return {"args": repair_args, "identity": identity, "adapter": adapter, "sixty": sixty, "panel": panel, "reports": reports, "source_hashes": hashes, "source_run": source, "candidate": args.candidate} def acceptance_checks(baseline, candidate, clean_baseline, clean_candidate, diagnostics): def number(value): return value is not None and math.isfinite(float(value)) def compare(left, right, fn): return bool(number(left) and number(right) and fn(float(left), float(right))) checks = {"mixed_background_improves": compare(candidate["background_only_false_motion_rate"], baseline["background_only_false_motion_rate"], lambda a, b: a < b)} for name, old, new in (("mixed", baseline, candidate), ("clean", clean_baseline, clean_candidate)): checks[name + "_accuracy_drop_le_1pp"] = compare(new["control_accuracy"], old["control_accuracy"], lambda a, b: a >= b - .01) checks[name + "_missed_motion_rise_le_2pp"] = compare(new["missed_motion_rate"], old["missed_motion_rate"], lambda a, b: a <= b + .02) checks[name + "_joint_mae_rise_le_10pct"] = compare(new["joint_mae_radians"], old["joint_mae_radians"], lambda a, b: a <= b * 1.10) checks["mixed_terminal_idle_not_fewer"] = (diagnostics["candidate_mixed"]["sessions_with_terminal_idle_dwell_0_4s"] >= diagnostics["baseline_mixed"]["sessions_with_terminal_idle_dwell_0_4s"]) return checks def workflow(args, prepared): from scripts.infer.third_party_metrics import aggregate context = prepared["args"] context.run_dir.mkdir(parents=True, exist_ok=True) def phase(name, **fields): write_json(context.run_dir / "workflow_state.json", {"phase": name, "time": time.time(), **fields}) write_json(context.run_dir / "identity.json", {**prepared["identity"], "verification_entry_sha256": repair.digest(Path(__file__).resolve()), "source_run": str(prepared["source_run"]), "source_hashes": prepared["source_hashes"], "candidate": args.candidate, "adapter": str(prepared["adapter"]), "fitting_performed": False, "feature_extraction_performed": False}) write_json(context.run_dir / "selection.json", {"fixed_sixty_ids": prepared["sixty"], "paired_twentyfour_ids": prepared["panel"]}) phase("verify_mixed_closed_loop", candidate=args.candidate, count=60) mixed = repair.parallel(context, prepared["identity"], prepared["sixty"], "candidate_mixed", adapter=prepared["adapter"]) phase("verify_clean_closed_loop", candidate=args.candidate, count=24) clean = repair.parallel(context, prepared["identity"], prepared["panel"], "candidate_clean", view="clean", adapter=prepared["adapter"]) rows = {"baseline_mixed": prepared["reports"]["baseline_mixed"]["samples"], "baseline_clean": prepared["reports"]["baseline_clean"]["samples"], "candidate_mixed": mixed, "candidate_clean": clean} metrics = {name: aggregate(values) for name, values in rows.items()} diagnostics = {name: repair.summarize_diagnostics(values) for name, values in rows.items()} checks = acceptance_checks(metrics["baseline_mixed"], metrics["candidate_mixed"], metrics["baseline_clean"], metrics["candidate_clean"], diagnostics) for path, expected in prepared["source_hashes"].items(): if repair.digest(path) != expected: raise ValueError(f"Immutable verification source changed during execution: {path}") write_json(context.run_dir / "result.json", {"status": "complete", "candidate": args.candidate, "adapter": str(prepared["adapter"]), "not_full_state_resume_checkpoint": True, "promoted": False, "automatic_scale_up": False, "old_weights_modified": False, "fitting_performed": False, "feature_extraction_performed": False, "pilot_acceptance_passed": all(checks.values()), "acceptance_checks": checks, "diagnostic_summary": diagnostics, **metrics, "source_report_hashes": prepared["source_hashes"], "scope": "Existing adapter only; original fixed train60 mixed + paired train24 clean, unchanged per-session seeds; not held-out generalization", "acceptance_policy": "mixed background improves; mixed/clean accuracy drop<=1pp, missed motion rise<=2pp, joint MAE rise<=10%; mixed terminal idle not fewer"}) phase("completed", candidate=args.candidate, pilot_acceptance_passed=all(checks.values())) def build_launch_spec(args, prepared): """Bind both managed commands to the exact accepted manifest explicitly.""" manifest = prepared["args"].manifest argv = [sys.executable, str(Path(__file__).resolve()), "--source-run", str(args.source_run), "--candidate", args.candidate, "--run-dir", str(args.run_dir), "--manifest", str(manifest)] return {"schema": 1, "project_root": str(PROJECT), "gpu": 3, "run_dir": str(args.run_dir), "manifest_path": str(manifest), "readiness_file": str(manifest.parent / "acceptance.json"), "argv": [*argv, "--worker"], "preflight_command": [*argv, "--check-only"]} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--source-run", type=Path, required=True) parser.add_argument("--candidate", choices=CANDIDATES, required=True) parser.add_argument("--run-dir", type=Path, required=True) parser.add_argument("--manifest", type=Path, help="Optional explicit binding; must equal the source frozen manifest") modes = parser.add_mutually_exclusive_group(required=True) for name in ("check-only", "launch", "worker"): modes.add_argument("--" + name, action="store_true") args = parser.parse_args() args.source_run = repair.local(args.source_run, output=True) args.run_dir = repair.local(args.run_dir, output=True) prepared = prepare_verification(args) if args.check_only: print(json.dumps({"passed": True, "candidate": args.candidate, "mixed_count": 60, "clean_count": 24, "fits_model": False, "adapter": str(prepared["adapter"])})) return from scripts.train_third_party_strategy import setup_environment if args.launch: if args.run_dir.exists(): raise FileExistsError("Refusing to overwrite an existing experiment") setup_environment() spec = build_launch_spec(args, prepared) path = PROJECT / "tmp" / f"{args.run_dir.name}.launch.json" if path.exists(): raise FileExistsError(path) write_json(path, spec) subprocess.run([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(path)], check=True) else: if os.environ.get("CUDA_VISIBLE_DEVICES") != "3" or os.environ.get("HFD_NPROC_PER_NODE") != "1": raise RuntimeError("Use the managed GPU-3 launcher") setup_environment() workflow(args, prepared) if __name__ == "__main__": main()