code-HFD_lzk / scripts /verify_third_party_control_adapter.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
55e0599 verified
Raw History Blame Contribute Delete
15.8 kB
#!/usr/bin/env python3
"""Independently verify an already fitted control adapter; never fit a model.
Only completed, hash-bound repair runs may supply adapters and paired baselines.
The same frozen train60/train24 receive new raw-audio closed-loop predictions.
An adapter is not a full-state checkpoint and is never automatically promoted.
"""
from __future__ import annotations
import argparse
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import time
from types import SimpleNamespace
PROJECT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(PROJECT), str(PROJECT / "src")]
from scripts import train_third_party_control_repair as repair
from scripts.third_party_strategy_plan import write_json
CANDIDATES = ("uniform_ce", "balanced_ce")
SOURCE_CODE = ("scripts/train_third_party_control_repair.py",
"scripts/third_party_control_refit.py",
"scripts/third_party_control_diagnostics.py",
"scripts/third_party_clean_view.py")
def _read(path):
return json.loads(Path(path).read_text())
def _completed(path):
status = _read(path / "run_status.json")
if (status.get("phase") != "completed" or status.get("training_returncode") != 0
or status.get("launcher_returncode") != 0 or status.get("keeper_restore_verified") is not True):
raise ValueError("Source/base run must be completed successfully with keeper restored")
def _report(source, name, ids, seeds, *, view=None, adapter=None):
path = source / ("baseline_mixed.json" if name == "baseline_mixed" else f"{name}/report.json")
report = _read(path)
if report.get("count") != len(ids) or [r["session_id"] for r in report.get("samples", [])] != ids:
raise ValueError(f"Incomplete or different fixed selection: {name}")
if name != "baseline_mixed":
if (report.get("status") != "complete" or report.get("split") != "train_only"
or report.get("view") != view or report.get("adapter") != adapter):
raise ValueError(f"Incomplete/nonmatching source report: {name}")
for row in report["samples"]:
sid = row["session_id"]
if row.get("seed") != seeds[sid] or "metrics" not in row or "diagnostics" not in row:
raise ValueError(f"Unpaired or incomplete source sample: {name}/{sid}")
if name == "baseline_mixed":
continue
directory = repair.local(row["prediction_directory"], output=True)
if directory != source / name / "samples" / sid:
raise ValueError("Source prediction path differs from its report")
for filename in ("motion.npy", "infer_meta.json", "session_recipe.json", "result.json"):
if not (directory / filename).is_file():
raise ValueError(f"Missing completed prediction artifact: {name}/{sid}/{filename}")
if _read(directory / "result.json") != row:
raise ValueError("Source report differs from completed sample result")
metadata = _read(directory / "infer_meta.json")
if (metadata.get("session_id") != sid or metadata.get("audio_view") != view
or metadata.get("control_adapter") != adapter):
raise ValueError("Source prediction metadata differs from its report")
return report, path
def prepare_verification(args):
"""Read-only complete provenance gate, including current source code hashes."""
if args.candidate not in CANDIDATES:
raise ValueError("Unknown fitted control candidate")
source = repair.local(args.source_run, output=True)
output = repair.local(args.run_dir, output=True)
_completed(source)
identity = _read(source / "identity.json")
base = repair.local(identity["base_run"], output=True)
if output.is_relative_to(source) or output.is_relative_to(base) or source.is_relative_to(output) or base.is_relative_to(output):
raise ValueError("Use a fresh independent output run, not the immutable source/base")
_completed(base)
cfg = _read(base / "run_config.json")
manifest = repair.local(cfg["data"]["frozen_manifest"])
if getattr(args, "manifest", None) is not None and repair.local(args.manifest) != manifest:
raise ValueError("Explicit --manifest differs from the source frozen manifest")
step = int(identity["base_step"])
checkpoint = base / "checkpoints" / f"ckpt_{step}"
expected = {"base_weights_sha256": repair.digest(checkpoint / "model.safetensors"),
"base_config_sha256": repair.digest(base / "run_config.json"),
"manifest_sha256": repair.digest(manifest)}
if any(identity.get(k) != v for k, v in expected.items()):
raise ValueError("Source identity no longer matches base weights/config/manifest")
if (identity.get("training_count") != 300 or any(identity.get(k) != 0 for k in
("heldout_inputs", "visual_inputs", "other_task_inputs"))):
raise ValueError("Source repair scope was not the frozen audio-only train300")
code = identity.get("code_sha256", {})
if set(code) != set(SOURCE_CODE) or any(repair.digest(PROJECT / name) != code[name] for name in SOURCE_CODE):
raise ValueError("Source repair code changed; do not silently reuse its feature/clean policy")
repair_args = SimpleNamespace(base_run=base, step=step, manifest=manifest, run_dir=output)
selection, previous, sixty, panel = repair.prepare(repair_args)
chosen = _read(source / "selection.json")
if chosen.get("fixed_sixty_ids") != sixty or chosen.get("paired_twentyfour_ids") != panel:
raise ValueError("Source fixed60/paired24 selection differs from frozen training selection")
source_result = _read(source / "result.json")
if (source_result.get("status") != "complete" or source_result.get("candidate") not in CANDIDATES
or source_result.get("promoted") is not False or source_result.get("old_weights_modified") is not False
or _read(source / "workflow_state.json").get("phase") != "completed"):
raise ValueError("Source repair has not finished its complete paired candidate verification")
fit = _read(source / "fit_report.json")
if args.candidate not in fit.get("candidates", {}):
raise ValueError("Requested adapter was not fitted by this completed repair")
adapter = source / "adapters" / f"{args.candidate}.safetensors"
adapter_record = _read(adapter.with_suffix(".json"))
if (adapter_record.get("format") != "hfd_control_head_adapter_v1"
or adapter_record.get("candidate") != args.candidate
or adapter_record.get("not_full_state_resume_checkpoint") is not True
or adapter_record.get("selection_scope") != "train_only"
or adapter_record.get("base_checkpoint") != str(checkpoint)
or adapter_record.get("code_sha256") != code
or adapter_record.get("fit_config") != fit.get("config")
or any(adapter_record.get(k) != identity[k] for k in
("base_run", "base_step", "training_count", "heldout_inputs", "visual_inputs", "other_task_inputs"))
or any(adapter_record.get(k) != v for k, v in expected.items())
or repair.digest(adapter) != adapter_record.get("adapter_sha256")):
raise ValueError("Adapter hash/provenance does not bind this immutable base")
from safetensors.torch import load_file
state = load_file(str(adapter), device="cpu")
repair.build_head(state)
if (state["1.weight"].shape != (cfg["model"]["streaming_memory_dim"], cfg["model"]["view_feature_dim"])):
raise ValueError("Adapter dimensions do not match base control_head")
seeds = {r["session_id"]: r["seed"] for r in previous["samples"]}
selected_adapter = str(source / "adapters" / f"{source_result['candidate']}.safetensors")
if source_result.get("adapter") != selected_adapter:
raise ValueError("Source completed candidate identity is inconsistent")
reports, paths = {}, []
for name, ids, view, used_adapter in (
("baseline_mixed", sixty, None, None), ("baseline_clean", panel, "clean", None),
("candidate_mixed", sixty, "mixed", selected_adapter),
("candidate_clean", panel, "clean", selected_adapter)):
reports[name], path = _report(source, name, ids, seeds, view=view, adapter=used_adapter)
paths.append(path)
paths.extend(source / name for name in ("identity.json", "selection.json", "fit_report.json", "result.json", "workflow_state.json", "run_status.json"))
paths.extend((adapter, adapter.with_suffix(".json"), checkpoint / "model.safetensors", base / "run_config.json", manifest))
paths.extend(PROJECT / name for name in SOURCE_CODE)
hashes = {str(path): repair.digest(path) for path in paths}
return {"args": repair_args, "identity": identity, "adapter": adapter,
"sixty": sixty, "panel": panel, "reports": reports, "source_hashes": hashes,
"source_run": source, "candidate": args.candidate}
def acceptance_checks(baseline, candidate, clean_baseline, clean_candidate, diagnostics):
def number(value):
return value is not None and math.isfinite(float(value))
def compare(left, right, fn):
return bool(number(left) and number(right) and fn(float(left), float(right)))
checks = {"mixed_background_improves": compare(candidate["background_only_false_motion_rate"], baseline["background_only_false_motion_rate"], lambda a, b: a < b)}
for name, old, new in (("mixed", baseline, candidate), ("clean", clean_baseline, clean_candidate)):
checks[name + "_accuracy_drop_le_1pp"] = compare(new["control_accuracy"], old["control_accuracy"], lambda a, b: a >= b - .01)
checks[name + "_missed_motion_rise_le_2pp"] = compare(new["missed_motion_rate"], old["missed_motion_rate"], lambda a, b: a <= b + .02)
checks[name + "_joint_mae_rise_le_10pct"] = compare(new["joint_mae_radians"], old["joint_mae_radians"], lambda a, b: a <= b * 1.10)
checks["mixed_terminal_idle_not_fewer"] = (diagnostics["candidate_mixed"]["sessions_with_terminal_idle_dwell_0_4s"] >= diagnostics["baseline_mixed"]["sessions_with_terminal_idle_dwell_0_4s"])
return checks
def workflow(args, prepared):
from scripts.infer.third_party_metrics import aggregate
context = prepared["args"]
context.run_dir.mkdir(parents=True, exist_ok=True)
def phase(name, **fields):
write_json(context.run_dir / "workflow_state.json", {"phase": name, "time": time.time(), **fields})
write_json(context.run_dir / "identity.json", {**prepared["identity"],
"verification_entry_sha256": repair.digest(Path(__file__).resolve()),
"source_run": str(prepared["source_run"]), "source_hashes": prepared["source_hashes"],
"candidate": args.candidate, "adapter": str(prepared["adapter"]),
"fitting_performed": False, "feature_extraction_performed": False})
write_json(context.run_dir / "selection.json", {"fixed_sixty_ids": prepared["sixty"], "paired_twentyfour_ids": prepared["panel"]})
phase("verify_mixed_closed_loop", candidate=args.candidate, count=60)
mixed = repair.parallel(context, prepared["identity"], prepared["sixty"], "candidate_mixed", adapter=prepared["adapter"])
phase("verify_clean_closed_loop", candidate=args.candidate, count=24)
clean = repair.parallel(context, prepared["identity"], prepared["panel"], "candidate_clean", view="clean", adapter=prepared["adapter"])
rows = {"baseline_mixed": prepared["reports"]["baseline_mixed"]["samples"],
"baseline_clean": prepared["reports"]["baseline_clean"]["samples"],
"candidate_mixed": mixed, "candidate_clean": clean}
metrics = {name: aggregate(values) for name, values in rows.items()}
diagnostics = {name: repair.summarize_diagnostics(values) for name, values in rows.items()}
checks = acceptance_checks(metrics["baseline_mixed"], metrics["candidate_mixed"],
metrics["baseline_clean"], metrics["candidate_clean"], diagnostics)
for path, expected in prepared["source_hashes"].items():
if repair.digest(path) != expected:
raise ValueError(f"Immutable verification source changed during execution: {path}")
write_json(context.run_dir / "result.json", {"status": "complete", "candidate": args.candidate,
"adapter": str(prepared["adapter"]), "not_full_state_resume_checkpoint": True,
"promoted": False, "automatic_scale_up": False, "old_weights_modified": False,
"fitting_performed": False, "feature_extraction_performed": False,
"pilot_acceptance_passed": all(checks.values()), "acceptance_checks": checks,
"diagnostic_summary": diagnostics, **metrics,
"source_report_hashes": prepared["source_hashes"],
"scope": "Existing adapter only; original fixed train60 mixed + paired train24 clean, unchanged per-session seeds; not held-out generalization",
"acceptance_policy": "mixed background improves; mixed/clean accuracy drop<=1pp, missed motion rise<=2pp, joint MAE rise<=10%; mixed terminal idle not fewer"})
phase("completed", candidate=args.candidate, pilot_acceptance_passed=all(checks.values()))
def build_launch_spec(args, prepared):
"""Bind both managed commands to the exact accepted manifest explicitly."""
manifest = prepared["args"].manifest
argv = [sys.executable, str(Path(__file__).resolve()), "--source-run", str(args.source_run),
"--candidate", args.candidate, "--run-dir", str(args.run_dir), "--manifest", str(manifest)]
return {"schema": 1, "project_root": str(PROJECT), "gpu": 3, "run_dir": str(args.run_dir),
"manifest_path": str(manifest), "readiness_file": str(manifest.parent / "acceptance.json"),
"argv": [*argv, "--worker"], "preflight_command": [*argv, "--check-only"]}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--source-run", type=Path, required=True)
parser.add_argument("--candidate", choices=CANDIDATES, required=True)
parser.add_argument("--run-dir", type=Path, required=True)
parser.add_argument("--manifest", type=Path, help="Optional explicit binding; must equal the source frozen manifest")
modes = parser.add_mutually_exclusive_group(required=True)
for name in ("check-only", "launch", "worker"):
modes.add_argument("--" + name, action="store_true")
args = parser.parse_args()
args.source_run = repair.local(args.source_run, output=True)
args.run_dir = repair.local(args.run_dir, output=True)
prepared = prepare_verification(args)
if args.check_only:
print(json.dumps({"passed": True, "candidate": args.candidate, "mixed_count": 60,
"clean_count": 24, "fits_model": False, "adapter": str(prepared["adapter"])}))
return
from scripts.train_third_party_strategy import setup_environment
if args.launch:
if args.run_dir.exists():
raise FileExistsError("Refusing to overwrite an existing experiment")
setup_environment()
spec = build_launch_spec(args, prepared)
path = PROJECT / "tmp" / f"{args.run_dir.name}.launch.json"
if path.exists():
raise FileExistsError(path)
write_json(path, spec)
subprocess.run([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(path)], check=True)
else:
if os.environ.get("CUDA_VISIBLE_DEVICES") != "3" or os.environ.get("HFD_NPROC_PER_NODE") != "1":
raise RuntimeError("Use the managed GPU-3 launcher")
setup_environment()
workflow(args, prepared)
if __name__ == "__main__":
main()