Download scripts/verify_third_party_control_adapter.py from Tinghong-Ye22/code-HFD_lzk: direct link, hf CLI and curl.
- Browser
- Download file 15.8 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/verify_third_party_control_adapter.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD_lzk/scripts/verify_third_party_control_adapter.py
-
curl -L -o verify_third_party_control_adapter.py https://huggingface.co/Tinghong-Ye22/code-HFD_lzk/resolve/main/scripts/verify_third_party_control_adapter.py
15.8 kB
| #!/usr/bin/env python3 | |
| """Independently verify an already fitted control adapter; never fit a model. | |
| Only completed, hash-bound repair runs may supply adapters and paired baselines. | |
| The same frozen train60/train24 receive new raw-audio closed-loop predictions. | |
| An adapter is not a full-state checkpoint and is never automatically promoted. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import subprocess | |
| import sys | |
| import time | |
| from types import SimpleNamespace | |
| PROJECT = Path(__file__).resolve().parents[1] | |
| sys.path[:0] = [str(PROJECT), str(PROJECT / "src")] | |
| from scripts import train_third_party_control_repair as repair | |
| from scripts.third_party_strategy_plan import write_json | |
| CANDIDATES = ("uniform_ce", "balanced_ce") | |
| SOURCE_CODE = ("scripts/train_third_party_control_repair.py", | |
| "scripts/third_party_control_refit.py", | |
| "scripts/third_party_control_diagnostics.py", | |
| "scripts/third_party_clean_view.py") | |
| def _read(path): | |
| return json.loads(Path(path).read_text()) | |
| def _completed(path): | |
| status = _read(path / "run_status.json") | |
| if (status.get("phase") != "completed" or status.get("training_returncode") != 0 | |
| or status.get("launcher_returncode") != 0 or status.get("keeper_restore_verified") is not True): | |
| raise ValueError("Source/base run must be completed successfully with keeper restored") | |
| def _report(source, name, ids, seeds, *, view=None, adapter=None): | |
| path = source / ("baseline_mixed.json" if name == "baseline_mixed" else f"{name}/report.json") | |
| report = _read(path) | |
| if report.get("count") != len(ids) or [r["session_id"] for r in report.get("samples", [])] != ids: | |
| raise ValueError(f"Incomplete or different fixed selection: {name}") | |
| if name != "baseline_mixed": | |
| if (report.get("status") != "complete" or report.get("split") != "train_only" | |
| or report.get("view") != view or report.get("adapter") != adapter): | |
| raise ValueError(f"Incomplete/nonmatching source report: {name}") | |
| for row in report["samples"]: | |
| sid = row["session_id"] | |
| if row.get("seed") != seeds[sid] or "metrics" not in row or "diagnostics" not in row: | |
| raise ValueError(f"Unpaired or incomplete source sample: {name}/{sid}") | |
| if name == "baseline_mixed": | |
| continue | |
| directory = repair.local(row["prediction_directory"], output=True) | |
| if directory != source / name / "samples" / sid: | |
| raise ValueError("Source prediction path differs from its report") | |
| for filename in ("motion.npy", "infer_meta.json", "session_recipe.json", "result.json"): | |
| if not (directory / filename).is_file(): | |
| raise ValueError(f"Missing completed prediction artifact: {name}/{sid}/{filename}") | |
| if _read(directory / "result.json") != row: | |
| raise ValueError("Source report differs from completed sample result") | |
| metadata = _read(directory / "infer_meta.json") | |
| if (metadata.get("session_id") != sid or metadata.get("audio_view") != view | |
| or metadata.get("control_adapter") != adapter): | |
| raise ValueError("Source prediction metadata differs from its report") | |
| return report, path | |
| def prepare_verification(args): | |
| """Read-only complete provenance gate, including current source code hashes.""" | |
| if args.candidate not in CANDIDATES: | |
| raise ValueError("Unknown fitted control candidate") | |
| source = repair.local(args.source_run, output=True) | |
| output = repair.local(args.run_dir, output=True) | |
| _completed(source) | |
| identity = _read(source / "identity.json") | |
| base = repair.local(identity["base_run"], output=True) | |
| if output.is_relative_to(source) or output.is_relative_to(base) or source.is_relative_to(output) or base.is_relative_to(output): | |
| raise ValueError("Use a fresh independent output run, not the immutable source/base") | |
| _completed(base) | |
| cfg = _read(base / "run_config.json") | |
| manifest = repair.local(cfg["data"]["frozen_manifest"]) | |
| if getattr(args, "manifest", None) is not None and repair.local(args.manifest) != manifest: | |
| raise ValueError("Explicit --manifest differs from the source frozen manifest") | |
| step = int(identity["base_step"]) | |
| checkpoint = base / "checkpoints" / f"ckpt_{step}" | |
| expected = {"base_weights_sha256": repair.digest(checkpoint / "model.safetensors"), | |
| "base_config_sha256": repair.digest(base / "run_config.json"), | |
| "manifest_sha256": repair.digest(manifest)} | |
| if any(identity.get(k) != v for k, v in expected.items()): | |
| raise ValueError("Source identity no longer matches base weights/config/manifest") | |
| if (identity.get("training_count") != 300 or any(identity.get(k) != 0 for k in | |
| ("heldout_inputs", "visual_inputs", "other_task_inputs"))): | |
| raise ValueError("Source repair scope was not the frozen audio-only train300") | |
| code = identity.get("code_sha256", {}) | |
| if set(code) != set(SOURCE_CODE) or any(repair.digest(PROJECT / name) != code[name] for name in SOURCE_CODE): | |
| raise ValueError("Source repair code changed; do not silently reuse its feature/clean policy") | |
| repair_args = SimpleNamespace(base_run=base, step=step, manifest=manifest, run_dir=output) | |
| selection, previous, sixty, panel = repair.prepare(repair_args) | |
| chosen = _read(source / "selection.json") | |
| if chosen.get("fixed_sixty_ids") != sixty or chosen.get("paired_twentyfour_ids") != panel: | |
| raise ValueError("Source fixed60/paired24 selection differs from frozen training selection") | |
| source_result = _read(source / "result.json") | |
| if (source_result.get("status") != "complete" or source_result.get("candidate") not in CANDIDATES | |
| or source_result.get("promoted") is not False or source_result.get("old_weights_modified") is not False | |
| or _read(source / "workflow_state.json").get("phase") != "completed"): | |
| raise ValueError("Source repair has not finished its complete paired candidate verification") | |
| fit = _read(source / "fit_report.json") | |
| if args.candidate not in fit.get("candidates", {}): | |
| raise ValueError("Requested adapter was not fitted by this completed repair") | |
| adapter = source / "adapters" / f"{args.candidate}.safetensors" | |
| adapter_record = _read(adapter.with_suffix(".json")) | |
| if (adapter_record.get("format") != "hfd_control_head_adapter_v1" | |
| or adapter_record.get("candidate") != args.candidate | |
| or adapter_record.get("not_full_state_resume_checkpoint") is not True | |
| or adapter_record.get("selection_scope") != "train_only" | |
| or adapter_record.get("base_checkpoint") != str(checkpoint) | |
| or adapter_record.get("code_sha256") != code | |
| or adapter_record.get("fit_config") != fit.get("config") | |
| or any(adapter_record.get(k) != identity[k] for k in | |
| ("base_run", "base_step", "training_count", "heldout_inputs", "visual_inputs", "other_task_inputs")) | |
| or any(adapter_record.get(k) != v for k, v in expected.items()) | |
| or repair.digest(adapter) != adapter_record.get("adapter_sha256")): | |
| raise ValueError("Adapter hash/provenance does not bind this immutable base") | |
| from safetensors.torch import load_file | |
| state = load_file(str(adapter), device="cpu") | |
| repair.build_head(state) | |
| if (state["1.weight"].shape != (cfg["model"]["streaming_memory_dim"], cfg["model"]["view_feature_dim"])): | |
| raise ValueError("Adapter dimensions do not match base control_head") | |
| seeds = {r["session_id"]: r["seed"] for r in previous["samples"]} | |
| selected_adapter = str(source / "adapters" / f"{source_result['candidate']}.safetensors") | |
| if source_result.get("adapter") != selected_adapter: | |
| raise ValueError("Source completed candidate identity is inconsistent") | |
| reports, paths = {}, [] | |
| for name, ids, view, used_adapter in ( | |
| ("baseline_mixed", sixty, None, None), ("baseline_clean", panel, "clean", None), | |
| ("candidate_mixed", sixty, "mixed", selected_adapter), | |
| ("candidate_clean", panel, "clean", selected_adapter)): | |
| reports[name], path = _report(source, name, ids, seeds, view=view, adapter=used_adapter) | |
| paths.append(path) | |
| paths.extend(source / name for name in ("identity.json", "selection.json", "fit_report.json", "result.json", "workflow_state.json", "run_status.json")) | |
| paths.extend((adapter, adapter.with_suffix(".json"), checkpoint / "model.safetensors", base / "run_config.json", manifest)) | |
| paths.extend(PROJECT / name for name in SOURCE_CODE) | |
| hashes = {str(path): repair.digest(path) for path in paths} | |
| return {"args": repair_args, "identity": identity, "adapter": adapter, | |
| "sixty": sixty, "panel": panel, "reports": reports, "source_hashes": hashes, | |
| "source_run": source, "candidate": args.candidate} | |
| def acceptance_checks(baseline, candidate, clean_baseline, clean_candidate, diagnostics): | |
| def number(value): | |
| return value is not None and math.isfinite(float(value)) | |
| def compare(left, right, fn): | |
| return bool(number(left) and number(right) and fn(float(left), float(right))) | |
| checks = {"mixed_background_improves": compare(candidate["background_only_false_motion_rate"], baseline["background_only_false_motion_rate"], lambda a, b: a < b)} | |
| for name, old, new in (("mixed", baseline, candidate), ("clean", clean_baseline, clean_candidate)): | |
| checks[name + "_accuracy_drop_le_1pp"] = compare(new["control_accuracy"], old["control_accuracy"], lambda a, b: a >= b - .01) | |
| checks[name + "_missed_motion_rise_le_2pp"] = compare(new["missed_motion_rate"], old["missed_motion_rate"], lambda a, b: a <= b + .02) | |
| checks[name + "_joint_mae_rise_le_10pct"] = compare(new["joint_mae_radians"], old["joint_mae_radians"], lambda a, b: a <= b * 1.10) | |
| checks["mixed_terminal_idle_not_fewer"] = (diagnostics["candidate_mixed"]["sessions_with_terminal_idle_dwell_0_4s"] >= diagnostics["baseline_mixed"]["sessions_with_terminal_idle_dwell_0_4s"]) | |
| return checks | |
| def workflow(args, prepared): | |
| from scripts.infer.third_party_metrics import aggregate | |
| context = prepared["args"] | |
| context.run_dir.mkdir(parents=True, exist_ok=True) | |
| def phase(name, **fields): | |
| write_json(context.run_dir / "workflow_state.json", {"phase": name, "time": time.time(), **fields}) | |
| write_json(context.run_dir / "identity.json", {**prepared["identity"], | |
| "verification_entry_sha256": repair.digest(Path(__file__).resolve()), | |
| "source_run": str(prepared["source_run"]), "source_hashes": prepared["source_hashes"], | |
| "candidate": args.candidate, "adapter": str(prepared["adapter"]), | |
| "fitting_performed": False, "feature_extraction_performed": False}) | |
| write_json(context.run_dir / "selection.json", {"fixed_sixty_ids": prepared["sixty"], "paired_twentyfour_ids": prepared["panel"]}) | |
| phase("verify_mixed_closed_loop", candidate=args.candidate, count=60) | |
| mixed = repair.parallel(context, prepared["identity"], prepared["sixty"], "candidate_mixed", adapter=prepared["adapter"]) | |
| phase("verify_clean_closed_loop", candidate=args.candidate, count=24) | |
| clean = repair.parallel(context, prepared["identity"], prepared["panel"], "candidate_clean", view="clean", adapter=prepared["adapter"]) | |
| rows = {"baseline_mixed": prepared["reports"]["baseline_mixed"]["samples"], | |
| "baseline_clean": prepared["reports"]["baseline_clean"]["samples"], | |
| "candidate_mixed": mixed, "candidate_clean": clean} | |
| metrics = {name: aggregate(values) for name, values in rows.items()} | |
| diagnostics = {name: repair.summarize_diagnostics(values) for name, values in rows.items()} | |
| checks = acceptance_checks(metrics["baseline_mixed"], metrics["candidate_mixed"], | |
| metrics["baseline_clean"], metrics["candidate_clean"], diagnostics) | |
| for path, expected in prepared["source_hashes"].items(): | |
| if repair.digest(path) != expected: | |
| raise ValueError(f"Immutable verification source changed during execution: {path}") | |
| write_json(context.run_dir / "result.json", {"status": "complete", "candidate": args.candidate, | |
| "adapter": str(prepared["adapter"]), "not_full_state_resume_checkpoint": True, | |
| "promoted": False, "automatic_scale_up": False, "old_weights_modified": False, | |
| "fitting_performed": False, "feature_extraction_performed": False, | |
| "pilot_acceptance_passed": all(checks.values()), "acceptance_checks": checks, | |
| "diagnostic_summary": diagnostics, **metrics, | |
| "source_report_hashes": prepared["source_hashes"], | |
| "scope": "Existing adapter only; original fixed train60 mixed + paired train24 clean, unchanged per-session seeds; not held-out generalization", | |
| "acceptance_policy": "mixed background improves; mixed/clean accuracy drop<=1pp, missed motion rise<=2pp, joint MAE rise<=10%; mixed terminal idle not fewer"}) | |
| phase("completed", candidate=args.candidate, pilot_acceptance_passed=all(checks.values())) | |
| def build_launch_spec(args, prepared): | |
| """Bind both managed commands to the exact accepted manifest explicitly.""" | |
| manifest = prepared["args"].manifest | |
| argv = [sys.executable, str(Path(__file__).resolve()), "--source-run", str(args.source_run), | |
| "--candidate", args.candidate, "--run-dir", str(args.run_dir), "--manifest", str(manifest)] | |
| return {"schema": 1, "project_root": str(PROJECT), "gpu": 3, "run_dir": str(args.run_dir), | |
| "manifest_path": str(manifest), "readiness_file": str(manifest.parent / "acceptance.json"), | |
| "argv": [*argv, "--worker"], "preflight_command": [*argv, "--check-only"]} | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--source-run", type=Path, required=True) | |
| parser.add_argument("--candidate", choices=CANDIDATES, required=True) | |
| parser.add_argument("--run-dir", type=Path, required=True) | |
| parser.add_argument("--manifest", type=Path, help="Optional explicit binding; must equal the source frozen manifest") | |
| modes = parser.add_mutually_exclusive_group(required=True) | |
| for name in ("check-only", "launch", "worker"): | |
| modes.add_argument("--" + name, action="store_true") | |
| args = parser.parse_args() | |
| args.source_run = repair.local(args.source_run, output=True) | |
| args.run_dir = repair.local(args.run_dir, output=True) | |
| prepared = prepare_verification(args) | |
| if args.check_only: | |
| print(json.dumps({"passed": True, "candidate": args.candidate, "mixed_count": 60, | |
| "clean_count": 24, "fits_model": False, "adapter": str(prepared["adapter"])})) | |
| return | |
| from scripts.train_third_party_strategy import setup_environment | |
| if args.launch: | |
| if args.run_dir.exists(): | |
| raise FileExistsError("Refusing to overwrite an existing experiment") | |
| setup_environment() | |
| spec = build_launch_spec(args, prepared) | |
| path = PROJECT / "tmp" / f"{args.run_dir.name}.launch.json" | |
| if path.exists(): | |
| raise FileExistsError(path) | |
| write_json(path, spec) | |
| subprocess.run([sys.executable, str(PROJECT / "scripts/train/third_party_run.py"), "start", "--spec", str(path)], check=True) | |
| else: | |
| if os.environ.get("CUDA_VISIBLE_DEVICES") != "3" or os.environ.get("HFD_NPROC_PER_NODE") != "1": | |
| raise RuntimeError("Use the managed GPU-3 launcher") | |
| setup_environment() | |
| workflow(args, prepared) | |
| if __name__ == "__main__": | |
| main() | |