"""Reopen delivered pixels/traces and build skill receipts from actual runs.""" import argparse import hashlib import json import statistics from pathlib import Path import numpy as np from PIL import Image from experiments.r3.trace_audit import payloads from h3_mlx.compare import _verify_identity, image_metrics def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() def canonical(value): return hashlib.sha256( json.dumps( value, sort_keys=True, separators=(",", ":"), ensure_ascii=False ).encode() ).hexdigest() def main(a): root = Path(a.root) destination = Path(a.output) destination.mkdir(parents=True, exist_ok=True) contract = json.loads(Path(a.contract).read_text()) manifest = json.loads(Path(a.workload).read_text()) rows = [json.loads(line) for line in (root / "runs.jsonl").read_text().splitlines()] checks = [] anchors = {} for case in contract["quality_cases"]: baseline = next( r for r in rows if r["case"] == case and r["variant"] == "control" ) anchors[case] = baseline trace_anchors = { case: payloads(Path(row["output"]).with_suffix(".trace.safetensors")) for case, row in anchors.items() } for row in rows: path = Path(row["output"]) image = Image.open(path).convert("RGB") actual = hashlib.sha256(np.asarray(image).tobytes()).hexdigest() if actual != row["output_sha256"]: raise ValueError(f"Output changed: {path}") reference = Path(anchors[row["case"]]["output"]) _verify_identity(reference, path) metrics = image_metrics(Image.open(reference), image) trace = payloads(path.with_suffix(".trace.safetensors")) checks.append( { "output": str(path), "case": row["case"], "variant": row["variant"], "phase": row["phase"], "metrics": metrics, "trace_identical": trace == trace_anchors[row["case"]], "different_trace_tensors": [ k for k in trace if trace.get(k) != trace_anchors[row["case"]].get(k) ], } ) (destination / "quality-audit.json").write_text(json.dumps(checks, indent=2) + "\n") timed = [r for r in rows if r["phase"] == "timed"] summary = { "root": str(root), "cases": {}, "all_pixel_identical": all(c["metrics"]["pixel_identical"] for c in checks), "all_trace_identical": all(c["trace_identical"] for c in checks), "scope": "Paired warm complete generation including encoding-cache checks, six heads, decode, trace/PNG save. Parent physical footprint sampled every 100ms; both compiled variants can be resident.", } for case in contract["timing_cases"]: summary["cases"][case] = {} for variant in ("control", a.variant): rr = [r for r in timed if r["case"] == case and r["variant"] == variant] values = [r["seconds"] for r in rr] summary["cases"][case][variant] = { "samples_s": values, "median_s": statistics.median(values), "min_s": min(values), "max_s": max(values), "peak_physical_bytes": max(r["peak_memory_bytes"] for r in rr), "peak_mlx_bytes": max(r["mlx_peak_bytes"] for r in rr), } control = summary["cases"][case]["control"] candidate = summary["cases"][case][a.variant] summary["cases"][case]["reduction_pct"] = 100 * ( 1 - candidate["median_s"] / control["median_s"] ) (destination / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") if a.no_receipts: print(json.dumps(summary, indent=2)) return context = { "hardware": "Apple M5 Pro, 16 GPU cores, 64 GiB unified memory", "os": "macOS 27.0.1 arm64", "software": "MLX/MLX Metal 0.32.4; Python 3.12.14; strict precision environment", "environment": "AC Automatic power; MLX_ENABLE_TF32=0, MLX_CONV_WINOGRAD=0; user background apps unchanged; no thermal warning recorded", "timing_boundary": "Pipeline.generate total_s: warm text cache, original CPU RNG, six heads, decode, trace and PNG save. Excludes interpreter startup and external quality scoring.", "memory_metric": "Parent macOS physical footprint sampled at 100ms, current-interval maximum; includes Metal, excludes compiler XPC. MLX allocator separately; do not sum. Both compiled variants may be resident.", "cache_state": "Resident shared native weights and VAE; shape warmup for each variant. Exact conditioning caches warmed outside timed population.", "execution_policy": "Single GPU benchmark process; per-case repeats alternate control/candidate then candidate/control then control/candidate. Warmup and first-load excluded from timed population.", } code_hash = sha(root / "source-manifest.json") for variant in ("control", a.variant): cfg = dict(manifest["control_config"]) if variant != "control": cfg.update( kernel="metal-qk-exact" if "native-norm" in variant else "metal-exact", tune_matmul=True, ) description = { "id": variant, "code_sha256": code_hash, "model_sha256": sha("provenance/sources.json"), "config_sha256": canonical(cfg), "command": [ ".venv/bin/python", "-m", "experiments.r3.images", "--deployment", "--variants", a.variant, "--workload", a.workload, "--model", str(Path("../minimax-h3-image-opt/models/MiniMax-H3-Image").resolve()), "--text-encoder", str( Path( "../minimax-h3-image-opt/models/MiniMax-H3/text_encoder" ).resolve() ), "--output", str(root), ], } output_hashes = {} for case in contract["quality_cases"]: values = { r["output_sha256"] for r in rows if r["case"] == case and r["variant"] == variant } if len(values) != 1: raise ValueError( f"Missing or nondeterministic {case}/{variant}: {values}" ) output_hashes[case] = values.pop() relevant = [c for c in checks if c["variant"] == variant] receipt = { "schema_version": 1, "id": f"r3-deployment-{variant}", "campaign_id": contract["campaign_id"], "contract_sha256": canonical(contract), "complete": True, "change_class": "control" if variant == "control" else "exact", "hypothesis": "Native reduction matching QK/RoPE fusion and shape-qualified Metal 4 FC2 tiling reduce full generation latency without changing a denoising tensor or pixel.", "context": context, "variant": description, "quality": { "variant_sha256": canonical(description), "reference_sha256": contract["reference_sha256"], "evaluator_sha256": contract["evaluator_sha256"], "outputs": output_hashes, "metrics": { "max_abs_error": max( c["metrics"]["max_error_8bit"] for c in relevant ), "nonfinite_values": sum( r["nonfinite_values"] for r in rows if r["variant"] == variant ), }, }, "runs": [ { k: r[k] for k in ( "case", "repeat", "seconds", "peak_memory_bytes", "output_sha256", ) } for r in timed if r["variant"] == variant ], "evidence": [ {"path": str(path), "sha256": sha(path)} for path in ( root / "runs.jsonl", root / "source-manifest.json", destination / "quality-audit.json", destination / "summary.json", ) ], "extensions": { "canonical_output": "SHA256 decoded RGB uint8 bytes in row-major order", "model_identity": "model_sha256 is the pinned model/source provenance manifest hash; no new full checkpoint rehash", "reference_scope": "Incremental identity to MLX BF16 recipe, not CUDA or unpublished FP32 training weights", "trace_identical": all(c["trace_identical"] for c in relevant), }, } (destination / f"{variant}.receipt.json").write_text( json.dumps(receipt, indent=2) + "\n" ) print(json.dumps(summary, indent=2)) if __name__ == "__main__": p = argparse.ArgumentParser(description=__doc__) p.add_argument("--root", default="outputs/optimization-r3-deployment") p.add_argument("--variant", default="qk-fc2") p.add_argument("--contract", default="optimization/round-03/contract.json") p.add_argument("--workload", default="optimization/round-03/workload.json") p.add_argument( "--no-receipts", action="store_true", help="Audit research outputs without assigning release source identity", ) p.add_argument("--output", default="reports/optimization-r3/deployment") main(p.parse_args())