devin-lai's picture
Release v0.4.0: selective MLX model downloads
3da88e3 verified
Raw History Blame Contribute Delete
9.99 kB
"""Reopen delivered pixels/traces and build skill receipts from actual runs."""
import argparse
import hashlib
import json
import statistics
from pathlib import Path
import numpy as np
from PIL import Image
from experiments.r3.trace_audit import payloads
from h3_mlx.compare import _verify_identity, image_metrics
def sha(path):
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
def canonical(value):
return hashlib.sha256(
json.dumps(
value, sort_keys=True, separators=(",", ":"), ensure_ascii=False
).encode()
).hexdigest()
def main(a):
root = Path(a.root)
destination = Path(a.output)
destination.mkdir(parents=True, exist_ok=True)
contract = json.loads(Path(a.contract).read_text())
manifest = json.loads(Path(a.workload).read_text())
rows = [json.loads(line) for line in (root / "runs.jsonl").read_text().splitlines()]
checks = []
anchors = {}
for case in contract["quality_cases"]:
baseline = next(
r for r in rows if r["case"] == case and r["variant"] == "control"
)
anchors[case] = baseline
trace_anchors = {
case: payloads(Path(row["output"]).with_suffix(".trace.safetensors"))
for case, row in anchors.items()
}
for row in rows:
path = Path(row["output"])
image = Image.open(path).convert("RGB")
actual = hashlib.sha256(np.asarray(image).tobytes()).hexdigest()
if actual != row["output_sha256"]:
raise ValueError(f"Output changed: {path}")
reference = Path(anchors[row["case"]]["output"])
_verify_identity(reference, path)
metrics = image_metrics(Image.open(reference), image)
trace = payloads(path.with_suffix(".trace.safetensors"))
checks.append(
{
"output": str(path),
"case": row["case"],
"variant": row["variant"],
"phase": row["phase"],
"metrics": metrics,
"trace_identical": trace == trace_anchors[row["case"]],
"different_trace_tensors": [
k
for k in trace
if trace.get(k) != trace_anchors[row["case"]].get(k)
],
}
)
(destination / "quality-audit.json").write_text(json.dumps(checks, indent=2) + "\n")
timed = [r for r in rows if r["phase"] == "timed"]
summary = {
"root": str(root),
"cases": {},
"all_pixel_identical": all(c["metrics"]["pixel_identical"] for c in checks),
"all_trace_identical": all(c["trace_identical"] for c in checks),
"scope": "Paired warm complete generation including encoding-cache checks, six heads, decode, trace/PNG save. Parent physical footprint sampled every 100ms; both compiled variants can be resident.",
}
for case in contract["timing_cases"]:
summary["cases"][case] = {}
for variant in ("control", a.variant):
rr = [r for r in timed if r["case"] == case and r["variant"] == variant]
values = [r["seconds"] for r in rr]
summary["cases"][case][variant] = {
"samples_s": values,
"median_s": statistics.median(values),
"min_s": min(values),
"max_s": max(values),
"peak_physical_bytes": max(r["peak_memory_bytes"] for r in rr),
"peak_mlx_bytes": max(r["mlx_peak_bytes"] for r in rr),
}
control = summary["cases"][case]["control"]
candidate = summary["cases"][case][a.variant]
summary["cases"][case]["reduction_pct"] = 100 * (
1 - candidate["median_s"] / control["median_s"]
)
(destination / "summary.json").write_text(json.dumps(summary, indent=2) + "\n")
if a.no_receipts:
print(json.dumps(summary, indent=2))
return
context = {
"hardware": "Apple M5 Pro, 16 GPU cores, 64 GiB unified memory",
"os": "macOS 27.0.1 arm64",
"software": "MLX/MLX Metal 0.32.4; Python 3.12.14; strict precision environment",
"environment": "AC Automatic power; MLX_ENABLE_TF32=0, MLX_CONV_WINOGRAD=0; user background apps unchanged; no thermal warning recorded",
"timing_boundary": "Pipeline.generate total_s: warm text cache, original CPU RNG, six heads, decode, trace and PNG save. Excludes interpreter startup and external quality scoring.",
"memory_metric": "Parent macOS physical footprint sampled at 100ms, current-interval maximum; includes Metal, excludes compiler XPC. MLX allocator separately; do not sum. Both compiled variants may be resident.",
"cache_state": "Resident shared native weights and VAE; shape warmup for each variant. Exact conditioning caches warmed outside timed population.",
"execution_policy": "Single GPU benchmark process; per-case repeats alternate control/candidate then candidate/control then control/candidate. Warmup and first-load excluded from timed population.",
}
code_hash = sha(root / "source-manifest.json")
for variant in ("control", a.variant):
cfg = dict(manifest["control_config"])
if variant != "control":
cfg.update(
kernel="metal-qk-exact" if "native-norm" in variant else "metal-exact",
tune_matmul=True,
)
description = {
"id": variant,
"code_sha256": code_hash,
"model_sha256": sha("provenance/sources.json"),
"config_sha256": canonical(cfg),
"command": [
".venv/bin/python",
"-m",
"experiments.r3.images",
"--deployment",
"--variants",
a.variant,
"--workload",
a.workload,
"--model",
str(Path("../minimax-h3-image-opt/models/MiniMax-H3-Image").resolve()),
"--text-encoder",
str(
Path(
"../minimax-h3-image-opt/models/MiniMax-H3/text_encoder"
).resolve()
),
"--output",
str(root),
],
}
output_hashes = {}
for case in contract["quality_cases"]:
values = {
r["output_sha256"]
for r in rows
if r["case"] == case and r["variant"] == variant
}
if len(values) != 1:
raise ValueError(
f"Missing or nondeterministic {case}/{variant}: {values}"
)
output_hashes[case] = values.pop()
relevant = [c for c in checks if c["variant"] == variant]
receipt = {
"schema_version": 1,
"id": f"r3-deployment-{variant}",
"campaign_id": contract["campaign_id"],
"contract_sha256": canonical(contract),
"complete": True,
"change_class": "control" if variant == "control" else "exact",
"hypothesis": "Native reduction matching QK/RoPE fusion and shape-qualified Metal 4 FC2 tiling reduce full generation latency without changing a denoising tensor or pixel.",
"context": context,
"variant": description,
"quality": {
"variant_sha256": canonical(description),
"reference_sha256": contract["reference_sha256"],
"evaluator_sha256": contract["evaluator_sha256"],
"outputs": output_hashes,
"metrics": {
"max_abs_error": max(
c["metrics"]["max_error_8bit"] for c in relevant
),
"nonfinite_values": sum(
r["nonfinite_values"] for r in rows if r["variant"] == variant
),
},
},
"runs": [
{
k: r[k]
for k in (
"case",
"repeat",
"seconds",
"peak_memory_bytes",
"output_sha256",
)
}
for r in timed
if r["variant"] == variant
],
"evidence": [
{"path": str(path), "sha256": sha(path)}
for path in (
root / "runs.jsonl",
root / "source-manifest.json",
destination / "quality-audit.json",
destination / "summary.json",
)
],
"extensions": {
"canonical_output": "SHA256 decoded RGB uint8 bytes in row-major order",
"model_identity": "model_sha256 is the pinned model/source provenance manifest hash; no new full checkpoint rehash",
"reference_scope": "Incremental identity to MLX BF16 recipe, not CUDA or unpublished FP32 training weights",
"trace_identical": all(c["trace_identical"] for c in relevant),
},
}
(destination / f"{variant}.receipt.json").write_text(
json.dumps(receipt, indent=2) + "\n"
)
print(json.dumps(summary, indent=2))
if __name__ == "__main__":
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--root", default="outputs/optimization-r3-deployment")
p.add_argument("--variant", default="qk-fc2")
p.add_argument("--contract", default="optimization/round-03/contract.json")
p.add_argument("--workload", default="optimization/round-03/workload.json")
p.add_argument(
"--no-receipts",
action="store_true",
help="Audit research outputs without assigning release source identity",
)
p.add_argument("--output", default="reports/optimization-r3/deployment")
main(p.parse_args())