timestral-coreai / scripts /summarize_validation.py
jbilcke's picture
Publish Timestral CoreAI models, native Swift SDK and validated Python runtime
506f40d verified
Raw History Blame Contribute Delete
10.1 kB
"""Create a model catalog only after all four current bundles pass full native CPU validation."""
from __future__ import annotations
import argparse
from datetime import datetime
import json
import math
from pathlib import Path, PurePosixPath
import sys
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from scripts.validate import (
ACCEPTANCE, FIXTURE_NAMES, VARIANTS, atomic_json, code_hashes, fixtures,
load_reference_cache, provenance_snapshot, reference_provenance, sha256,
)
def require(condition, message):
if not condition:
raise ValueError(message)
def verify_files(bundle, manifest):
files = manifest.get("files", {})
require(isinstance(files, dict) and files, "bundle has no file inventory")
asset_bytes, asset_roots = 0, set()
for name, expected in files.items():
relative = PurePosixPath(name)
path = bundle / name
require(not relative.is_absolute() and ".." not in relative.parts and str(relative) == name,
f"unsafe bundle path: {name}")
require(path.is_file() and not path.is_symlink() and path.resolve().is_relative_to(bundle.resolve()),
f"missing or unsafe bundle file: {name}")
require(path.stat().st_size == expected["bytes"] and sha256(path) == expected["sha256"],
f"bundle file checksum mismatch: {name}")
if relative.parts[0].endswith(".aimodel"):
asset_roots.add(relative.parts[0])
asset_bytes += path.stat().st_size
require(asset_roots, "bundle contains no CoreAI assets")
for asset in asset_roots:
actual = {p.relative_to(bundle).as_posix() for p in (bundle / asset).rglob("*") if p.is_file()}
expected = {name for name in files if name.startswith(asset + "/")}
require(actual == expected, f"unverified files inside {asset}")
return asset_bytes, len(asset_roots), sum(record["bytes"] for record in files.values()) + (bundle / "manifest.json").stat().st_size
def check_metrics(metrics, label):
require(metrics.get("passed") and metrics.get("finite") and metrics.get("monotone"),
f"{label}: failed, nonfinite, or unordered quantiles")
for key, maximum in (("normalized_rmse", ACCEPTANCE["normalized_rmse_max"]),
("normalized_max_error", ACCEPTANCE["normalized_max_error_max"])):
value = metrics.get(key)
require(isinstance(value, (int, float)) and math.isfinite(value) and 0 <= value <= maximum,
f"{label}: {key} does not meet fixed acceptance")
def worst_metrics(records):
psnr = [item["psnr_db"] for item in records if item.get("psnr_db") is not None]
return dict(
normalized_rmse=max(item["normalized_rmse"] for item in records),
normalized_max_error=max(item["normalized_max_error"] for item in records),
max_absolute_error=max(item["max_absolute_error"] for item in records),
minimum_psnr_db=min(psnr) if psnr else None,
)
def check_variant(variant, root, bundles, reports, source, cache, upstream):
path = reports / f"{variant}.json"
report = json.loads(path.read_text())
require(report.get("variant") == variant, "report variant mismatch")
require(report.get("status") == "completed" and report.get("passed")
and report.get("extended_suite_passed"), "report is not a completed, accepted extended suite")
require(report.get("expected_fixture_count") == 9 and report.get("completed_fixture_count") == 9
and report.get("remaining_fixtures") == [], "report did not complete all nine fixtures")
names = report.get("requested_fixtures", [])
records = report.get("fixtures", [])
require(len(names) == 9 and set(names) == set(FIXTURE_NAMES), "report requested a subset or duplicate fixtures")
require(len(records) == 9 and {r.get("fixture") for r in records} == set(FIXTURE_NAMES),
"report fixture inventory is incomplete")
environment = report.get("environment", {})
require(environment.get("backend") == "macos-coreai"
and environment.get("requested_compute_unit") == "cpu"
and environment.get("actual_compute_unit") == "cpu"
and environment.get("reference_mode") == "cache", "report is not native CPU with cached references")
require(report.get("acceptance") == ACCEPTANCE, "acceptance thresholds changed")
bundle = bundles / variant
expected = provenance_snapshot(source, source / variant, bundle, upstream)
# The helper's own ROOT is the real checkout; explicit root supports checks of copied releases too.
expected["runtime_sha256"] = code_hashes(root / "timestral_coreai")
expected["validation_script_sha256"] = sha256(root / "scripts" / "validate.py")
require(report.get("provenance_unchanged") is True, "source changed during validation")
require(all(report.get(key) == value for key, value in expected.items()), "report hashes do not match current code/bundle/source")
require(report.get("provenance_after") == expected, "final provenance snapshot does not match current files")
cache_status = json.loads((cache / variant / "prepare-status.json").read_text())
expected_refs = {f"{name}:{backend}" for name in FIXTURE_NAMES for backend in ("mlx", "upstream")}
require(cache_status.get("status") == "completed"
and set(cache_status.get("completed_references", [])) == expected_refs,
"reference preparation did not complete both references for all nine fixtures")
require(cache_status.get("provenance") == reference_provenance(source, variant, upstream),
"reference preparation source hashes are stale")
cases = {case[0]: case for case in fixtures(extended=True)}
for record in records:
name = record["fixture"]
require(record.get("completed") is True, f"{name}: incomplete case")
check_metrics(record, name + " / MLX")
check_metrics(record.get("upstream_comparison", {}), name + " / original Mistral")
arrays, metadata = load_reference_cache(cache, variant, cases[name], source, upstream)
require(set(arrays) == {"mlx", "upstream"}, f"{name}: expected both cached references")
require(record.get("reference_cache") == metadata, f"{name}: cached references changed after validation")
manifest = json.loads((bundle / "manifest.json").read_text())
require(manifest.get("variant") == variant and manifest.get("dtype") == "float32", "bundle identity/precision mismatch")
require(manifest.get("source_manifest_sha256") == expected["source_manifest_sha256"], "bundle source manifest mismatch")
require(not manifest.get("diagnostic_exporter"), "diagnostic export must be promoted through the production exporter")
for relative, expected_hash in manifest.get("exporter_sha256", {}).items():
require(sha256(root / relative) == expected_hash, f"bundle exporter code is stale: {relative}")
asset_bytes, asset_count, bundle_bytes = verify_files(bundle, manifest)
elapsed = (datetime.fromisoformat(report["updated_at"]) - datetime.fromisoformat(report["started_at"])).total_seconds()
conversion = manifest.get("conversion", {})
return dict(
variant=variant, path=str(bundle), passed=True, fixture_count=9, compute_unit="cpu", backend="macos-coreai",
format=manifest.get("format"), parameters=conversion.get("parameter_count"),
asset_bytes=asset_bytes, asset_count=asset_count, bundle_bytes=bundle_bytes,
bundle_manifest_sha256=expected["bundle_manifest_sha256"],
source_manifest_sha256=expected["source_manifest_sha256"],
validation_report=str(path), validation_report_sha256=sha256(path),
elapsed_seconds=elapsed, coreai_inference_seconds=sum(item["coreai_seconds"] for item in records),
worst_error_vs_mlx=worst_metrics(records),
worst_error_vs_original=worst_metrics([item["upstream_comparison"] for item in records]),
)
def summarize(root, bundles, reports, source, cache, upstream):
initial_code = code_hashes(root / "timestral_coreai")
models, errors = [], []
for variant in VARIANTS:
try:
models.append(check_variant(variant, root, bundles, reports, source, cache, upstream))
except (OSError, ValueError, KeyError, TypeError) as exc:
errors.append(dict(variant=variant, error=str(exc)))
if code_hashes(root / "timestral_coreai") != initial_code:
errors.append(dict(error="Package source changed while verifying the catalog"))
return dict(
format="timestral-coreai-collection-v1", passed=not errors and len(models) == 4,
expected_variants=list(VARIANTS), runtime_sha256=initial_code, acceptance=ACCEPTANCE,
notes="All four variants require nine accepted fixtures against cached MLX and original Mistral references. Timings include specialization and are not benchmarks.",
models=models, errors=errors,
)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", type=Path, default=ROOT)
parser.add_argument("--bundles", type=Path)
parser.add_argument("--reports", type=Path)
parser.add_argument("--source", type=Path)
parser.add_argument("--reference-cache", type=Path)
parser.add_argument("--upstream", type=Path)
parser.add_argument("--output", type=Path, help="Write catalog here only if every acceptance check passes")
args = parser.parse_args()
root = args.root.resolve()
result = summarize(
root, args.bundles or root, args.reports or root / "reports",
args.source or root.parent / "timestral-mlx",
args.reference_cache or root / "reports" / "reference-cache",
args.upstream or root / "references" / "timestral-a534bdbd244d64f3d9d743f46c7bf6fc330ed4ff" / "src",
)
print(json.dumps(result, indent=2, allow_nan=False))
if not result["passed"]:
raise SystemExit(1)
if args.output:
atomic_json(args.output, result)
if __name__ == "__main__":
main()