"""Create a model catalog only after all four current bundles pass full native CPU validation.""" from __future__ import annotations import argparse from datetime import datetime import json import math from pathlib import Path, PurePosixPath import sys ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from scripts.validate import ( ACCEPTANCE, FIXTURE_NAMES, VARIANTS, atomic_json, code_hashes, fixtures, load_reference_cache, provenance_snapshot, reference_provenance, sha256, ) def require(condition, message): if not condition: raise ValueError(message) def verify_files(bundle, manifest): files = manifest.get("files", {}) require(isinstance(files, dict) and files, "bundle has no file inventory") asset_bytes, asset_roots = 0, set() for name, expected in files.items(): relative = PurePosixPath(name) path = bundle / name require(not relative.is_absolute() and ".." not in relative.parts and str(relative) == name, f"unsafe bundle path: {name}") require(path.is_file() and not path.is_symlink() and path.resolve().is_relative_to(bundle.resolve()), f"missing or unsafe bundle file: {name}") require(path.stat().st_size == expected["bytes"] and sha256(path) == expected["sha256"], f"bundle file checksum mismatch: {name}") if relative.parts[0].endswith(".aimodel"): asset_roots.add(relative.parts[0]) asset_bytes += path.stat().st_size require(asset_roots, "bundle contains no CoreAI assets") for asset in asset_roots: actual = {p.relative_to(bundle).as_posix() for p in (bundle / asset).rglob("*") if p.is_file()} expected = {name for name in files if name.startswith(asset + "/")} require(actual == expected, f"unverified files inside {asset}") return asset_bytes, len(asset_roots), sum(record["bytes"] for record in files.values()) + (bundle / "manifest.json").stat().st_size def check_metrics(metrics, label): require(metrics.get("passed") and metrics.get("finite") and metrics.get("monotone"), f"{label}: failed, nonfinite, or unordered quantiles") for key, maximum in (("normalized_rmse", ACCEPTANCE["normalized_rmse_max"]), ("normalized_max_error", ACCEPTANCE["normalized_max_error_max"])): value = metrics.get(key) require(isinstance(value, (int, float)) and math.isfinite(value) and 0 <= value <= maximum, f"{label}: {key} does not meet fixed acceptance") def worst_metrics(records): psnr = [item["psnr_db"] for item in records if item.get("psnr_db") is not None] return dict( normalized_rmse=max(item["normalized_rmse"] for item in records), normalized_max_error=max(item["normalized_max_error"] for item in records), max_absolute_error=max(item["max_absolute_error"] for item in records), minimum_psnr_db=min(psnr) if psnr else None, ) def check_variant(variant, root, bundles, reports, source, cache, upstream): path = reports / f"{variant}.json" report = json.loads(path.read_text()) require(report.get("variant") == variant, "report variant mismatch") require(report.get("status") == "completed" and report.get("passed") and report.get("extended_suite_passed"), "report is not a completed, accepted extended suite") require(report.get("expected_fixture_count") == 9 and report.get("completed_fixture_count") == 9 and report.get("remaining_fixtures") == [], "report did not complete all nine fixtures") names = report.get("requested_fixtures", []) records = report.get("fixtures", []) require(len(names) == 9 and set(names) == set(FIXTURE_NAMES), "report requested a subset or duplicate fixtures") require(len(records) == 9 and {r.get("fixture") for r in records} == set(FIXTURE_NAMES), "report fixture inventory is incomplete") environment = report.get("environment", {}) require(environment.get("backend") == "macos-coreai" and environment.get("requested_compute_unit") == "cpu" and environment.get("actual_compute_unit") == "cpu" and environment.get("reference_mode") == "cache", "report is not native CPU with cached references") require(report.get("acceptance") == ACCEPTANCE, "acceptance thresholds changed") bundle = bundles / variant expected = provenance_snapshot(source, source / variant, bundle, upstream) # The helper's own ROOT is the real checkout; explicit root supports checks of copied releases too. expected["runtime_sha256"] = code_hashes(root / "timestral_coreai") expected["validation_script_sha256"] = sha256(root / "scripts" / "validate.py") require(report.get("provenance_unchanged") is True, "source changed during validation") require(all(report.get(key) == value for key, value in expected.items()), "report hashes do not match current code/bundle/source") require(report.get("provenance_after") == expected, "final provenance snapshot does not match current files") cache_status = json.loads((cache / variant / "prepare-status.json").read_text()) expected_refs = {f"{name}:{backend}" for name in FIXTURE_NAMES for backend in ("mlx", "upstream")} require(cache_status.get("status") == "completed" and set(cache_status.get("completed_references", [])) == expected_refs, "reference preparation did not complete both references for all nine fixtures") require(cache_status.get("provenance") == reference_provenance(source, variant, upstream), "reference preparation source hashes are stale") cases = {case[0]: case for case in fixtures(extended=True)} for record in records: name = record["fixture"] require(record.get("completed") is True, f"{name}: incomplete case") check_metrics(record, name + " / MLX") check_metrics(record.get("upstream_comparison", {}), name + " / original Mistral") arrays, metadata = load_reference_cache(cache, variant, cases[name], source, upstream) require(set(arrays) == {"mlx", "upstream"}, f"{name}: expected both cached references") require(record.get("reference_cache") == metadata, f"{name}: cached references changed after validation") manifest = json.loads((bundle / "manifest.json").read_text()) require(manifest.get("variant") == variant and manifest.get("dtype") == "float32", "bundle identity/precision mismatch") require(manifest.get("source_manifest_sha256") == expected["source_manifest_sha256"], "bundle source manifest mismatch") require(not manifest.get("diagnostic_exporter"), "diagnostic export must be promoted through the production exporter") for relative, expected_hash in manifest.get("exporter_sha256", {}).items(): require(sha256(root / relative) == expected_hash, f"bundle exporter code is stale: {relative}") asset_bytes, asset_count, bundle_bytes = verify_files(bundle, manifest) elapsed = (datetime.fromisoformat(report["updated_at"]) - datetime.fromisoformat(report["started_at"])).total_seconds() conversion = manifest.get("conversion", {}) return dict( variant=variant, path=str(bundle), passed=True, fixture_count=9, compute_unit="cpu", backend="macos-coreai", format=manifest.get("format"), parameters=conversion.get("parameter_count"), asset_bytes=asset_bytes, asset_count=asset_count, bundle_bytes=bundle_bytes, bundle_manifest_sha256=expected["bundle_manifest_sha256"], source_manifest_sha256=expected["source_manifest_sha256"], validation_report=str(path), validation_report_sha256=sha256(path), elapsed_seconds=elapsed, coreai_inference_seconds=sum(item["coreai_seconds"] for item in records), worst_error_vs_mlx=worst_metrics(records), worst_error_vs_original=worst_metrics([item["upstream_comparison"] for item in records]), ) def summarize(root, bundles, reports, source, cache, upstream): initial_code = code_hashes(root / "timestral_coreai") models, errors = [], [] for variant in VARIANTS: try: models.append(check_variant(variant, root, bundles, reports, source, cache, upstream)) except (OSError, ValueError, KeyError, TypeError) as exc: errors.append(dict(variant=variant, error=str(exc))) if code_hashes(root / "timestral_coreai") != initial_code: errors.append(dict(error="Package source changed while verifying the catalog")) return dict( format="timestral-coreai-collection-v1", passed=not errors and len(models) == 4, expected_variants=list(VARIANTS), runtime_sha256=initial_code, acceptance=ACCEPTANCE, notes="All four variants require nine accepted fixtures against cached MLX and original Mistral references. Timings include specialization and are not benchmarks.", models=models, errors=errors, ) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--root", type=Path, default=ROOT) parser.add_argument("--bundles", type=Path) parser.add_argument("--reports", type=Path) parser.add_argument("--source", type=Path) parser.add_argument("--reference-cache", type=Path) parser.add_argument("--upstream", type=Path) parser.add_argument("--output", type=Path, help="Write catalog here only if every acceptance check passes") args = parser.parse_args() root = args.root.resolve() result = summarize( root, args.bundles or root, args.reports or root / "reports", args.source or root.parent / "timestral-mlx", args.reference_cache or root / "reports" / "reference-cache", args.upstream or root / "references" / "timestral-a534bdbd244d64f3d9d743f46c7bf6fc330ed4ff" / "src", ) print(json.dumps(result, indent=2, allow_nan=False)) if not result["passed"]: raise SystemExit(1) if args.output: atomic_json(args.output, result) if __name__ == "__main__": main()