File size: 10,073 Bytes
506f40d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 | """Create a model catalog only after all four current bundles pass full native CPU validation."""
from __future__ import annotations
import argparse
from datetime import datetime
import json
import math
from pathlib import Path, PurePosixPath
import sys
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from scripts.validate import (
ACCEPTANCE, FIXTURE_NAMES, VARIANTS, atomic_json, code_hashes, fixtures,
load_reference_cache, provenance_snapshot, reference_provenance, sha256,
)
def require(condition, message):
if not condition:
raise ValueError(message)
def verify_files(bundle, manifest):
files = manifest.get("files", {})
require(isinstance(files, dict) and files, "bundle has no file inventory")
asset_bytes, asset_roots = 0, set()
for name, expected in files.items():
relative = PurePosixPath(name)
path = bundle / name
require(not relative.is_absolute() and ".." not in relative.parts and str(relative) == name,
f"unsafe bundle path: {name}")
require(path.is_file() and not path.is_symlink() and path.resolve().is_relative_to(bundle.resolve()),
f"missing or unsafe bundle file: {name}")
require(path.stat().st_size == expected["bytes"] and sha256(path) == expected["sha256"],
f"bundle file checksum mismatch: {name}")
if relative.parts[0].endswith(".aimodel"):
asset_roots.add(relative.parts[0])
asset_bytes += path.stat().st_size
require(asset_roots, "bundle contains no CoreAI assets")
for asset in asset_roots:
actual = {p.relative_to(bundle).as_posix() for p in (bundle / asset).rglob("*") if p.is_file()}
expected = {name for name in files if name.startswith(asset + "/")}
require(actual == expected, f"unverified files inside {asset}")
return asset_bytes, len(asset_roots), sum(record["bytes"] for record in files.values()) + (bundle / "manifest.json").stat().st_size
def check_metrics(metrics, label):
require(metrics.get("passed") and metrics.get("finite") and metrics.get("monotone"),
f"{label}: failed, nonfinite, or unordered quantiles")
for key, maximum in (("normalized_rmse", ACCEPTANCE["normalized_rmse_max"]),
("normalized_max_error", ACCEPTANCE["normalized_max_error_max"])):
value = metrics.get(key)
require(isinstance(value, (int, float)) and math.isfinite(value) and 0 <= value <= maximum,
f"{label}: {key} does not meet fixed acceptance")
def worst_metrics(records):
psnr = [item["psnr_db"] for item in records if item.get("psnr_db") is not None]
return dict(
normalized_rmse=max(item["normalized_rmse"] for item in records),
normalized_max_error=max(item["normalized_max_error"] for item in records),
max_absolute_error=max(item["max_absolute_error"] for item in records),
minimum_psnr_db=min(psnr) if psnr else None,
)
def check_variant(variant, root, bundles, reports, source, cache, upstream):
path = reports / f"{variant}.json"
report = json.loads(path.read_text())
require(report.get("variant") == variant, "report variant mismatch")
require(report.get("status") == "completed" and report.get("passed")
and report.get("extended_suite_passed"), "report is not a completed, accepted extended suite")
require(report.get("expected_fixture_count") == 9 and report.get("completed_fixture_count") == 9
and report.get("remaining_fixtures") == [], "report did not complete all nine fixtures")
names = report.get("requested_fixtures", [])
records = report.get("fixtures", [])
require(len(names) == 9 and set(names) == set(FIXTURE_NAMES), "report requested a subset or duplicate fixtures")
require(len(records) == 9 and {r.get("fixture") for r in records} == set(FIXTURE_NAMES),
"report fixture inventory is incomplete")
environment = report.get("environment", {})
require(environment.get("backend") == "macos-coreai"
and environment.get("requested_compute_unit") == "cpu"
and environment.get("actual_compute_unit") == "cpu"
and environment.get("reference_mode") == "cache", "report is not native CPU with cached references")
require(report.get("acceptance") == ACCEPTANCE, "acceptance thresholds changed")
bundle = bundles / variant
expected = provenance_snapshot(source, source / variant, bundle, upstream)
# The helper's own ROOT is the real checkout; explicit root supports checks of copied releases too.
expected["runtime_sha256"] = code_hashes(root / "timestral_coreai")
expected["validation_script_sha256"] = sha256(root / "scripts" / "validate.py")
require(report.get("provenance_unchanged") is True, "source changed during validation")
require(all(report.get(key) == value for key, value in expected.items()), "report hashes do not match current code/bundle/source")
require(report.get("provenance_after") == expected, "final provenance snapshot does not match current files")
cache_status = json.loads((cache / variant / "prepare-status.json").read_text())
expected_refs = {f"{name}:{backend}" for name in FIXTURE_NAMES for backend in ("mlx", "upstream")}
require(cache_status.get("status") == "completed"
and set(cache_status.get("completed_references", [])) == expected_refs,
"reference preparation did not complete both references for all nine fixtures")
require(cache_status.get("provenance") == reference_provenance(source, variant, upstream),
"reference preparation source hashes are stale")
cases = {case[0]: case for case in fixtures(extended=True)}
for record in records:
name = record["fixture"]
require(record.get("completed") is True, f"{name}: incomplete case")
check_metrics(record, name + " / MLX")
check_metrics(record.get("upstream_comparison", {}), name + " / original Mistral")
arrays, metadata = load_reference_cache(cache, variant, cases[name], source, upstream)
require(set(arrays) == {"mlx", "upstream"}, f"{name}: expected both cached references")
require(record.get("reference_cache") == metadata, f"{name}: cached references changed after validation")
manifest = json.loads((bundle / "manifest.json").read_text())
require(manifest.get("variant") == variant and manifest.get("dtype") == "float32", "bundle identity/precision mismatch")
require(manifest.get("source_manifest_sha256") == expected["source_manifest_sha256"], "bundle source manifest mismatch")
require(not manifest.get("diagnostic_exporter"), "diagnostic export must be promoted through the production exporter")
for relative, expected_hash in manifest.get("exporter_sha256", {}).items():
require(sha256(root / relative) == expected_hash, f"bundle exporter code is stale: {relative}")
asset_bytes, asset_count, bundle_bytes = verify_files(bundle, manifest)
elapsed = (datetime.fromisoformat(report["updated_at"]) - datetime.fromisoformat(report["started_at"])).total_seconds()
conversion = manifest.get("conversion", {})
return dict(
variant=variant, path=str(bundle), passed=True, fixture_count=9, compute_unit="cpu", backend="macos-coreai",
format=manifest.get("format"), parameters=conversion.get("parameter_count"),
asset_bytes=asset_bytes, asset_count=asset_count, bundle_bytes=bundle_bytes,
bundle_manifest_sha256=expected["bundle_manifest_sha256"],
source_manifest_sha256=expected["source_manifest_sha256"],
validation_report=str(path), validation_report_sha256=sha256(path),
elapsed_seconds=elapsed, coreai_inference_seconds=sum(item["coreai_seconds"] for item in records),
worst_error_vs_mlx=worst_metrics(records),
worst_error_vs_original=worst_metrics([item["upstream_comparison"] for item in records]),
)
def summarize(root, bundles, reports, source, cache, upstream):
initial_code = code_hashes(root / "timestral_coreai")
models, errors = [], []
for variant in VARIANTS:
try:
models.append(check_variant(variant, root, bundles, reports, source, cache, upstream))
except (OSError, ValueError, KeyError, TypeError) as exc:
errors.append(dict(variant=variant, error=str(exc)))
if code_hashes(root / "timestral_coreai") != initial_code:
errors.append(dict(error="Package source changed while verifying the catalog"))
return dict(
format="timestral-coreai-collection-v1", passed=not errors and len(models) == 4,
expected_variants=list(VARIANTS), runtime_sha256=initial_code, acceptance=ACCEPTANCE,
notes="All four variants require nine accepted fixtures against cached MLX and original Mistral references. Timings include specialization and are not benchmarks.",
models=models, errors=errors,
)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--root", type=Path, default=ROOT)
parser.add_argument("--bundles", type=Path)
parser.add_argument("--reports", type=Path)
parser.add_argument("--source", type=Path)
parser.add_argument("--reference-cache", type=Path)
parser.add_argument("--upstream", type=Path)
parser.add_argument("--output", type=Path, help="Write catalog here only if every acceptance check passes")
args = parser.parse_args()
root = args.root.resolve()
result = summarize(
root, args.bundles or root, args.reports or root / "reports",
args.source or root.parent / "timestral-mlx",
args.reference_cache or root / "reports" / "reference-cache",
args.upstream or root / "references" / "timestral-a534bdbd244d64f3d9d743f46c7bf6fc330ed4ff" / "src",
)
print(json.dumps(result, indent=2, allow_nan=False))
if not result["passed"]:
raise SystemExit(1)
if args.output:
atomic_json(args.output, result)
if __name__ == "__main__":
main()
|