Download scripts/summarize_validation.py from jbilcke/timestral-coreai: direct link, hf CLI and curl.
- Browser
- Download file 10.1 kB
-
https://huggingface.co/jbilcke/timestral-coreai/resolve/main/scripts/summarize_validation.py
- Command line
-
hf download hf://jbilcke/timestral-coreai/scripts/summarize_validation.py
-
curl -L -o summarize_validation.py https://huggingface.co/jbilcke/timestral-coreai/resolve/main/scripts/summarize_validation.py
10.1 kB
| """Create a model catalog only after all four current bundles pass full native CPU validation.""" | |
| from __future__ import annotations | |
| import argparse | |
| from datetime import datetime | |
| import json | |
| import math | |
| from pathlib import Path, PurePosixPath | |
| import sys | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT)) | |
| from scripts.validate import ( | |
| ACCEPTANCE, FIXTURE_NAMES, VARIANTS, atomic_json, code_hashes, fixtures, | |
| load_reference_cache, provenance_snapshot, reference_provenance, sha256, | |
| ) | |
| def require(condition, message): | |
| if not condition: | |
| raise ValueError(message) | |
| def verify_files(bundle, manifest): | |
| files = manifest.get("files", {}) | |
| require(isinstance(files, dict) and files, "bundle has no file inventory") | |
| asset_bytes, asset_roots = 0, set() | |
| for name, expected in files.items(): | |
| relative = PurePosixPath(name) | |
| path = bundle / name | |
| require(not relative.is_absolute() and ".." not in relative.parts and str(relative) == name, | |
| f"unsafe bundle path: {name}") | |
| require(path.is_file() and not path.is_symlink() and path.resolve().is_relative_to(bundle.resolve()), | |
| f"missing or unsafe bundle file: {name}") | |
| require(path.stat().st_size == expected["bytes"] and sha256(path) == expected["sha256"], | |
| f"bundle file checksum mismatch: {name}") | |
| if relative.parts[0].endswith(".aimodel"): | |
| asset_roots.add(relative.parts[0]) | |
| asset_bytes += path.stat().st_size | |
| require(asset_roots, "bundle contains no CoreAI assets") | |
| for asset in asset_roots: | |
| actual = {p.relative_to(bundle).as_posix() for p in (bundle / asset).rglob("*") if p.is_file()} | |
| expected = {name for name in files if name.startswith(asset + "/")} | |
| require(actual == expected, f"unverified files inside {asset}") | |
| return asset_bytes, len(asset_roots), sum(record["bytes"] for record in files.values()) + (bundle / "manifest.json").stat().st_size | |
| def check_metrics(metrics, label): | |
| require(metrics.get("passed") and metrics.get("finite") and metrics.get("monotone"), | |
| f"{label}: failed, nonfinite, or unordered quantiles") | |
| for key, maximum in (("normalized_rmse", ACCEPTANCE["normalized_rmse_max"]), | |
| ("normalized_max_error", ACCEPTANCE["normalized_max_error_max"])): | |
| value = metrics.get(key) | |
| require(isinstance(value, (int, float)) and math.isfinite(value) and 0 <= value <= maximum, | |
| f"{label}: {key} does not meet fixed acceptance") | |
| def worst_metrics(records): | |
| psnr = [item["psnr_db"] for item in records if item.get("psnr_db") is not None] | |
| return dict( | |
| normalized_rmse=max(item["normalized_rmse"] for item in records), | |
| normalized_max_error=max(item["normalized_max_error"] for item in records), | |
| max_absolute_error=max(item["max_absolute_error"] for item in records), | |
| minimum_psnr_db=min(psnr) if psnr else None, | |
| ) | |
| def check_variant(variant, root, bundles, reports, source, cache, upstream): | |
| path = reports / f"{variant}.json" | |
| report = json.loads(path.read_text()) | |
| require(report.get("variant") == variant, "report variant mismatch") | |
| require(report.get("status") == "completed" and report.get("passed") | |
| and report.get("extended_suite_passed"), "report is not a completed, accepted extended suite") | |
| require(report.get("expected_fixture_count") == 9 and report.get("completed_fixture_count") == 9 | |
| and report.get("remaining_fixtures") == [], "report did not complete all nine fixtures") | |
| names = report.get("requested_fixtures", []) | |
| records = report.get("fixtures", []) | |
| require(len(names) == 9 and set(names) == set(FIXTURE_NAMES), "report requested a subset or duplicate fixtures") | |
| require(len(records) == 9 and {r.get("fixture") for r in records} == set(FIXTURE_NAMES), | |
| "report fixture inventory is incomplete") | |
| environment = report.get("environment", {}) | |
| require(environment.get("backend") == "macos-coreai" | |
| and environment.get("requested_compute_unit") == "cpu" | |
| and environment.get("actual_compute_unit") == "cpu" | |
| and environment.get("reference_mode") == "cache", "report is not native CPU with cached references") | |
| require(report.get("acceptance") == ACCEPTANCE, "acceptance thresholds changed") | |
| bundle = bundles / variant | |
| expected = provenance_snapshot(source, source / variant, bundle, upstream) | |
| # The helper's own ROOT is the real checkout; explicit root supports checks of copied releases too. | |
| expected["runtime_sha256"] = code_hashes(root / "timestral_coreai") | |
| expected["validation_script_sha256"] = sha256(root / "scripts" / "validate.py") | |
| require(report.get("provenance_unchanged") is True, "source changed during validation") | |
| require(all(report.get(key) == value for key, value in expected.items()), "report hashes do not match current code/bundle/source") | |
| require(report.get("provenance_after") == expected, "final provenance snapshot does not match current files") | |
| cache_status = json.loads((cache / variant / "prepare-status.json").read_text()) | |
| expected_refs = {f"{name}:{backend}" for name in FIXTURE_NAMES for backend in ("mlx", "upstream")} | |
| require(cache_status.get("status") == "completed" | |
| and set(cache_status.get("completed_references", [])) == expected_refs, | |
| "reference preparation did not complete both references for all nine fixtures") | |
| require(cache_status.get("provenance") == reference_provenance(source, variant, upstream), | |
| "reference preparation source hashes are stale") | |
| cases = {case[0]: case for case in fixtures(extended=True)} | |
| for record in records: | |
| name = record["fixture"] | |
| require(record.get("completed") is True, f"{name}: incomplete case") | |
| check_metrics(record, name + " / MLX") | |
| check_metrics(record.get("upstream_comparison", {}), name + " / original Mistral") | |
| arrays, metadata = load_reference_cache(cache, variant, cases[name], source, upstream) | |
| require(set(arrays) == {"mlx", "upstream"}, f"{name}: expected both cached references") | |
| require(record.get("reference_cache") == metadata, f"{name}: cached references changed after validation") | |
| manifest = json.loads((bundle / "manifest.json").read_text()) | |
| require(manifest.get("variant") == variant and manifest.get("dtype") == "float32", "bundle identity/precision mismatch") | |
| require(manifest.get("source_manifest_sha256") == expected["source_manifest_sha256"], "bundle source manifest mismatch") | |
| require(not manifest.get("diagnostic_exporter"), "diagnostic export must be promoted through the production exporter") | |
| for relative, expected_hash in manifest.get("exporter_sha256", {}).items(): | |
| require(sha256(root / relative) == expected_hash, f"bundle exporter code is stale: {relative}") | |
| asset_bytes, asset_count, bundle_bytes = verify_files(bundle, manifest) | |
| elapsed = (datetime.fromisoformat(report["updated_at"]) - datetime.fromisoformat(report["started_at"])).total_seconds() | |
| conversion = manifest.get("conversion", {}) | |
| return dict( | |
| variant=variant, path=str(bundle), passed=True, fixture_count=9, compute_unit="cpu", backend="macos-coreai", | |
| format=manifest.get("format"), parameters=conversion.get("parameter_count"), | |
| asset_bytes=asset_bytes, asset_count=asset_count, bundle_bytes=bundle_bytes, | |
| bundle_manifest_sha256=expected["bundle_manifest_sha256"], | |
| source_manifest_sha256=expected["source_manifest_sha256"], | |
| validation_report=str(path), validation_report_sha256=sha256(path), | |
| elapsed_seconds=elapsed, coreai_inference_seconds=sum(item["coreai_seconds"] for item in records), | |
| worst_error_vs_mlx=worst_metrics(records), | |
| worst_error_vs_original=worst_metrics([item["upstream_comparison"] for item in records]), | |
| ) | |
| def summarize(root, bundles, reports, source, cache, upstream): | |
| initial_code = code_hashes(root / "timestral_coreai") | |
| models, errors = [], [] | |
| for variant in VARIANTS: | |
| try: | |
| models.append(check_variant(variant, root, bundles, reports, source, cache, upstream)) | |
| except (OSError, ValueError, KeyError, TypeError) as exc: | |
| errors.append(dict(variant=variant, error=str(exc))) | |
| if code_hashes(root / "timestral_coreai") != initial_code: | |
| errors.append(dict(error="Package source changed while verifying the catalog")) | |
| return dict( | |
| format="timestral-coreai-collection-v1", passed=not errors and len(models) == 4, | |
| expected_variants=list(VARIANTS), runtime_sha256=initial_code, acceptance=ACCEPTANCE, | |
| notes="All four variants require nine accepted fixtures against cached MLX and original Mistral references. Timings include specialization and are not benchmarks.", | |
| models=models, errors=errors, | |
| ) | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--root", type=Path, default=ROOT) | |
| parser.add_argument("--bundles", type=Path) | |
| parser.add_argument("--reports", type=Path) | |
| parser.add_argument("--source", type=Path) | |
| parser.add_argument("--reference-cache", type=Path) | |
| parser.add_argument("--upstream", type=Path) | |
| parser.add_argument("--output", type=Path, help="Write catalog here only if every acceptance check passes") | |
| args = parser.parse_args() | |
| root = args.root.resolve() | |
| result = summarize( | |
| root, args.bundles or root, args.reports or root / "reports", | |
| args.source or root.parent / "timestral-mlx", | |
| args.reference_cache or root / "reports" / "reference-cache", | |
| args.upstream or root / "references" / "timestral-a534bdbd244d64f3d9d743f46c7bf6fc330ed4ff" / "src", | |
| ) | |
| print(json.dumps(result, indent=2, allow_nan=False)) | |
| if not result["passed"]: | |
| raise SystemExit(1) | |
| if args.output: | |
| atomic_json(args.output, result) | |
| if __name__ == "__main__": | |
| main() | |