File size: 10,073 Bytes
506f40d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
"""Create a model catalog only after all four current bundles pass full native CPU validation."""
from __future__ import annotations

import argparse
from datetime import datetime
import json
import math
from pathlib import Path, PurePosixPath
import sys

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from scripts.validate import (
    ACCEPTANCE, FIXTURE_NAMES, VARIANTS, atomic_json, code_hashes, fixtures,
    load_reference_cache, provenance_snapshot, reference_provenance, sha256,
)


def require(condition, message):
    if not condition:
        raise ValueError(message)


def verify_files(bundle, manifest):
    files = manifest.get("files", {})
    require(isinstance(files, dict) and files, "bundle has no file inventory")
    asset_bytes, asset_roots = 0, set()
    for name, expected in files.items():
        relative = PurePosixPath(name)
        path = bundle / name
        require(not relative.is_absolute() and ".." not in relative.parts and str(relative) == name,
                f"unsafe bundle path: {name}")
        require(path.is_file() and not path.is_symlink() and path.resolve().is_relative_to(bundle.resolve()),
                f"missing or unsafe bundle file: {name}")
        require(path.stat().st_size == expected["bytes"] and sha256(path) == expected["sha256"],
                f"bundle file checksum mismatch: {name}")
        if relative.parts[0].endswith(".aimodel"):
            asset_roots.add(relative.parts[0])
            asset_bytes += path.stat().st_size
    require(asset_roots, "bundle contains no CoreAI assets")
    for asset in asset_roots:
        actual = {p.relative_to(bundle).as_posix() for p in (bundle / asset).rglob("*") if p.is_file()}
        expected = {name for name in files if name.startswith(asset + "/")}
        require(actual == expected, f"unverified files inside {asset}")
    return asset_bytes, len(asset_roots), sum(record["bytes"] for record in files.values()) + (bundle / "manifest.json").stat().st_size


def check_metrics(metrics, label):
    require(metrics.get("passed") and metrics.get("finite") and metrics.get("monotone"),
            f"{label}: failed, nonfinite, or unordered quantiles")
    for key, maximum in (("normalized_rmse", ACCEPTANCE["normalized_rmse_max"]),
                         ("normalized_max_error", ACCEPTANCE["normalized_max_error_max"])):
        value = metrics.get(key)
        require(isinstance(value, (int, float)) and math.isfinite(value) and 0 <= value <= maximum,
                f"{label}: {key} does not meet fixed acceptance")


def worst_metrics(records):
    psnr = [item["psnr_db"] for item in records if item.get("psnr_db") is not None]
    return dict(
        normalized_rmse=max(item["normalized_rmse"] for item in records),
        normalized_max_error=max(item["normalized_max_error"] for item in records),
        max_absolute_error=max(item["max_absolute_error"] for item in records),
        minimum_psnr_db=min(psnr) if psnr else None,
    )


def check_variant(variant, root, bundles, reports, source, cache, upstream):
    path = reports / f"{variant}.json"
    report = json.loads(path.read_text())
    require(report.get("variant") == variant, "report variant mismatch")
    require(report.get("status") == "completed" and report.get("passed")
            and report.get("extended_suite_passed"), "report is not a completed, accepted extended suite")
    require(report.get("expected_fixture_count") == 9 and report.get("completed_fixture_count") == 9
            and report.get("remaining_fixtures") == [], "report did not complete all nine fixtures")
    names = report.get("requested_fixtures", [])
    records = report.get("fixtures", [])
    require(len(names) == 9 and set(names) == set(FIXTURE_NAMES), "report requested a subset or duplicate fixtures")
    require(len(records) == 9 and {r.get("fixture") for r in records} == set(FIXTURE_NAMES),
            "report fixture inventory is incomplete")
    environment = report.get("environment", {})
    require(environment.get("backend") == "macos-coreai"
            and environment.get("requested_compute_unit") == "cpu"
            and environment.get("actual_compute_unit") == "cpu"
            and environment.get("reference_mode") == "cache", "report is not native CPU with cached references")
    require(report.get("acceptance") == ACCEPTANCE, "acceptance thresholds changed")
    bundle = bundles / variant
    expected = provenance_snapshot(source, source / variant, bundle, upstream)
    # The helper's own ROOT is the real checkout; explicit root supports checks of copied releases too.
    expected["runtime_sha256"] = code_hashes(root / "timestral_coreai")
    expected["validation_script_sha256"] = sha256(root / "scripts" / "validate.py")
    require(report.get("provenance_unchanged") is True, "source changed during validation")
    require(all(report.get(key) == value for key, value in expected.items()), "report hashes do not match current code/bundle/source")
    require(report.get("provenance_after") == expected, "final provenance snapshot does not match current files")

    cache_status = json.loads((cache / variant / "prepare-status.json").read_text())
    expected_refs = {f"{name}:{backend}" for name in FIXTURE_NAMES for backend in ("mlx", "upstream")}
    require(cache_status.get("status") == "completed"
            and set(cache_status.get("completed_references", [])) == expected_refs,
            "reference preparation did not complete both references for all nine fixtures")
    require(cache_status.get("provenance") == reference_provenance(source, variant, upstream),
            "reference preparation source hashes are stale")
    cases = {case[0]: case for case in fixtures(extended=True)}
    for record in records:
        name = record["fixture"]
        require(record.get("completed") is True, f"{name}: incomplete case")
        check_metrics(record, name + " / MLX")
        check_metrics(record.get("upstream_comparison", {}), name + " / original Mistral")
        arrays, metadata = load_reference_cache(cache, variant, cases[name], source, upstream)
        require(set(arrays) == {"mlx", "upstream"}, f"{name}: expected both cached references")
        require(record.get("reference_cache") == metadata, f"{name}: cached references changed after validation")

    manifest = json.loads((bundle / "manifest.json").read_text())
    require(manifest.get("variant") == variant and manifest.get("dtype") == "float32", "bundle identity/precision mismatch")
    require(manifest.get("source_manifest_sha256") == expected["source_manifest_sha256"], "bundle source manifest mismatch")
    require(not manifest.get("diagnostic_exporter"), "diagnostic export must be promoted through the production exporter")
    for relative, expected_hash in manifest.get("exporter_sha256", {}).items():
        require(sha256(root / relative) == expected_hash, f"bundle exporter code is stale: {relative}")
    asset_bytes, asset_count, bundle_bytes = verify_files(bundle, manifest)
    elapsed = (datetime.fromisoformat(report["updated_at"]) - datetime.fromisoformat(report["started_at"])).total_seconds()
    conversion = manifest.get("conversion", {})
    return dict(
        variant=variant, path=str(bundle), passed=True, fixture_count=9, compute_unit="cpu", backend="macos-coreai",
        format=manifest.get("format"), parameters=conversion.get("parameter_count"),
        asset_bytes=asset_bytes, asset_count=asset_count, bundle_bytes=bundle_bytes,
        bundle_manifest_sha256=expected["bundle_manifest_sha256"],
        source_manifest_sha256=expected["source_manifest_sha256"],
        validation_report=str(path), validation_report_sha256=sha256(path),
        elapsed_seconds=elapsed, coreai_inference_seconds=sum(item["coreai_seconds"] for item in records),
        worst_error_vs_mlx=worst_metrics(records),
        worst_error_vs_original=worst_metrics([item["upstream_comparison"] for item in records]),
    )


def summarize(root, bundles, reports, source, cache, upstream):
    initial_code = code_hashes(root / "timestral_coreai")
    models, errors = [], []
    for variant in VARIANTS:
        try:
            models.append(check_variant(variant, root, bundles, reports, source, cache, upstream))
        except (OSError, ValueError, KeyError, TypeError) as exc:
            errors.append(dict(variant=variant, error=str(exc)))
    if code_hashes(root / "timestral_coreai") != initial_code:
        errors.append(dict(error="Package source changed while verifying the catalog"))
    return dict(
        format="timestral-coreai-collection-v1", passed=not errors and len(models) == 4,
        expected_variants=list(VARIANTS), runtime_sha256=initial_code, acceptance=ACCEPTANCE,
        notes="All four variants require nine accepted fixtures against cached MLX and original Mistral references. Timings include specialization and are not benchmarks.",
        models=models, errors=errors,
    )


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--root", type=Path, default=ROOT)
    parser.add_argument("--bundles", type=Path)
    parser.add_argument("--reports", type=Path)
    parser.add_argument("--source", type=Path)
    parser.add_argument("--reference-cache", type=Path)
    parser.add_argument("--upstream", type=Path)
    parser.add_argument("--output", type=Path, help="Write catalog here only if every acceptance check passes")
    args = parser.parse_args()
    root = args.root.resolve()
    result = summarize(
        root, args.bundles or root, args.reports or root / "reports",
        args.source or root.parent / "timestral-mlx",
        args.reference_cache or root / "reports" / "reference-cache",
        args.upstream or root / "references" / "timestral-a534bdbd244d64f3d9d743f46c7bf6fc330ed4ff" / "src",
    )
    print(json.dumps(result, indent=2, allow_nan=False))
    if not result["passed"]:
        raise SystemExit(1)
    if args.output:
        atomic_json(args.output, result)


if __name__ == "__main__":
    main()