Download scripts/validate_swift.py from jbilcke/timestral-coreai: direct link, hf CLI and curl.
- Browser
- Download file 14.1 kB
-
https://huggingface.co/jbilcke/timestral-coreai/resolve/main/scripts/validate_swift.py
- Command line
-
hf download hf://jbilcke/timestral-coreai/scripts/validate_swift.py
-
curl -L -o validate_swift.py https://huggingface.co/jbilcke/timestral-coreai/resolve/main/scripts/validate_swift.py
14.1 kB
| """Export portable reference fixtures and compare the native Swift SDK without MLX or Torch.""" | |
| from __future__ import annotations | |
| import argparse | |
| from datetime import datetime, timezone | |
| import json | |
| from pathlib import Path | |
| import platform | |
| import subprocess | |
| import sys | |
| import time | |
| import numpy as np | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT)) | |
| from scripts.validate import ( | |
| ACCEPTANCE, FIXTURE_NAMES, VARIANTS, atomic_json, compare, fixture_signature, | |
| fixtures, load_reference_cache, sha256, | |
| ) | |
| DTYPES = {"float32": np.dtype("<f4"), "float64": np.dtype("<f8"), "bool": np.dtype("u1")} | |
| def default_executable(name): | |
| candidates = [ROOT / ".build" / folder / name for folder in ( | |
| "release", "debug", "out/Products/Release", "out/Products/Debug", | |
| "arm64-apple-macosx/release", "arm64-apple-macosx/debug", | |
| )] | |
| return next((path for path in candidates if path.is_file()), candidates[0]) | |
| def run_cli(command, timeout): | |
| """Keep child workers in the outer guard's process group and clean them on timeout.""" | |
| import psutil | |
| process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) | |
| try: | |
| stdout, stderr = process.communicate(timeout=timeout) | |
| return subprocess.CompletedProcess(command, process.returncode, stdout, stderr) | |
| except BaseException: | |
| try: | |
| parent = psutil.Process(process.pid) | |
| children = parent.children(recursive=True) | |
| for child in children + [parent]: | |
| try: | |
| child.terminate() | |
| except psutil.NoSuchProcess: | |
| pass | |
| _, alive = psutil.wait_procs(children + [parent], timeout=3) | |
| for child in alive: | |
| try: | |
| child.kill() | |
| except psutil.NoSuchProcess: | |
| pass | |
| except psutil.NoSuchProcess: | |
| pass | |
| process.communicate(timeout=3) | |
| raise | |
| def write_tensor(base, relative, array): | |
| dtype = "float64" if array.dtype == np.float64 else "bool" if array.dtype == bool else "float32" | |
| converted = np.ascontiguousarray(array, dtype=DTYPES[dtype]) | |
| path = base / relative | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| path.write_bytes(converted.tobytes()) | |
| return dict(file=relative, dtype=dtype, shape=list(converted.shape), bytes=path.stat().st_size, sha256=sha256(path)) | |
| def read_tensor(base, descriptor): | |
| path = (base / descriptor["file"]).resolve() | |
| if not path.is_relative_to(base.resolve()) or not path.is_file(): | |
| raise ValueError("Tensor path must resolve to a file inside its fixture/result directory") | |
| dtype = DTYPES.get(descriptor.get("dtype")) | |
| shape = descriptor.get("shape") | |
| if dtype is None or not isinstance(shape, list) or not shape or any(not isinstance(v, int) or v < 1 for v in shape): | |
| raise ValueError("Invalid portable tensor dtype or shape") | |
| count = 1 | |
| for dimension in shape: | |
| count *= dimension | |
| if count * dtype.itemsize != descriptor.get("bytes") or path.stat().st_size != descriptor["bytes"]: | |
| raise ValueError("Portable tensor byte count does not match shape") | |
| if sha256(path) != descriptor.get("sha256"): | |
| raise ValueError("Portable tensor checksum mismatch") | |
| result = np.fromfile(path, dtype=dtype).reshape(shape) | |
| return result.astype(bool) if descriptor["dtype"] == "bool" else result | |
| def export_fixtures(output, source, cache, upstream): | |
| output.mkdir(parents=True, exist_ok=True) | |
| entries = [] | |
| for case in fixtures(extended=True): | |
| name, context, horizon, kwargs = case | |
| directory = output / name | |
| directory.mkdir(exist_ok=True) | |
| inputs = {"context": write_tensor(directory, "context.f32le", context)} | |
| options = {"symmetric_averaging": True, "clip_nonnegative": True} | |
| for key, value in kwargs.items(): | |
| if isinstance(value, np.ndarray): | |
| extension = "f64le" if value.dtype == np.float64 else "f32le" | |
| inputs[key] = write_tensor(directory, f"{key}.{extension}", value) | |
| else: | |
| options[key] = value | |
| references, provenance = {}, {} | |
| for variant in VARIANTS: | |
| arrays, metadata = load_reference_cache(cache, variant, case, source, upstream) | |
| if set(arrays) != {"mlx", "upstream"}: | |
| raise ValueError("Swift fixtures require both MLX and original Mistral reference arrays") | |
| references[variant] = { | |
| key: write_tensor(directory, f"references/{variant}/{key}.f32le", value) | |
| for key, value in arrays.items() | |
| } | |
| provenance[variant] = metadata | |
| quantile_levels = json.loads((source / "5M" / "config.json").read_text())["quantile_knots"] | |
| document = dict( | |
| format="timestral-swift-fixture-v1", name=name, prediction_length=horizon, | |
| inputs=inputs, options=options, quantile_levels=quantile_levels, | |
| references=references, provenance=provenance, original_fixture_signature=fixture_signature(case), | |
| ) | |
| path = directory / "fixture.json" | |
| atomic_json(path, document) | |
| entries.append(dict(name=name, file=path.relative_to(output).as_posix(), sha256=sha256(path))) | |
| manifest = dict(format="timestral-swift-fixtures-v1", byte_order="little-endian", | |
| variants=list(VARIANTS), fixtures=entries, exporter_sha256=sha256(Path(__file__)), | |
| notes="Binary FP32/FP64 arrays preserve NaNs and large categorical codes; no reference models loaded during export.") | |
| atomic_json(output / "manifest.json", manifest) | |
| return manifest | |
| def swift_provenance(root, binary, worker, bundle): | |
| files = sorted((root / "swift" / "Sources").rglob("*.swift")) | |
| files += [root / "Package.swift"] | |
| return dict( | |
| swift_sources_sha256={path.relative_to(root).as_posix(): sha256(path) for path in files}, | |
| binary_sha256=sha256(binary), worker_binary_sha256=sha256(worker), | |
| bundle_manifest_sha256=sha256(bundle / "manifest.json"), | |
| validator_sha256=sha256(Path(__file__)), | |
| ) | |
| def validate_variant(variant, *, root, bundles, fixture_root, binary, worker, reports, | |
| selected=None, timeout=120, cache_mode="adaptive"): | |
| manifest = json.loads((fixture_root / "manifest.json").read_text()) | |
| if manifest.get("format") != "timestral-swift-fixtures-v1": | |
| raise ValueError("Unsupported Swift fixture collection") | |
| entries = {entry["name"]: entry for entry in manifest["fixtures"]} | |
| names = list(dict.fromkeys(selected)) if selected else list(FIXTURE_NAMES) | |
| before = swift_provenance(root, binary, worker, bundles / variant) | |
| report = dict( | |
| format="timestral-swift-validation-v1", variant=variant, status="running", passed=False, | |
| requested_fixtures=names, expected_fixture_count=len(names), acceptance=ACCEPTANCE, | |
| started_at=datetime.now(timezone.utc).isoformat(), provenance=before, | |
| fixture_manifest_sha256=sha256(fixture_root / "manifest.json"), | |
| environment={"platform": platform.platform(), "python": platform.python_version(), | |
| "swift": subprocess.check_output(["xcrun", "swift", "--version"], text=True).strip()}, | |
| fixtures=[], notes="Native Swift preprocessing, queue/worker runtime, and postprocessing; compared against both pinned references. Timings are not controlled benchmarks.", | |
| ) | |
| def checkpoint(status=None, **updates): | |
| if status: | |
| report["status"] = status | |
| report.update(updates, updated_at=datetime.now(timezone.utc).isoformat()) | |
| report["provenance_after"] = swift_provenance(root, binary, worker, bundles / variant) | |
| unchanged = report["provenance_after"] == before | |
| done = {record["fixture"] for record in report["fixtures"] if record.get("passed")} | |
| report["provenance_unchanged"] = unchanged | |
| report["passed"] = bool(report["status"] == "completed" and unchanged and done == set(names)) | |
| report["extended_suite_passed"] = bool(report["passed"] and done == set(FIXTURE_NAMES)) | |
| report["completed_fixture_count"] = len(report["fixtures"]) | |
| atomic_json(reports / f"{variant}.json", report) | |
| status = "failed" | |
| checkpoint(active_fixture=None) | |
| try: | |
| for name in names: | |
| entry = entries[name] | |
| fixture_path = fixture_root / entry["file"] | |
| if sha256(fixture_path) != entry["sha256"]: | |
| raise ValueError(f"Fixture manifest checksum mismatch: {name}") | |
| fixture = json.loads(fixture_path.read_text()) | |
| # Validate raw input bytes before handing them to the native executable. | |
| for descriptor in fixture["inputs"].values(): | |
| read_tensor(fixture_path.parent, descriptor) | |
| result_path = reports / "outputs" / variant / name / "forecast.json" | |
| result_path.parent.mkdir(parents=True, exist_ok=True) | |
| checkpoint(active_fixture=name) | |
| command = [str(binary), "validate-fixture", "--bundle", str(bundles / variant), | |
| "--fixture", str(fixture_path), "--output", str(result_path), | |
| "--worker", str(worker), "--cache-mode", cache_mode] | |
| started = time.perf_counter() | |
| completed = run_cli(command, timeout) | |
| if completed.returncode: | |
| raise RuntimeError(f"Swift CLI exited {completed.returncode}: {completed.stderr[-4000:]}") | |
| result = json.loads(result_path.read_text()) | |
| if result.get("format") != "timestral-swift-forecast-v1": | |
| raise ValueError("Swift CLI returned an unsupported result format") | |
| actual = read_tensor(result_path.parent, result["quantiles"]) | |
| if result["quantile_levels"] != fixture["quantile_levels"]: | |
| raise ValueError("Swift quantile levels differ from the reference") | |
| comparisons = { | |
| backend: compare(actual, read_tensor(fixture_path.parent, descriptor)) | |
| for backend, descriptor in fixture["references"][variant].items() | |
| } | |
| passed = set(comparisons) == {"mlx", "upstream"} and all(v["passed"] for v in comparisons.values()) | |
| record = dict(fixture=name, passed=passed, seconds=time.perf_counter() - started, | |
| comparisons=comparisons, swift_result=result, | |
| fixture_sha256=entry["sha256"], result_sha256=sha256(result_path)) | |
| report["fixtures"].append(record) | |
| checkpoint(active_fixture=None) | |
| print(f"{variant} / {name}: passed={passed}, nRMSE={comparisons['upstream'].get('normalized_rmse')}", flush=True) | |
| if not passed: | |
| report["error"] = "Stopped at the first numerical failure" | |
| break | |
| else: | |
| status = "completed" | |
| except BaseException as exc: | |
| report["error"] = f"{type(exc).__name__}: {exc}" | |
| status = "interrupted" if isinstance(exc, KeyboardInterrupt) else "failed" | |
| checkpoint(status) | |
| if isinstance(exc, (KeyboardInterrupt, SystemExit)): | |
| raise | |
| checkpoint(status, active_fixture=None) | |
| return report["passed"] | |
| def parse_args(argv=None): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| subparsers = parser.add_subparsers(dest="command", required=True) | |
| export = subparsers.add_parser("export-fixtures", help="Export the nine fixtures and both references as portable binary tensors") | |
| export.add_argument("--output", type=Path, default=ROOT / "swift" / "Fixtures") | |
| export.add_argument("--source", type=Path, default=ROOT.parent / "timestral-mlx") | |
| export.add_argument("--reference-cache", type=Path, default=ROOT / "reports" / "reference-cache") | |
| export.add_argument("--upstream", type=Path, default=ROOT / "references" / "timestral-a534bdbd244d64f3d9d743f46c7bf6fc330ed4ff" / "src") | |
| validate = subparsers.add_parser("validate", help="Run the compiled Swift CLI and compare both portable references") | |
| validate.add_argument("variants", nargs="*", metavar="VARIANT") | |
| validate.add_argument("--binary", type=Path, default=default_executable("timestral")) | |
| validate.add_argument("--worker", type=Path, default=default_executable("timestral-worker")) | |
| validate.add_argument("--bundles", type=Path, default=ROOT) | |
| validate.add_argument("--fixture-root", type=Path, default=ROOT / "swift" / "Fixtures") | |
| validate.add_argument("--reports", type=Path, default=ROOT / "reports" / "swift") | |
| validate.add_argument("--fixture", action="append", choices=FIXTURE_NAMES) | |
| validate.add_argument("--timeout", type=float, default=120) | |
| validate.add_argument("--cache-mode", choices=("adaptive", "warm", "sequential"), default="adaptive") | |
| args = parser.parse_args(argv) | |
| if args.command == "validate": | |
| args.variants = args.variants or list(VARIANTS) | |
| if any(variant not in VARIANTS for variant in args.variants): | |
| parser.error(f"variants must be selected from {', '.join(VARIANTS)}") | |
| return args | |
| def main(): | |
| args = parse_args() | |
| if args.command == "export-fixtures": | |
| result = export_fixtures(args.output, args.source, args.reference_cache, args.upstream) | |
| print(f"Exported {len(result['fixtures'])} portable fixtures with all four variants' references to {args.output}") | |
| return | |
| for variant in args.variants: | |
| passed = validate_variant( | |
| variant, root=ROOT, bundles=args.bundles.resolve(), fixture_root=args.fixture_root.resolve(), | |
| binary=args.binary.resolve(), worker=args.worker.resolve(), reports=args.reports.resolve(), | |
| selected=args.fixture, timeout=args.timeout, cache_mode=args.cache_mode, | |
| ) | |
| if not passed: | |
| raise SystemExit("Swift validation failed; inspect the checkpointed report") | |
| if __name__ == "__main__": | |
| main() | |