jbilcke's picture
Publish Timestral CoreAI models, native Swift SDK and validated Python runtime
506f40d verified
Raw History Blame Contribute Delete
7.66 kB
"""Offline native CoreAI smoke checks, including the full trained context length."""
from __future__ import annotations
import argparse
import json
import platform
import signal
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
import numpy as np
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from timestral_coreai import TimestralForecaster
from scripts.validate import ValidationInterrupted, atomic_json, code_hashes, interrupt_handler, sha256
VARIANTS = ("5M", "28M", "123M", "209M")
def parse_args(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("variants", nargs="*", metavar="VARIANT",
help="5M, 28M, 123M, or 209M (default: all four)")
parser.add_argument("--bundles", type=Path, default=ROOT)
parser.add_argument("--report", type=Path, default=ROOT / "reports" / "full-context.json")
parser.add_argument("--compute-unit", choices=("cpu",), default="cpu",
help="Use the supported native CoreAI CPU execution policy")
args = parser.parse_args(argv)
args.variants = args.variants or list(VARIANTS)
if any(variant not in VARIANTS for variant in args.variants):
parser.error(f"variants must be selected from {', '.join(VARIANTS)}")
return args
def run_smoke(variants, bundles, report_path, compute_unit="cpu"):
before = code_hashes(ROOT / "timestral_coreai")
report = dict(
platform=platform.platform(), status="running", passed=False, requested_variants=variants,
started_at=datetime.now(timezone.utc).isoformat(), runtime_sha256=before,
smoke_script_sha256=sha256(Path(__file__)), variants=[],
notes="Full context and exact truncation through the public forecasting API. Run one variant per guarded process. Timings include first-call specialization.",
)
def checkpoint(status=None, **updates):
if status:
report["status"] = status
report.update(updates, updated_at=datetime.now(timezone.utc).isoformat())
after = code_hashes(ROOT / "timestral_coreai")
report.update(runtime_sha256_after=after, provenance_unchanged=before == after)
completed = {r["variant"] for r in report["variants"] if r.get("passed")}
report["remaining_variants"] = [v for v in variants if v not in completed]
report["passed"] = bool(report["status"] == "completed" and before == after and set(variants) == completed)
report["all_variants_passed"] = bool(report["passed"] and completed == set(VARIANTS))
atomic_json(report_path, report)
forecaster, pending_interruption = None, None
status = "failed"
checkpoint(active_variant=None, active_stage="initializing")
try:
for variant in variants:
checkpoint(active_variant=variant, active_stage="loading_model")
manifest_hash = sha256(bundles / variant / "manifest.json")
forecaster = TimestralForecaster.from_pretrained(bundles / variant, compute_unit=compute_unit)
runtime = forecaster.model.runtime
backend = getattr(runtime, "backend", "unknown")
actual_compute_unit = getattr(runtime, "compute_unit", "unknown")
if backend != "macos-coreai" or actual_compute_unit != compute_unit:
raise RuntimeError("Smoke check requires the requested compute policy on native macOS CoreAI.")
context_length = forecaster.config.context_length
t = np.arange(context_length + 17, dtype=np.float32)
context = np.sin(t * 2 * np.pi / 24) + .0001 * t
checkpoint(active_stage="full_context_inference")
start = time.perf_counter()
actual = forecaster.forecast(context, 32, symmetric_averaging=False,
clip_nonnegative=False).quantiles
seconds = time.perf_counter() - start
checkpoint(active_stage="truncated_context_inference")
second_start = time.perf_counter()
truncated = forecaster.forecast(context[-context_length:], 32,
symmetric_averaging=False, clip_nonnegative=False).quantiles
assert actual.shape == (1, 1, 32, len(forecaster.quantile_levels))
assert np.isfinite(actual).all()
assert (np.diff(actual, axis=-1) >= 0).all()
np.testing.assert_array_equal(actual, truncated)
assert "torch" not in sys.modules, "inference imported PyTorch"
assert "mlx.core" not in sys.modules, "inference imported MLX"
final_manifest_hash = sha256(bundles / variant / "manifest.json")
if final_manifest_hash != manifest_hash:
raise RuntimeError("Bundle changed during the smoke check.")
record = dict(variant=variant, context_steps=context_length,
input_steps=context_length + 17, horizon=32, finite=True,
monotone=True, truncation_exact=True, torch_imported=False,
mlx_imported=False, import_check_scope="controller_process",
seconds=seconds, truncated_seconds=time.perf_counter() - second_start,
requested_compute_unit=compute_unit, actual_compute_unit=actual_compute_unit,
backend=backend, execution_mode=getattr(runtime, "execution_mode", "single-native-model"),
bundle_manifest_sha256=manifest_hash, passed=True)
worker_imports = getattr(runtime, "worker_imports", None)
if worker_imports is not None:
record["worker_imports"] = worker_imports
if any(worker_imports.values()):
raise RuntimeError("A native worker imported a reference inference framework.")
report["variants"].append(record)
checkpoint(active_stage="cleanup")
print(json.dumps(record), flush=True)
forecaster.close()
forecaster = None
status = "completed"
except (KeyboardInterrupt, ValidationInterrupted) as exc:
status, pending_interruption = "interrupted", exc
report["error"] = f"{type(exc).__name__}: {exc}"
except Exception as exc:
report["error"] = f"{type(exc).__name__}: {exc}"
finally:
checkpoint(status, interrupted_variant=report.get("active_variant"), active_stage="cleanup")
try:
if forecaster is not None:
forecaster.close()
except Exception as exc:
if status != "interrupted":
status = "failed"
report["cleanup_error"] = f"{type(exc).__name__}: {exc}"
checkpoint(status, active_variant=None, active_stage=None)
if pending_interruption is not None:
raise pending_interruption
return report["passed"]
def main():
args = parse_args()
previous_handlers = {signum: signal.signal(signum, interrupt_handler)
for signum in (signal.SIGINT, signal.SIGTERM)}
try:
passed = run_smoke(args.variants, args.bundles, args.report, args.compute_unit)
except ValidationInterrupted as exc:
raise SystemExit(128 + exc.signum) from exc
except KeyboardInterrupt as exc:
raise SystemExit(130) from exc
finally:
for signum, handler in previous_handlers.items():
signal.signal(signum, handler)
if not passed:
raise SystemExit("Native CoreAI smoke check failed; inspect the checkpointed report.")
if __name__ == "__main__":
main()