Download scripts/validate.py from jbilcke/timestral-coreai: direct link, hf CLI and curl.
- Browser
- Download file 25.6 kB
-
https://huggingface.co/jbilcke/timestral-coreai/resolve/main/scripts/validate.py
- Command line
-
hf download hf://jbilcke/timestral-coreai/scripts/validate.py
-
curl -L -o validate.py https://huggingface.co/jbilcke/timestral-coreai/resolve/main/scripts/validate.py
25.6 kB
| """Compare native CoreAI forecasts with the original, local Timestral MLX port. | |
| These deterministic synthetic fixtures test implementation parity, not forecasting | |
| accuracy on a real-world benchmark. Run after scripts/convert.py. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import gc | |
| import hashlib | |
| import importlib.metadata | |
| import json | |
| import os | |
| import platform | |
| import signal | |
| import sys | |
| import time | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| import numpy as np | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT)) | |
| VARIANTS = ("5M", "28M", "123M", "209M") | |
| ACCEPTANCE = {"normalized_rmse_max": 1e-4, "normalized_max_error_max": 2e-3} | |
| STANDARD_FIXTURES = ( | |
| "univariate_missing_96", "multivariate_batched_65", "numeric_covariates_97", | |
| "categorical_covariates_97", "autoregressive_1057", "constant_65", "all_missing_65", | |
| ) | |
| FIXTURE_NAMES = STANDARD_FIXTURES + ( | |
| "singleton_context_1_horizon_1", "full_context_32768_horizon_32", | |
| ) | |
| def sha256(path): | |
| digest = hashlib.sha256() | |
| with Path(path).open("rb") as stream: | |
| for block in iter(lambda: stream.read(8 * 1024 * 1024), b""): | |
| digest.update(block) | |
| return digest.hexdigest() | |
| def code_hashes(package): | |
| return {p.name: sha256(p) for p in sorted(package.glob("*.py")) if not p.name.startswith("._")} | |
| def fixtures(extended=False): | |
| """The seven original fixtures, optionally followed by the input-shape bounds.""" | |
| rng = np.random.default_rng(20261020) | |
| def signal(b, v, t): | |
| return (3 * np.sin(np.arange(t) * 2 * np.pi / 24) | |
| + rng.normal(0, .2, (b, v, t)).cumsum(-1) + 10).astype(np.float32) | |
| x = signal(1, 1, 257) | |
| x[..., 10:16] = np.nan | |
| yield "univariate_missing_96", x, 96, {} | |
| x = signal(2, 2, 129) | |
| x[0, 0, :40] = np.nan | |
| x[1, 1, 60:70] = np.nan | |
| yield "multivariate_batched_65", x, 65, {} | |
| x = signal(1, 1, 193) | |
| past, known = signal(1, 2, 193), signal(1, 2, 193 + 97) | |
| past[0, 0, 20:28] = np.nan | |
| known[:, 1] = np.arange(193 + 97) % 2 | |
| known[0, 0, 210:215] = np.nan | |
| yield "numeric_covariates_97", x, 97, dict(past_covariates=past, known_covariates=known) | |
| codes = (np.arange(193 + 97) % 4 + 2**26).astype(np.float64)[None, None] | |
| codes[..., 5:9] = np.nan | |
| yield "categorical_covariates_97", x, 97, dict( | |
| categorical_past_covariates=codes[..., :193], categorical_known_covariates=codes) | |
| x = signal(1, 1, 257) | |
| yield "autoregressive_1057", x, 1057, dict(symmetric_averaging=False) | |
| yield "constant_65", np.full(65, 5, np.float32), 65, {} | |
| yield "all_missing_65", np.full(65, np.nan, np.float32), 65, {} | |
| if extended: | |
| yield "singleton_context_1_horizon_1", np.array([2.5], np.float32), 1, {} | |
| t = np.arange(32768, dtype=np.float32) | |
| x = np.sin(t * 2 * np.pi / 24) + .0001 * t | |
| yield "full_context_32768_horizon_32", x, 32, dict( | |
| symmetric_averaging=False, clip_nonnegative=False) | |
| def compare(actual, expected): | |
| if actual.shape != expected.shape: | |
| return {"passed": False, "actual_shape": list(actual.shape), | |
| "expected_shape": list(expected.shape), "error": "output shape mismatch"} | |
| finite = bool(np.isfinite(actual).all() and np.isfinite(expected).all()) | |
| monotone = bool((np.diff(actual, axis=-1) >= 0).all()) | |
| if not finite: | |
| return {"passed": False, "finite": False, "monotone": monotone} | |
| delta = actual.astype(np.float64) - expected.astype(np.float64) | |
| reference_rms = float(np.sqrt(np.mean(expected.astype(np.float64) ** 2))) | |
| rmse = float(np.sqrt(np.mean(delta ** 2))) | |
| maximum = float(np.max(np.abs(delta))) | |
| scale = max(reference_rms, 1e-6) | |
| normalized_rmse, normalized_max = rmse / scale, maximum / scale | |
| peak = float(np.max(np.abs(expected))) | |
| # JSON has no infinity: exact agreement is represented separately. | |
| psnr = float(20 * np.log10(max(peak, 1e-6) / rmse)) if rmse else None | |
| return dict( | |
| passed=bool(monotone and normalized_rmse <= ACCEPTANCE["normalized_rmse_max"] | |
| and normalized_max <= ACCEPTANCE["normalized_max_error_max"]), | |
| finite=finite, monotone=monotone, shape=list(actual.shape), | |
| max_absolute_error=maximum, rmse=rmse, reference_rms=reference_rms, | |
| normalized_rmse=normalized_rmse, normalized_max_error=normalized_max, | |
| psnr_db=psnr, exact_match=bool(rmse == 0), | |
| ) | |
| def upstream_forecaster(source_bundle): | |
| """Use original Mistral architecture and unchanged tensors; reverse name-only rename.""" | |
| import torch | |
| from safetensors.torch import load_file | |
| from timestral import TimestralConfig, TimestralForecaster, TimestralModel | |
| torch.set_num_threads(1) | |
| model = TimestralModel(TimestralConfig.load(source_bundle / "config.json")) | |
| state = load_file(source_bundle / "model.safetensors", device="cpu") | |
| model.load_state_dict({name.replace(".gate.", "._gate."): value | |
| for name, value in state.items()}, strict=True) | |
| return TimestralForecaster(model.eval()) | |
| def provenance_snapshot(source, source_bundle, bundle, upstream): | |
| snapshot = dict( | |
| source_manifest_sha256=sha256(source_bundle / "manifest.json"), | |
| bundle_manifest_sha256=sha256(bundle / "manifest.json"), | |
| source_runtime_sha256=code_hashes(source / "timestral_mlx"), | |
| runtime_sha256=code_hashes(ROOT / "timestral_coreai"), | |
| validation_script_sha256=sha256(Path(__file__)), | |
| ) | |
| if upstream: | |
| snapshot["upstream_runtime_sha256"] = code_hashes(upstream / "timestral") | |
| return snapshot | |
| def select_cases(extended=False, selected=None): | |
| available = {case[0]: case for case in fixtures(extended=extended or bool(selected))} | |
| names = list(dict.fromkeys(selected)) if selected else list(available) | |
| unknown = set(names) - set(available) | |
| if unknown: | |
| raise ValueError(f"Unknown fixtures: {sorted(unknown)}") | |
| return [available[name] for name in names] | |
| def fixture_signature(case): | |
| def describe(value): | |
| if isinstance(value, np.ndarray): | |
| array = np.ascontiguousarray(value) | |
| return dict(shape=list(array.shape), dtype=array.dtype.str, | |
| sha256=hashlib.sha256(array.tobytes()).hexdigest()) | |
| return value.item() if isinstance(value, np.generic) else value | |
| name, context, horizon, kwargs = case | |
| return dict(name=name, context=describe(context), prediction_length=horizon, | |
| kwargs={key: describe(value) for key, value in sorted(kwargs.items())}) | |
| def reference_provenance(source, variant, upstream=None): | |
| provenance = dict( | |
| source_manifest_sha256=sha256(source / variant / "manifest.json"), | |
| source_runtime_sha256=code_hashes(source / "timestral_mlx"), | |
| ) | |
| if upstream: | |
| provenance["upstream_runtime_sha256"] = code_hashes(upstream / "timestral") | |
| provenance["upstream_requested_revision"] = "a534bdbd244d64f3d9d743f46c7bf6fc330ed4ff" | |
| return provenance | |
| def atomic_json(path, data): | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| temporary = path.with_suffix(path.suffix + ".tmp") | |
| with temporary.open("w") as stream: | |
| json.dump(data, stream, indent=2, allow_nan=False) | |
| stream.write("\n") | |
| stream.flush() | |
| os.fsync(stream.fileno()) | |
| temporary.replace(path) | |
| def write_reference_cache(cache, variant, case, arrays, provenance, versions, expected_names): | |
| directory = cache / variant | |
| directory.mkdir(parents=True, exist_ok=True) | |
| path = directory / f"{case[0]}.npz" | |
| temporary = path.with_suffix(".npz.tmp") | |
| with temporary.open("wb") as stream: | |
| np.savez(stream, **arrays) | |
| stream.flush() | |
| os.fsync(stream.fileno()) | |
| temporary.replace(path) | |
| metadata = dict( | |
| format="timestral-reference-cache-v1", variant=variant, | |
| fixture=fixture_signature(case), provenance=provenance, environment=versions, | |
| expected_references=expected_names, references=sorted(arrays), | |
| complete=set(arrays) == set(expected_names), | |
| arrays_sha256=sha256(path), arrays_bytes=path.stat().st_size, | |
| ) | |
| atomic_json(path.with_suffix(".json"), metadata) | |
| def load_reference_cache(cache, variant, case, source, upstream=None): | |
| path = cache / variant / f"{case[0]}.npz" | |
| metadata = json.loads(path.with_suffix(".json").read_text()) | |
| if (metadata.get("format") != "timestral-reference-cache-v1" | |
| or metadata.get("variant") != variant or not metadata.get("complete")): | |
| raise ValueError(f"Incomplete or incompatible reference cache: {path}") | |
| if metadata.get("fixture") != fixture_signature(case): | |
| raise ValueError(f"Reference cache input/horizon/covariate signature mismatch: {case[0]}") | |
| current = reference_provenance(source, variant, upstream) | |
| if any(metadata.get("provenance", {}).get(key) != value for key, value in current.items()): | |
| raise ValueError(f"Reference cache source provenance mismatch: {case[0]}") | |
| if path.stat().st_size != metadata.get("arrays_bytes") or sha256(path) != metadata.get("arrays_sha256"): | |
| raise ValueError(f"Reference cache integrity check failed: {path}") | |
| with np.load(path, allow_pickle=False) as stored: | |
| arrays = {name: stored[name].copy() for name in stored.files} | |
| if set(arrays) != set(metadata.get("references", [])) or "mlx" not in arrays: | |
| raise ValueError(f"Reference cache array inventory mismatch: {path}") | |
| if upstream and "upstream" not in arrays: | |
| raise ValueError(f"Reference cache lacks the requested original Mistral reference: {path}") | |
| if any(value.dtype != np.float32 or not np.isfinite(value).all() for value in arrays.values()): | |
| raise ValueError(f"Reference cache must contain finite FP32 arrays: {path}") | |
| return arrays, metadata | |
| class ValidationInterrupted(BaseException): | |
| def __init__(self, signum): | |
| self.signum = signum | |
| super().__init__(f"received signal {signum}") | |
| def interrupt_handler(signum, frame): | |
| raise ValidationInterrupted(signum) | |
| def runtime_environment(compute_unit, cached): | |
| return dict( | |
| python=platform.python_version(), platform=platform.platform(), numpy=np.__version__, | |
| coreai_core=importlib.metadata.version("coreai-core"), | |
| requested_compute_unit=compute_unit, reference_mode="cache" if cached else "live", | |
| USE_LOCAL_COREAI=os.environ.get("USE_LOCAL_COREAI"), | |
| USE_OS_COREAI=os.environ.get("USE_OS_COREAI"), | |
| ) | |
| def validate(variant, source, bundles, reports, upstream=None, extended=False, | |
| selected=None, keep_going=False, compute_unit="cpu", reference_cache=None): | |
| from timestral_coreai import TimestralForecaster | |
| source_bundle, bundle = source / variant, bundles / variant | |
| before = provenance_snapshot(source, source_bundle, bundle, upstream) | |
| cases = select_cases(extended, selected) | |
| names = [case[0] for case in cases] | |
| report = dict( | |
| variant=variant, reference="local Timestral MLX FP32", passed=False, | |
| status="running", extended=extended, requested_fixtures=names, | |
| validation_scope="selected_fixtures" if selected else "extended_suite" if extended else "standard_suite", | |
| expected_fixture_count=len(cases), fail_fast=not keep_going, | |
| acceptance=ACCEPTANCE, **before, | |
| environment=runtime_environment(compute_unit, reference_cache is not None), | |
| started_at=datetime.now(timezone.utc).isoformat(), fixtures=[], | |
| notes=("Acceptance covers only requested_fixtures. standard_suite_passed and extended_suite_passed " | |
| "are true only when all respective cases passed in this run. Synthetic implementation " | |
| "parity, not real-world forecasting accuracy. Timings include specialization/first-call " | |
| "overhead. PSNR uses peak absolute reference / RMSE; null exact_match means infinity."), | |
| ) | |
| if upstream: | |
| report["upstream"] = dict( | |
| reference="original Mistral Timestral CPU FP32", runtime_sha256=before["upstream_runtime_sha256"], | |
| requested_revision="a534bdbd244d64f3d9d743f46c7bf6fc330ed4ff", | |
| weight_transform="Only .gate. names changed back to ._gate.; trained tensors unchanged.") | |
| def checkpoint(status=None, **updates): | |
| if status: | |
| report["status"] = status | |
| report.update(updates) | |
| after = provenance_snapshot(source, source_bundle, bundle, upstream) | |
| unchanged = before == after | |
| report.update(provenance_after=after, provenance_unchanged=unchanged, | |
| updated_at=datetime.now(timezone.utc).isoformat()) | |
| done = {item["fixture"] for item in report["fixtures"] if item.get("completed")} | |
| passed_names = {item["fixture"] for item in report["fixtures"] if item.get("passed")} | |
| report["completed_fixture_count"] = len(done) | |
| report["remaining_fixtures"] = [name for name in names if name not in done] | |
| report["passed"] = bool(report["status"] == "completed" and unchanged and set(names) == passed_names) | |
| report["standard_suite_passed"] = bool(report["passed"] and set(STANDARD_FIXTURES) <= passed_names) | |
| report["extended_suite_passed"] = bool(report["passed"] and set(FIXTURE_NAMES) <= passed_names) | |
| atomic_json(reports / f"{variant}.json", report) | |
| print(f"Validating {variant}: {len(cases)} requested fixture(s), compute_unit={compute_unit}", flush=True) | |
| candidate = reference = original = mx = None | |
| pending_interruption = None | |
| status = "failed" | |
| checkpoint(active_fixture=None, active_stage="initializing") | |
| try: | |
| if reference_cache is None: | |
| checkpoint(active_stage="loading_mlx_reference") | |
| import mlx.core as mx | |
| from timestral_mlx import TimestralForecaster as MLXForecaster | |
| reference = MLXForecaster.from_pretrained(source_bundle) | |
| report["environment"].update(mlx=importlib.metadata.version("mlx"), mlx_device=str(mx.default_device())) | |
| if upstream: | |
| checkpoint(active_stage="loading_upstream_reference") | |
| original = upstream_forecaster(source_bundle) | |
| report["upstream"].update(torch=importlib.metadata.version("torch"), torch_threads=1) | |
| checkpoint(active_stage="loading_coreai") | |
| candidate = TimestralForecaster.from_pretrained(bundle, compute_unit=compute_unit) | |
| runtime = candidate.model.runtime | |
| report["environment"].update( | |
| actual_compute_unit=getattr(runtime, "compute_unit", "default"), | |
| backend=getattr(runtime, "backend", "unknown"), | |
| ) | |
| for case in cases: | |
| name, context, horizon, kwargs = case | |
| checkpoint(active_fixture=name, active_stage="loading_reference_cache" if reference_cache else "mlx_reference") | |
| cached_metadata = None | |
| start = time.perf_counter() | |
| if reference_cache: | |
| references, cached_metadata = load_reference_cache(reference_cache, variant, case, source, upstream) | |
| expected = references["mlx"] | |
| else: | |
| expected = reference.forecast(context, horizon, **kwargs).quantiles | |
| source_seconds = time.perf_counter() - start | |
| checkpoint(active_stage="coreai_inference") | |
| start = time.perf_counter() | |
| actual = candidate.forecast(context, horizon, **kwargs).quantiles | |
| record = dict(fixture=name, completed=True, source_seconds=source_seconds, | |
| coreai_seconds=time.perf_counter() - start, **compare(actual, expected)) | |
| if cached_metadata: | |
| record["reference_cache"] = cached_metadata | |
| record["source_seconds_kind"] = "cache_load" | |
| original_output = references.get("upstream") | |
| elif original is not None: | |
| checkpoint(active_stage="upstream_reference") | |
| start = time.perf_counter() | |
| original_output = original.forecast(context, horizon, **kwargs).quantiles.numpy() | |
| record["upstream_seconds"] = time.perf_counter() - start | |
| else: | |
| original_output = None | |
| if original_output is not None: | |
| record["upstream_comparison"] = compare(actual, original_output) | |
| record["passed"] = record["passed"] and record["upstream_comparison"]["passed"] | |
| report["fixtures"].append(record) | |
| checkpoint(active_fixture=None, active_stage="between_fixtures") | |
| print(f" {name}: nRMSE={record.get('normalized_rmse', float('nan')):.3g}, " | |
| f"PSNR={record.get('psnr_db')}, passed={record['passed']}", flush=True) | |
| if not record["passed"] and not keep_going: | |
| report["stopped_reason"] = "first numerical failure (default fail-fast policy)" | |
| break | |
| else: | |
| status = "completed" if all(item["passed"] for item in report["fixtures"]) else "failed" | |
| except (KeyboardInterrupt, ValidationInterrupted) as exc: | |
| status = "interrupted" | |
| pending_interruption = exc | |
| report["stopped_reason"] = f"{type(exc).__name__}: {exc}" | |
| except Exception as exc: | |
| report["error"] = f"{type(exc).__name__}: {exc}" | |
| report["stopped_reason"] = "exception; later fixtures were not attempted" | |
| finally: | |
| # Persist before cleanup: native cleanup can itself wait on outstanding work. | |
| checkpoint(status, active_stage="cleanup", interrupted_fixture=report.get("active_fixture")) | |
| try: | |
| if candidate is not None: | |
| candidate.close() | |
| candidate = reference = original = None | |
| gc.collect() | |
| if mx is not None: | |
| mx.clear_cache() | |
| except Exception as exc: | |
| status = "failed" if status != "interrupted" else status | |
| report["cleanup_error"] = f"{type(exc).__name__}: {exc}" | |
| checkpoint(status, active_fixture=None, active_stage=None) | |
| if pending_interruption is not None: | |
| raise pending_interruption | |
| return report["passed"] | |
| def prepare_reference_cache(variant, source, cache, upstream=None, extended=False, selected=None): | |
| """Prepare reference outputs with one model loaded at a time; never load CoreAI.""" | |
| import mlx.core as mx | |
| from timestral_mlx import TimestralForecaster as MLXForecaster | |
| cases = select_cases(extended, selected) | |
| provenance = reference_provenance(source, variant, upstream) | |
| versions = dict(python=platform.python_version(), platform=platform.platform(), | |
| numpy=np.__version__, mlx=importlib.metadata.version("mlx")) | |
| expected_names = ["mlx", "upstream"] if upstream else ["mlx"] | |
| outputs = {case[0]: {} for case in cases} | |
| status = dict(variant=variant, status="running", requested_fixtures=[case[0] for case in cases], | |
| expected_references=expected_names, provenance=provenance, completed_references=[]) | |
| def checkpoint(**updates): | |
| status.update(updates, updated_at=datetime.now(timezone.utc).isoformat()) | |
| atomic_json(cache / variant / "prepare-status.json", status) | |
| model = None | |
| checkpoint(active_stage="loading_mlx", active_fixture=None) | |
| try: | |
| model = MLXForecaster.from_pretrained(source / variant) | |
| for case in cases: | |
| name, context, horizon, kwargs = case | |
| checkpoint(active_stage="mlx_reference", active_fixture=name) | |
| outputs[name]["mlx"] = model.forecast(context, horizon, **kwargs).quantiles | |
| write_reference_cache(cache, variant, case, outputs[name], provenance, versions, expected_names) | |
| status["completed_references"].append(f"{name}:mlx") | |
| checkpoint() | |
| print(f"{variant}: cached MLX reference {name}", flush=True) | |
| model = None | |
| gc.collect() | |
| mx.clear_cache() | |
| if upstream: | |
| checkpoint(active_stage="loading_upstream", active_fixture=None) | |
| model = upstream_forecaster(source / variant) | |
| versions["torch"] = importlib.metadata.version("torch") | |
| for case in cases: | |
| name, context, horizon, kwargs = case | |
| checkpoint(active_stage="upstream_reference", active_fixture=name) | |
| outputs[name]["upstream"] = model.forecast(context, horizon, **kwargs).quantiles.numpy().copy() | |
| write_reference_cache(cache, variant, case, outputs[name], provenance, versions, expected_names) | |
| status["completed_references"].append(f"{name}:upstream") | |
| checkpoint() | |
| print(f"{variant}: cached original Mistral reference {name}", flush=True) | |
| if reference_provenance(source, variant, upstream) != provenance: | |
| raise RuntimeError("Reference source changed during cache preparation; discard this run's cache.") | |
| checkpoint(status="completed", active_stage=None, active_fixture=None) | |
| except (KeyboardInterrupt, ValidationInterrupted) as exc: | |
| checkpoint(status="interrupted", error=f"{type(exc).__name__}: {exc}") | |
| raise | |
| except Exception as exc: | |
| checkpoint(status="failed", error=f"{type(exc).__name__}: {exc}") | |
| raise | |
| finally: | |
| model = None | |
| gc.collect() | |
| mx.clear_cache() | |
| def parse_args(argv=None): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("variants", nargs="*", metavar="VARIANT", | |
| help="5M, 28M, 123M, or 209M (default: all four)") | |
| parser.add_argument("--source", type=Path, default=ROOT.parent / "timestral-mlx") | |
| parser.add_argument("--bundles", type=Path, default=ROOT) | |
| parser.add_argument("--reports", type=Path, default=ROOT / "reports", | |
| help="Report directory; each run replaces VARIANT.json, so use separate directories for subsets") | |
| parser.add_argument("--upstream", type=Path, | |
| help="Optional pinned original Mistral checkout (or its src directory) for a second reference") | |
| parser.add_argument("--extended", action="store_true", | |
| help="Also compare singleton context/horizon and full 32768-step context against references") | |
| parser.add_argument("--fixture", action="append", choices=FIXTURE_NAMES, | |
| help="Run only this case; repeat for more cases (explicit order retained)") | |
| parser.add_argument("--keep-going", action="store_true", | |
| help="Diagnostic mode: continue after numerical failures; default stops at the first failure") | |
| parser.add_argument("--compute-unit", choices=("cpu",), default="cpu", | |
| help="Native CoreAI CPU execution, the supported policy for this port") | |
| cache = parser.add_mutually_exclusive_group() | |
| cache.add_argument("--reference-cache", type=Path, | |
| help="Use verified cached reference arrays without importing/loading MLX or Torch") | |
| cache.add_argument("--prepare-reference-cache", type=Path, | |
| help="Prepare MLX and optional original Torch references sequentially, without CoreAI inference") | |
| args = parser.parse_args(argv) | |
| args.variants = args.variants or list(VARIANTS) | |
| if any(variant not in VARIANTS for variant in args.variants): | |
| parser.error(f"variants must be selected from {', '.join(VARIANTS)}") | |
| return args | |
| def main(): | |
| args = parse_args() | |
| source = args.source.expanduser().resolve() | |
| if not (source / "timestral_mlx" / "__init__.py").is_file(): | |
| raise SystemExit(f"source is not a Timestral MLX checkout: {source}") | |
| args.source = source | |
| upstream = args.upstream.expanduser().resolve() if args.upstream else None | |
| if upstream is not None: | |
| if (upstream / "src" / "timestral" / "__init__.py").is_file(): | |
| upstream = upstream / "src" | |
| if not (upstream / "timestral" / "__init__.py").is_file(): | |
| raise SystemExit(f"upstream does not contain the original timestral package: {upstream}") | |
| args.upstream = upstream | |
| sys.path.insert(0, str(args.source)) | |
| if args.upstream is not None: | |
| sys.path.insert(0, str(args.upstream)) | |
| previous_handlers = {signum: signal.signal(signum, interrupt_handler) | |
| for signum in (signal.SIGINT, signal.SIGTERM)} | |
| passed = [] | |
| try: | |
| for variant in args.variants: | |
| if args.prepare_reference_cache: | |
| prepare_reference_cache(variant, args.source, args.prepare_reference_cache, | |
| args.upstream, args.extended, args.fixture) | |
| passed.append(True) | |
| else: | |
| passed.append(validate( | |
| variant, args.source, args.bundles, args.reports, args.upstream, args.extended, | |
| args.fixture, args.keep_going, args.compute_unit, args.reference_cache)) | |
| if not passed[-1] and not args.keep_going: | |
| break | |
| except ValidationInterrupted as exc: | |
| raise SystemExit(128 + exc.signum) from exc | |
| except KeyboardInterrupt as exc: | |
| raise SystemExit(130) from exc | |
| finally: | |
| for signum, handler in previous_handlers.items(): | |
| signal.signal(signum, handler) | |
| if not all(passed): | |
| raise SystemExit("CoreAI did not pass numerical parity; inspect reports before using the exports.") | |
| if __name__ == "__main__": | |
| main() | |