"""Run reproducible validation stages and maintain the upload gate report.""" from __future__ import annotations import argparse import json import os import subprocess import sys import xml.etree.ElementTree as ET from datetime import datetime, timezone from pathlib import Path from .artifact import sha256_file from .constants import NUMERICALLY_VALIDATED_MAX_TOKENS, SOURCE_REVISION STAGE_COMMANDS = { "client_compatibility": [ "tests/test_client_contract.py", "tests/test_export_builder.py", "tests/test_quantization.py", "tests/test_validation.py", ], "normal_parity": ["tests/test_parity.py", "-m", "parity and not slow"], "slow_65536_parity": ["tests/test_parity.py", "-m", "slow"], "structural_262144": ["tests/test_structure.py"], "quantized_runtime": ["tests/test_quantized_runtime.py"], } def _accelerator_preflight(name: str) -> None: """Fail early with an actionable message for unavailable requested devices.""" if name not in {"normal_parity", "slow_65536_parity"}: return import onnxruntime as ort import torch requested_ort = os.environ.get("ORT_PARITY_DEVICE") if requested_ort == "cuda" and "CUDAExecutionProvider" not in ort.get_available_providers(): raise SystemExit( "ORT_PARITY_DEVICE=cuda was requested, but CUDAExecutionProvider is unavailable. " "CUDA cannot run on macOS. On an NVIDIA Linux/Windows host, replace onnxruntime " "with onnxruntime-gpu==1.30.0 and invoke the validator with `uv run --no-sync` " "so uv does not restore the CPU wheel." ) requested_torch = os.environ.get("PYTORCH_PARITY_DEVICE", requested_ort) if requested_torch == "cuda" and not torch.cuda.is_available(): raise SystemExit( "PYTORCH_PARITY_DEVICE=cuda was requested (or inherited from ORT_PARITY_DEVICE), " "but PyTorch CUDA is unavailable. Run this certification on an NVIDIA CUDA host." ) if requested_torch == "mps" and not torch.backends.mps.is_available(): raise SystemExit( "PYTORCH_PARITY_DEVICE=mps was requested, but PyTorch reports MPS unavailable." ) def _run_stage(name: str, artifact: Path) -> dict: _accelerator_preflight(name) environment = os.environ.copy() environment["OPENTHAI_ONNX_ARTIFACT"] = str(artifact) if name == "normal_parity": environment["RUN_PARITY"] = "1" if name == "slow_65536_parity": environment["RUN_PARITY"] = "1" environment["RUN_SLOW_65536"] = "1" junit_path = artifact / f".{name}.junit.xml" command = [ sys.executable, "-m", "pytest", "-q", *STAGE_COMMANDS[name], f"--junitxml={junit_path}", ] completed = subprocess.run(command, env=environment, text=True) counts = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0} if junit_path.is_file(): try: root = ET.parse(junit_path).getroot() suites = [root] if root.tag == "testsuite" else list(root.iter("testsuite")) for suite in suites: for key in counts: counts[key] += int(suite.attrib.get(key, 0)) finally: junit_path.unlink(missing_ok=True) passed = ( completed.returncode == 0 and counts["tests"] > 0 and counts["failures"] == 0 and counts["errors"] == 0 and counts["skipped"] == 0 ) return { "passed": passed, "returncode": completed.returncode, "command": command, "test_counts": counts, "completed_at": datetime.now(timezone.utc).isoformat(), "failure_hint": ( "validation process was killed (signal 9), usually because host or GPU memory was exhausted" if completed.returncode == -9 else None ), } def validate(artifact: Path, stages: list[str]) -> Path: report_path = artifact / "validation-report.json" manifest_path = artifact / "artifact-manifest.json" manifest_sha256 = sha256_file(manifest_path) if report_path.exists(): report = json.loads(report_path.read_text(encoding="utf-8")) if report.get("artifact_manifest_sha256") != manifest_sha256: report = {} else: report = {} if not report: report = { "format_version": 1, "source_revision": SOURCE_REVISION, "artifact_manifest_sha256": manifest_sha256, "numerical_parity_certified_through_tokens": NUMERICALLY_VALIDATED_MAX_TOKENS, "lengths_above_65536": "structural validation only", "stages": {}, } for stage in stages: print(f"\n=== {stage} ===", flush=True) report["stages"][stage] = _run_stage(stage, artifact) report["upload_gate"] = { "required_stages": list(STAGE_COMMANDS), "passed": all(report["stages"].get(item, {}).get("passed", False) for item in STAGE_COMMANDS), } report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") if not report["stages"][stage]["passed"]: hint = report["stages"][stage].get("failure_hint") suffix = f" ({hint})" if hint else "" raise SystemExit(f"validation stage failed: {stage}{suffix}") return report_path def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--artifact", type=Path, required=True) group = parser.add_mutually_exclusive_group(required=True) group.add_argument("--client", action="store_true") group.add_argument("--normal", action="store_true") group.add_argument("--slow", action="store_true") group.add_argument("--structural", action="store_true") group.add_argument("--quantized", action="store_true") group.add_argument("--all", action="store_true") return parser def main(argv: list[str] | None = None): args = build_parser().parse_args(argv) selection = { "client": ["client_compatibility"], "normal": ["client_compatibility", "normal_parity"], "slow": ["slow_65536_parity"], "structural": ["structural_262144"], "quantized": ["client_compatibility", "structural_262144", "quantized_runtime"], "all": list(STAGE_COMMANDS), } selected = next(key for key in selection if getattr(args, key)) report = validate(args.artifact.resolve(), selection[selected]) print(f"Validation report: {report}") if __name__ == "__main__": main()