imtk's picture
Publish validated FP32, INT8, and INT4 ONNX Runtime bundle
1910337 verified
Raw History Blame Contribute Delete
6.66 kB
"""Run reproducible validation stages and maintain the upload gate report."""
from __future__ import annotations
import argparse
import json
import os
import subprocess
import sys
import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from pathlib import Path
from .artifact import sha256_file
from .constants import NUMERICALLY_VALIDATED_MAX_TOKENS, SOURCE_REVISION
STAGE_COMMANDS = {
"client_compatibility": [
"tests/test_client_contract.py",
"tests/test_export_builder.py",
"tests/test_quantization.py",
"tests/test_validation.py",
],
"normal_parity": ["tests/test_parity.py", "-m", "parity and not slow"],
"slow_65536_parity": ["tests/test_parity.py", "-m", "slow"],
"structural_262144": ["tests/test_structure.py"],
"quantized_runtime": ["tests/test_quantized_runtime.py"],
}
def _accelerator_preflight(name: str) -> None:
"""Fail early with an actionable message for unavailable requested devices."""
if name not in {"normal_parity", "slow_65536_parity"}:
return
import onnxruntime as ort
import torch
requested_ort = os.environ.get("ORT_PARITY_DEVICE")
if requested_ort == "cuda" and "CUDAExecutionProvider" not in ort.get_available_providers():
raise SystemExit(
"ORT_PARITY_DEVICE=cuda was requested, but CUDAExecutionProvider is unavailable. "
"CUDA cannot run on macOS. On an NVIDIA Linux/Windows host, replace onnxruntime "
"with onnxruntime-gpu==1.30.0 and invoke the validator with `uv run --no-sync` "
"so uv does not restore the CPU wheel."
)
requested_torch = os.environ.get("PYTORCH_PARITY_DEVICE", requested_ort)
if requested_torch == "cuda" and not torch.cuda.is_available():
raise SystemExit(
"PYTORCH_PARITY_DEVICE=cuda was requested (or inherited from ORT_PARITY_DEVICE), "
"but PyTorch CUDA is unavailable. Run this certification on an NVIDIA CUDA host."
)
if requested_torch == "mps" and not torch.backends.mps.is_available():
raise SystemExit(
"PYTORCH_PARITY_DEVICE=mps was requested, but PyTorch reports MPS unavailable."
)
def _run_stage(name: str, artifact: Path) -> dict:
_accelerator_preflight(name)
environment = os.environ.copy()
environment["OPENTHAI_ONNX_ARTIFACT"] = str(artifact)
if name == "normal_parity":
environment["RUN_PARITY"] = "1"
if name == "slow_65536_parity":
environment["RUN_PARITY"] = "1"
environment["RUN_SLOW_65536"] = "1"
junit_path = artifact / f".{name}.junit.xml"
command = [
sys.executable,
"-m",
"pytest",
"-q",
*STAGE_COMMANDS[name],
f"--junitxml={junit_path}",
]
completed = subprocess.run(command, env=environment, text=True)
counts = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
if junit_path.is_file():
try:
root = ET.parse(junit_path).getroot()
suites = [root] if root.tag == "testsuite" else list(root.iter("testsuite"))
for suite in suites:
for key in counts:
counts[key] += int(suite.attrib.get(key, 0))
finally:
junit_path.unlink(missing_ok=True)
passed = (
completed.returncode == 0
and counts["tests"] > 0
and counts["failures"] == 0
and counts["errors"] == 0
and counts["skipped"] == 0
)
return {
"passed": passed,
"returncode": completed.returncode,
"command": command,
"test_counts": counts,
"completed_at": datetime.now(timezone.utc).isoformat(),
"failure_hint": (
"validation process was killed (signal 9), usually because host or GPU memory was exhausted"
if completed.returncode == -9
else None
),
}
def validate(artifact: Path, stages: list[str]) -> Path:
report_path = artifact / "validation-report.json"
manifest_path = artifact / "artifact-manifest.json"
manifest_sha256 = sha256_file(manifest_path)
if report_path.exists():
report = json.loads(report_path.read_text(encoding="utf-8"))
if report.get("artifact_manifest_sha256") != manifest_sha256:
report = {}
else:
report = {}
if not report:
report = {
"format_version": 1,
"source_revision": SOURCE_REVISION,
"artifact_manifest_sha256": manifest_sha256,
"numerical_parity_certified_through_tokens": NUMERICALLY_VALIDATED_MAX_TOKENS,
"lengths_above_65536": "structural validation only",
"stages": {},
}
for stage in stages:
print(f"\n=== {stage} ===", flush=True)
report["stages"][stage] = _run_stage(stage, artifact)
report["upload_gate"] = {
"required_stages": list(STAGE_COMMANDS),
"passed": all(report["stages"].get(item, {}).get("passed", False) for item in STAGE_COMMANDS),
}
report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
if not report["stages"][stage]["passed"]:
hint = report["stages"][stage].get("failure_hint")
suffix = f" ({hint})" if hint else ""
raise SystemExit(f"validation stage failed: {stage}{suffix}")
return report_path
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--artifact", type=Path, required=True)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("--client", action="store_true")
group.add_argument("--normal", action="store_true")
group.add_argument("--slow", action="store_true")
group.add_argument("--structural", action="store_true")
group.add_argument("--quantized", action="store_true")
group.add_argument("--all", action="store_true")
return parser
def main(argv: list[str] | None = None):
args = build_parser().parse_args(argv)
selection = {
"client": ["client_compatibility"],
"normal": ["client_compatibility", "normal_parity"],
"slow": ["slow_65536_parity"],
"structural": ["structural_262144"],
"quantized": ["client_compatibility", "structural_262144", "quantized_runtime"],
"all": list(STAGE_COMMANDS),
}
selected = next(key for key in selection if getattr(args, key))
report = validate(args.artifact.resolve(), selection[selected])
print(f"Validation report: {report}")
if __name__ == "__main__":
main()