timestral-coreai / scripts /benchmark_swift.py
jbilcke's picture
Publish Timestral CoreAI models, native Swift SDK and validated Python runtime
506f40d verified
Raw History Blame Contribute Delete
4.39 kB
"""Run the native Swift workload under fixed budgets and bind its report to source/assets."""
from __future__ import annotations
import argparse
from datetime import datetime, timezone
import json
from pathlib import Path
import subprocess
import sys
from validate_swift import ROOT, atomic_json, default_executable, sha256, swift_provenance
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--binary", type=Path, default=default_executable("timestral"))
parser.add_argument("--worker", type=Path, default=default_executable("timestral-worker"))
parser.add_argument("--bundle", type=Path, default=ROOT / "5M")
parser.add_argument("--requests", type=int, default=1000)
parser.add_argument("--series", type=int, default=8)
parser.add_argument("--concurrency", type=int, default=8)
parser.add_argument("--max-workers", type=int, default=1)
parser.add_argument("--output", type=Path, default=ROOT / "reports/swift-stress-5M.json")
args = parser.parse_args()
if min(args.requests, args.series, args.concurrency, args.max_workers) < 1:
parser.error("request, series and concurrency counts must be positive")
binary, worker, bundle, output = (p.resolve() for p in (args.binary, args.worker, args.bundle, args.output))
output.parent.mkdir(parents=True, exist_ok=True)
guard_path = output.with_name("guard-" + output.name)
provenance_path = output.with_name(output.stem + "-provenance.json")
guard_script = ROOT / "scripts/run_guarded.py"
before = swift_provenance(ROOT, binary, worker, bundle)
report = dict(format="timestral-swift-benchmark-provenance-v1", passed=False,
status="running", started_at=datetime.now(timezone.utc).isoformat(),
provenance=before, benchmark_script_sha256=sha256(Path(__file__)),
guard_script_sha256=sha256(guard_script))
atomic_json(provenance_path, report)
# A failed launch must not accidentally certify an earlier successful run.
for path in (output, guard_path):
if path.exists():
path.replace(path.with_name(path.name + ".previous"))
command = [sys.executable, str(guard_script), "--report", str(guard_path), "--",
str(binary), "stress", "--bundle", str(bundle), "--worker", str(worker),
"--requests", str(args.requests), "--series", str(args.series),
"--concurrency", str(args.concurrency), "--checkpoint-every", "100",
"--max-workers", str(args.max_workers),
"--output", str(output)]
report["command"] = command
try:
completed = subprocess.run(command, check=False)
report["exit_code"] = completed.returncode
result = json.loads(output.read_text()) if output.exists() else {}
guard = json.loads(guard_path.read_text()) if guard_path.exists() else {}
report["provenance_after"] = swift_provenance(ROOT, binary, worker, bundle)
report["provenance_unchanged"] = report["provenance_after"] == before
report["report_sha256"] = sha256(output) if output.exists() else None
report["guard_report_sha256"] = sha256(guard_path) if guard_path.exists() else None
report["passed"] = bool(
completed.returncode == 0 and report["provenance_unchanged"]
and guard.get("completed") and guard.get("exit_code") == 0
and guard.get("stopped_reason") is None
and result.get("status") == "completed" and result.get("passed")
and result.get("requested_requests") == args.requests
and result.get("completed_requests") == args.requests
and result.get("distinct_series") == min(args.requests, args.series)
and result.get("configuration", {}).get("max_concurrent_workers") == args.max_workers
)
report["status"] = "completed" if report["passed"] else "failed"
except BaseException as error:
report.update(status="failed", error=f"{type(error).__name__}: {error}")
raise
finally:
report["finished_at"] = datetime.now(timezone.utc).isoformat()
atomic_json(provenance_path, report)
print(json.dumps(dict(passed=report["passed"], provenance=str(provenance_path))), flush=True)
return 0 if report["passed"] else 1
if __name__ == "__main__":
raise SystemExit(main())