File size: 4,391 Bytes
506f40d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | """Run the native Swift workload under fixed budgets and bind its report to source/assets."""
from __future__ import annotations
import argparse
from datetime import datetime, timezone
import json
from pathlib import Path
import subprocess
import sys
from validate_swift import ROOT, atomic_json, default_executable, sha256, swift_provenance
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--binary", type=Path, default=default_executable("timestral"))
parser.add_argument("--worker", type=Path, default=default_executable("timestral-worker"))
parser.add_argument("--bundle", type=Path, default=ROOT / "5M")
parser.add_argument("--requests", type=int, default=1000)
parser.add_argument("--series", type=int, default=8)
parser.add_argument("--concurrency", type=int, default=8)
parser.add_argument("--max-workers", type=int, default=1)
parser.add_argument("--output", type=Path, default=ROOT / "reports/swift-stress-5M.json")
args = parser.parse_args()
if min(args.requests, args.series, args.concurrency, args.max_workers) < 1:
parser.error("request, series and concurrency counts must be positive")
binary, worker, bundle, output = (p.resolve() for p in (args.binary, args.worker, args.bundle, args.output))
output.parent.mkdir(parents=True, exist_ok=True)
guard_path = output.with_name("guard-" + output.name)
provenance_path = output.with_name(output.stem + "-provenance.json")
guard_script = ROOT / "scripts/run_guarded.py"
before = swift_provenance(ROOT, binary, worker, bundle)
report = dict(format="timestral-swift-benchmark-provenance-v1", passed=False,
status="running", started_at=datetime.now(timezone.utc).isoformat(),
provenance=before, benchmark_script_sha256=sha256(Path(__file__)),
guard_script_sha256=sha256(guard_script))
atomic_json(provenance_path, report)
# A failed launch must not accidentally certify an earlier successful run.
for path in (output, guard_path):
if path.exists():
path.replace(path.with_name(path.name + ".previous"))
command = [sys.executable, str(guard_script), "--report", str(guard_path), "--",
str(binary), "stress", "--bundle", str(bundle), "--worker", str(worker),
"--requests", str(args.requests), "--series", str(args.series),
"--concurrency", str(args.concurrency), "--checkpoint-every", "100",
"--max-workers", str(args.max_workers),
"--output", str(output)]
report["command"] = command
try:
completed = subprocess.run(command, check=False)
report["exit_code"] = completed.returncode
result = json.loads(output.read_text()) if output.exists() else {}
guard = json.loads(guard_path.read_text()) if guard_path.exists() else {}
report["provenance_after"] = swift_provenance(ROOT, binary, worker, bundle)
report["provenance_unchanged"] = report["provenance_after"] == before
report["report_sha256"] = sha256(output) if output.exists() else None
report["guard_report_sha256"] = sha256(guard_path) if guard_path.exists() else None
report["passed"] = bool(
completed.returncode == 0 and report["provenance_unchanged"]
and guard.get("completed") and guard.get("exit_code") == 0
and guard.get("stopped_reason") is None
and result.get("status") == "completed" and result.get("passed")
and result.get("requested_requests") == args.requests
and result.get("completed_requests") == args.requests
and result.get("distinct_series") == min(args.requests, args.series)
and result.get("configuration", {}).get("max_concurrent_workers") == args.max_workers
)
report["status"] = "completed" if report["passed"] else "failed"
except BaseException as error:
report.update(status="failed", error=f"{type(error).__name__}: {error}")
raise
finally:
report["finished_at"] = datetime.now(timezone.utc).isoformat()
atomic_json(provenance_path, report)
print(json.dumps(dict(passed=report["passed"], provenance=str(provenance_path))), flush=True)
return 0 if report["passed"] else 1
if __name__ == "__main__":
raise SystemExit(main())
|