File size: 4,391 Bytes
506f40d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
"""Run the native Swift workload under fixed budgets and bind its report to source/assets."""
from __future__ import annotations

import argparse
from datetime import datetime, timezone
import json
from pathlib import Path
import subprocess
import sys

from validate_swift import ROOT, atomic_json, default_executable, sha256, swift_provenance


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--binary", type=Path, default=default_executable("timestral"))
    parser.add_argument("--worker", type=Path, default=default_executable("timestral-worker"))
    parser.add_argument("--bundle", type=Path, default=ROOT / "5M")
    parser.add_argument("--requests", type=int, default=1000)
    parser.add_argument("--series", type=int, default=8)
    parser.add_argument("--concurrency", type=int, default=8)
    parser.add_argument("--max-workers", type=int, default=1)
    parser.add_argument("--output", type=Path, default=ROOT / "reports/swift-stress-5M.json")
    args = parser.parse_args()
    if min(args.requests, args.series, args.concurrency, args.max_workers) < 1:
        parser.error("request, series and concurrency counts must be positive")
    binary, worker, bundle, output = (p.resolve() for p in (args.binary, args.worker, args.bundle, args.output))
    output.parent.mkdir(parents=True, exist_ok=True)
    guard_path = output.with_name("guard-" + output.name)
    provenance_path = output.with_name(output.stem + "-provenance.json")
    guard_script = ROOT / "scripts/run_guarded.py"
    before = swift_provenance(ROOT, binary, worker, bundle)
    report = dict(format="timestral-swift-benchmark-provenance-v1", passed=False,
                  status="running", started_at=datetime.now(timezone.utc).isoformat(),
                  provenance=before, benchmark_script_sha256=sha256(Path(__file__)),
                  guard_script_sha256=sha256(guard_script))
    atomic_json(provenance_path, report)
    # A failed launch must not accidentally certify an earlier successful run.
    for path in (output, guard_path):
        if path.exists():
            path.replace(path.with_name(path.name + ".previous"))
    command = [sys.executable, str(guard_script), "--report", str(guard_path), "--",
               str(binary), "stress", "--bundle", str(bundle), "--worker", str(worker),
               "--requests", str(args.requests), "--series", str(args.series),
               "--concurrency", str(args.concurrency), "--checkpoint-every", "100",
               "--max-workers", str(args.max_workers),
               "--output", str(output)]
    report["command"] = command
    try:
        completed = subprocess.run(command, check=False)
        report["exit_code"] = completed.returncode
        result = json.loads(output.read_text()) if output.exists() else {}
        guard = json.loads(guard_path.read_text()) if guard_path.exists() else {}
        report["provenance_after"] = swift_provenance(ROOT, binary, worker, bundle)
        report["provenance_unchanged"] = report["provenance_after"] == before
        report["report_sha256"] = sha256(output) if output.exists() else None
        report["guard_report_sha256"] = sha256(guard_path) if guard_path.exists() else None
        report["passed"] = bool(
            completed.returncode == 0 and report["provenance_unchanged"]
            and guard.get("completed") and guard.get("exit_code") == 0
            and guard.get("stopped_reason") is None
            and result.get("status") == "completed" and result.get("passed")
            and result.get("requested_requests") == args.requests
            and result.get("completed_requests") == args.requests
            and result.get("distinct_series") == min(args.requests, args.series)
            and result.get("configuration", {}).get("max_concurrent_workers") == args.max_workers
        )
        report["status"] = "completed" if report["passed"] else "failed"
    except BaseException as error:
        report.update(status="failed", error=f"{type(error).__name__}: {error}")
        raise
    finally:
        report["finished_at"] = datetime.now(timezone.utc).isoformat()
        atomic_json(provenance_path, report)
    print(json.dumps(dict(passed=report["passed"], provenance=str(provenance_path))), flush=True)
    return 0 if report["passed"] else 1


if __name__ == "__main__":
    raise SystemExit(main())