Download scripts/verify_runtime.py from Gaurav711/HyperFlow: direct link, hf CLI and curl.
- Browser
- Download file 10.4 kB
-
https://huggingface.co/spaces/Gaurav711/HyperFlow/resolve/main/scripts/verify_runtime.py
- Command line
-
hf download hf://spaces/Gaurav711/HyperFlow/scripts/verify_runtime.py
-
curl -L -o verify_runtime.py https://huggingface.co/spaces/Gaurav711/HyperFlow/resolve/main/scripts/verify_runtime.py
10.4 kB
| """Measure isolated simulated commerce and exercise restart recovery. | |
| Run `python scripts/verify_runtime.py`. Set PG_TEST_URL for PostgreSQL; its role | |
| must create disposable databases. Without it the report explicitly says SQLite. | |
| The supplied PostgreSQL database is never migrated. No provider or paid calls run. | |
| """ | |
| from concurrent.futures import ThreadPoolExecutor | |
| from contextlib import contextmanager | |
| from datetime import datetime, timezone | |
| import argparse | |
| import importlib.metadata | |
| import json | |
| import math | |
| import os | |
| from pathlib import Path | |
| import platform | |
| import subprocess | |
| import sys | |
| import tempfile | |
| import time | |
| import uuid | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT)) | |
| from sqlalchemy import create_engine, text | |
| from sqlalchemy.engine import make_url | |
| from fastapi.testclient import TestClient | |
| from backend.flagship.api import create_app | |
| from backend.flagship.commerce import Commerce | |
| from backend.flagship.identity import Identity | |
| from backend.flagship.provider import Provider | |
| from backend.flagship.store import Store | |
| def isolated_database(): | |
| configured = os.getenv("PG_TEST_URL") | |
| if configured: | |
| base = make_url(configured) | |
| if base.get_backend_name() != "postgresql": | |
| raise ValueError("PG_TEST_URL must use PostgreSQL") | |
| name = "hf_runtime_" + uuid.uuid4().hex | |
| admin = create_engine(base, isolation_level="AUTOCOMMIT") | |
| with admin.connect() as connection: | |
| connection.execute(text(f'CREATE DATABASE "{name}"')) | |
| try: | |
| yield base.set(database=name).render_as_string(hide_password=False) | |
| finally: | |
| with admin.connect() as connection: | |
| connection.execute(text(f'DROP DATABASE "{name}" WITH (FORCE)')) | |
| admin.dispose() | |
| else: | |
| with tempfile.TemporaryDirectory(prefix="hyperflow-runtime-") as directory: | |
| yield "sqlite:///" + str(Path(directory) / "runtime.sqlite") | |
| def checkout(store, actor, scenario="success"): | |
| commerce = Commerce(store, Provider(store, Identity(store))) | |
| proposal = commerce.propose(actor, {"provider": "im", "tool": "update_cart", "mode": "replay", | |
| "arguments": {"selectedAddressId": "scenario-address", "items": [ | |
| {"spinId": "scenario-milk-0", "skuId": "scenario-sku-milk-0", "quantity": 1}]}}) | |
| edit = commerce.approve(actor, proposal["id"], proposal["version"]) | |
| if commerce.execute(actor, edit["id"], "cart-" + proposal["id"])["data"]["state"] != "SUCCEEDED": | |
| raise AssertionError("Simulation setup failed") | |
| proposal = commerce.propose(actor, {"provider": "im", "tool": "checkout", "mode": "replay", "scenario": scenario, | |
| "arguments": {"addressId": "scenario-address", "paymentMethod": "COD"}}) | |
| return commerce, commerce.approve(actor, proposal["id"], proposal["version"]) | |
| def child_recover(account_id, action_id): | |
| store = Store() | |
| try: | |
| actor = store.account(account_id) | |
| commerce = Commerce(store, Provider(store, Identity(store))) | |
| before = commerce.execute(actor, action_id, "retry-after-process-restart")["data"]["state"] | |
| after = commerce.reconcile(actor, action_id)["data"]["state"] | |
| print(json.dumps({"retry_state": before, "reconciled_state": after, | |
| "receipts": len(store.list("simulation", account_id)[0]["data"]["receipts"])})) | |
| finally: | |
| store.engine.dispose() | |
| def machine(): | |
| result = {"system": platform.system(), "release": platform.release(), | |
| "architecture": platform.machine(), "logical_cpus": os.cpu_count(), "python": platform.python_version()} | |
| if platform.system() == "Darwin": | |
| for name, key in [("machdep.cpu.brand_string", "cpu"), ("hw.memsize", "memory_bytes")]: | |
| response = subprocess.run(["sysctl", "-n", name], capture_output=True, text=True) | |
| if response.returncode == 0: | |
| result[key] = response.stdout.strip() | |
| result["packages"] = {name: importlib.metadata.version(name) for name in ("fastapi", "sqlalchemy", "httpx")} | |
| return result | |
| def percentile(values, quantile): | |
| return round(sorted(values)[max(0, math.ceil(quantile * len(values)) - 1)], 3) | |
| def run(url, clients, workers): | |
| store = Store(url) | |
| store.migrate() | |
| identity = Identity(store) | |
| fixtures = [] | |
| for index in range(clients): | |
| actor = identity.provision("runtime-" + str(index), "isolated-runtime-password", "user") | |
| _, action = checkout(store, actor) | |
| fixtures.append((actor, action, identity.issue(actor)["access_token"])) | |
| with store.engine.connect() as connection: | |
| version = connection.execute(text("SELECT version()" if store.engine.dialect.name == "postgresql" else "SELECT sqlite_version()" )).scalar_one() | |
| # Two calls per action exercise concurrent duplicate submission, not just throughput. | |
| requests = [fixture for fixture in fixtures for _ in range(2)] | |
| with TestClient(create_app(store=store)) as client: | |
| def attempt(fixture): | |
| actor, action, token = fixture | |
| started = time.perf_counter() | |
| try: | |
| response = client.post(f"/api/v2/actions/{action['id']}/execute", headers={ | |
| "Authorization": "Bearer " + token, "Idempotency-Key": "runtime-duplicate"}) | |
| ok = response.status_code == 200 and response.json().get("data", {}).get("state") == "SUCCEEDED" | |
| return {"duration_ms": (time.perf_counter() - started) * 1000, "ok": ok, "status": response.status_code} | |
| except Exception as exc: | |
| return {"duration_ms": (time.perf_counter() - started) * 1000, "ok": False, "exception": type(exc).__name__} | |
| started = time.perf_counter() | |
| with ThreadPoolExecutor(max_workers=workers) as pool: | |
| outcomes = list(pool.map(attempt, requests)) | |
| elapsed = time.perf_counter() - started | |
| receipt_counts = [len(store.list("simulation", actor["id"])[0]["data"]["receipts"]) for actor, _, _ in fixtures] | |
| latencies = [o["duration_ms"] for o in outcomes] | |
| actor = identity.provision("restart", "isolated-restart-password", "user") | |
| commerce, action = checkout(store, actor, "lost_response") | |
| before = commerce.execute(actor, action["id"], "lost-response")["data"]["state"] | |
| cursor = store.recent_events(actor["id"])[-1]["id"] | |
| env = {**os.environ, "DATABASE_URL": url, "PYTHONPATH": str(ROOT)} | |
| # Disable optional Redis transport: restart proof relies on database events. | |
| env.pop("REDIS_URL", None) | |
| restarted = subprocess.run([sys.executable, str(Path(__file__).resolve()), "--recover", actor["id"], action["id"]], | |
| env=env, cwd=ROOT, capture_output=True, text=True, timeout=45) | |
| if restarted.returncode: | |
| raise RuntimeError("Recovery subprocess failed: " + restarted.stderr[-1000:]) | |
| recovery = json.loads(restarted.stdout) | |
| job = store.create("training_job", actor["id"], {"state": "RUNNING", "attempts": 1, "payload": {}, "source": "user_input"}) | |
| worker = subprocess.run([sys.executable, "-m", "backend.flagship.worker", "--once"], | |
| env=env, cwd=ROOT, capture_output=True, text=True, timeout=45) | |
| if worker.returncode: | |
| raise RuntimeError("Worker subprocess failed: " + worker.stderr[-1000:]) | |
| replacement = Store(url) | |
| try: | |
| resumed = replacement.get("training_job", job["id"], actor["id"])["data"] | |
| events_after_restart = replacement.recent_events(actor["id"], after=cursor) | |
| finally: | |
| replacement.engine.dispose() | |
| report = {"schema_version": 1, "captured_at": datetime.now(timezone.utc).isoformat(), "hardware": machine(), | |
| "database": {"backend": store.engine.dialect.name, "version": version, "isolation": "disposable database"}, | |
| "method": "In-process ASGI HTTP clients; real database; explicit provider simulation; no external model/provider traffic", | |
| "load": {"accounts": clients, "concurrent_threads": workers, "requests": len(outcomes), | |
| "elapsed_seconds": round(elapsed, 3), "requests_per_second": round(len(outcomes) / elapsed, 3), | |
| "p50_ms": percentile(latencies, .5), "p95_ms": percentile(latencies, .95), | |
| "errors": sum(not o["ok"] for o in outcomes), "simulated_orders": sum(receipt_counts), | |
| "duplicate_simulated_orders": sum(max(0, count - 1) for count in receipt_counts)}, | |
| "restart": {"lost_response_state": before, **recovery, "worker_previous_state": "RUNNING", | |
| "worker_resumed_state": resumed["state"], "worker_attempts": resumed["attempts"], | |
| "durable_events_after_cursor": len(events_after_restart), "redis_enabled": False}, | |
| "limitations": ["Local ASGI test excludes socket/TLS, browser, Redis, and provider latency", | |
| "Controlled simulation proves local behavior, not upstream exactly-once execution", | |
| "Worker resumes missing-input job; this drill does not benchmark model training"]} | |
| store.engine.dispose() | |
| report["passed"] = (report["load"]["errors"] == 0 and report["load"]["duplicate_simulated_orders"] == 0 | |
| and sum(receipt_counts) == clients and before == "UNKNOWN" and recovery == { | |
| "retry_state": "UNKNOWN", "reconciled_state": "SUCCEEDED", "receipts": 1} | |
| and resumed["state"] == "INSUFFICIENT_DATA" and resumed["attempts"] == 2 and bool(events_after_restart)) | |
| return report | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--clients", type=int, default=16) | |
| parser.add_argument("--workers", type=int, default=4) | |
| parser.add_argument("--output", type=Path, default=ROOT / ".local/evidence/runtime/results.json") | |
| parser.add_argument("--recover", nargs=2, metavar=("ACCOUNT_ID", "ACTION_ID"), help=argparse.SUPPRESS) | |
| args = parser.parse_args() | |
| if args.recover: | |
| child_recover(*args.recover) | |
| return | |
| if not 10 <= args.clients <= 100 or not 1 <= args.workers <= 16: | |
| parser.error("Use 10..100 clients and 1..16 workers") | |
| with isolated_database() as url: | |
| report = run(url, args.clients, args.workers) | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") | |
| print(json.dumps(report, indent=2, allow_nan=False)) | |
| raise SystemExit(0 if report["passed"] else 1) | |
| if __name__ == "__main__": | |
| main() | |