HyperFlow / scripts /load_api.py
Gaurav711's picture
Deploy 891f179
2b86c60 verified
Raw History Blame Contribute Delete
9.28 kB
#!/usr/bin/env python3
"""Scheduled localhost socket load against a disposable PostgreSQL application.
No live provider or model traffic. Write cases include fresh simulation setup
through Commerce before the timed HTTP execution; preparation is reported and
included in scheduled-to-completion latency. This is not production capacity.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor, as_completed
import hashlib
import json
import os
import resource
from pathlib import Path
import socket
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
os.environ['HYPERFLOW_ENABLE_LLM'] = '0'
from scripts.verify_runtime import isolated_database, checkout, machine
from backend.flagship.api import create_app
from backend.flagship.identity import Identity
from backend.flagship.provider import Provider
from backend.flagship.store import Store
import httpx
import uvicorn
def percentile(values, fraction):
ordered = sorted(values)
return ordered[min(len(ordered)-1, int((len(ordered)-1)*fraction))] if ordered else None
def run(output, seconds=300, runs=3):
if not os.getenv('PG_TEST_URL'):
raise ValueError('PG_TEST_URL must point to an isolated PostgreSQL administration database')
report = {'schema_version': 1, 'protocol': 'hyperflow-socket-load-v1', 'source': 'controlled_simulation',
'method': 'Scheduled localhost HTTP; real PostgreSQL; no TLS or external provider/model latency',
'hardware': machine(), 'configuration': {'runs': runs, 'seconds_per_run': seconds,
'scheduled_requests_per_second': 5, 'accounts': 16, 'read_fraction': .8, 'executor_workers': 8},
'limitations': ['Simulation writes use fresh client-side Commerce setup before HTTP execution; preparation is measured separately.',
'Approval preparation contributes database load and scheduled completion latency.',
'Read paths list account actions; write paths execute approved simulated checkout actions.',
'No browser, TLS, Redis, model, external provider or background training workload.',
'Fixed offered load is not a measurement of maximum sustainable capacity.'], 'runs': []}
with isolated_database() as url:
store = Store(url)
store.migrate()
identity = Identity(store)
actors = [identity.provision('load_' + str(i), 'isolated-load-password-123') for i in range(16)]
headers = [{'Authorization': 'Bearer ' + identity.issue(actor)['access_token']} for actor in actors]
def deny(request):
raise AssertionError('External provider traffic is prohibited')
provider = Provider(store, identity, httpx.MockTransport(deny))
app = create_app(store, provider)
listener = socket.socket()
listener.bind(('127.0.0.1', 0))
listener.listen(128)
base = 'http://127.0.0.1:' + str(listener.getsockname()[1])
server = uvicorn.Server(uvicorn.Config(app, log_level='warning', access_log=False))
thread = threading.Thread(target=lambda: server.run(sockets=[listener]), daemon=True)
thread.start()
try:
with httpx.Client(base_url=base, timeout=10) as client:
started = time.monotonic()
while not server.started:
if not thread.is_alive() or time.monotonic() - started > 15:
raise RuntimeError('Isolated API failed to start')
time.sleep(.05)
for _ in range(30):
client.get('/health').raise_for_status()
time.sleep(1 if seconds >= 300 else .01)
for run_index in range(runs):
rows = []
epoch = time.monotonic()
def operation(index, due):
actor_index = index % 16
is_write = index % 5 == 4
preparation = time.monotonic()
action_id = None
try:
if is_write:
_, action = checkout(store, actors[actor_index])
action_id = action['id']
prepared = time.monotonic()
if is_write:
response = client.post('/api/v2/actions/' + action_id + '/execute',
headers={**headers[actor_index], 'Idempotency-Key': 'load-' + action_id})
else:
response = client.get('/api/v2/actions', headers=headers[actor_index])
completed = time.monotonic()
state = response.json().get('data', {}).get('state') if is_write and response.status_code == 200 else None
return {'index': index, 'kind': 'action' if is_write else 'read', 'status': response.status_code,
'succeeded': response.status_code == 200 and (not is_write or state == 'SUCCEEDED'),
'action_id': action_id, 'state': state, 'preparation_ms': (prepared-preparation)*1000,
'http_ms': (completed-prepared)*1000, 'scheduled_completion_ms': (completed-due)*1000}
except Exception as exc:
return {'index': index, 'kind': 'action' if is_write else 'read', 'succeeded': False,
'error_type': type(exc).__name__, 'scheduled_completion_ms': (time.monotonic()-due)*1000}
with ThreadPoolExecutor(max_workers=8) as pool:
pending = []
for index in range(seconds * 5):
due = epoch + index / 5
pause = due - time.monotonic()
if pause > 0:
time.sleep(pause)
pending.append(pool.submit(operation, index, due))
for future in as_completed(pending):
rows.append(future.result())
metrics = {}
for kind in ('read', 'action'):
selected = [x for x in rows if x['kind'] == kind]
latencies = [x['scheduled_completion_ms'] for x in selected]
metrics[kind] = {'requests': len(selected), 'failed': sum(not x['succeeded'] for x in selected),
'p50_ms': percentile(latencies, .5), 'p95_ms': percentile(latencies, .95), 'p99_ms': percentile(latencies, .99),
'http_p95_ms': percentile([x['http_ms'] for x in selected if 'http_ms' in x], .95),
'preparation_p95_ms': percentile([x['preparation_ms'] for x in selected if 'preparation_ms' in x], .95)}
report['runs'].append({'run': run_index+1, 'elapsed_seconds': time.monotonic()-epoch,
'requests': len(rows), 'metrics': metrics, 'results': sorted(rows, key=lambda x:x['index'])})
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(report, indent=2, allow_nan=False)+'\n')
print(json.dumps({'run': run_index+1, 'metrics': metrics}), flush=True)
finally:
server.should_exit = True
thread.join(15)
listener.close()
store.engine.dispose()
if thread.is_alive():
raise RuntimeError('Isolated API did not stop')
usage = resource.getrusage(resource.RUSAGE_SELF)
report['resource_usage'] = {'peak_rss_bytes': usage.ru_maxrss if sys.platform == 'darwin' else usage.ru_maxrss * 1024,
'cpu_user_seconds': usage.ru_utime, 'cpu_system_seconds': usage.ru_stime,
'scope': 'Load client and API share one process; PostgreSQL process excluded'}
report['full_protocol'] = seconds == 300 and runs == 3
report['passed'] = report['full_protocol'] and all(
row['metrics']['read']['failed'] == 0 and row['metrics']['action']['failed'] == 0
and row['metrics']['read']['p95_ms'] < 250 and row['metrics']['action']['p95_ms'] < 1000 for row in report['runs'])
paths = [Path(__file__), ROOT/'backend/flagship/api.py', ROOT/'backend/flagship/commerce.py', ROOT/'backend/flagship/provider.py']
report['source_sha256'] = {str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest() for p in paths}
output.write_text(json.dumps(report, indent=2, allow_nan=False)+'\n')
return report
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--seconds', type=int, default=300, choices=range(1,301))
parser.add_argument('--runs', type=int, default=3, choices=range(1,4))
args = parser.parse_args()
result = run(args.output, args.seconds, args.runs)
print(json.dumps({'passed': result['passed'], 'full_protocol': result['full_protocol']}))
raise SystemExit(0 if result['passed'] or not result['full_protocol'] else 1)