File size: 9,282 Bytes
2b86c60 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 | #!/usr/bin/env python3
"""Scheduled localhost socket load against a disposable PostgreSQL application.
No live provider or model traffic. Write cases include fresh simulation setup
through Commerce before the timed HTTP execution; preparation is reported and
included in scheduled-to-completion latency. This is not production capacity.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor, as_completed
import hashlib
import json
import os
import resource
from pathlib import Path
import socket
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
os.environ['HYPERFLOW_ENABLE_LLM'] = '0'
from scripts.verify_runtime import isolated_database, checkout, machine
from backend.flagship.api import create_app
from backend.flagship.identity import Identity
from backend.flagship.provider import Provider
from backend.flagship.store import Store
import httpx
import uvicorn
def percentile(values, fraction):
ordered = sorted(values)
return ordered[min(len(ordered)-1, int((len(ordered)-1)*fraction))] if ordered else None
def run(output, seconds=300, runs=3):
if not os.getenv('PG_TEST_URL'):
raise ValueError('PG_TEST_URL must point to an isolated PostgreSQL administration database')
report = {'schema_version': 1, 'protocol': 'hyperflow-socket-load-v1', 'source': 'controlled_simulation',
'method': 'Scheduled localhost HTTP; real PostgreSQL; no TLS or external provider/model latency',
'hardware': machine(), 'configuration': {'runs': runs, 'seconds_per_run': seconds,
'scheduled_requests_per_second': 5, 'accounts': 16, 'read_fraction': .8, 'executor_workers': 8},
'limitations': ['Simulation writes use fresh client-side Commerce setup before HTTP execution; preparation is measured separately.',
'Approval preparation contributes database load and scheduled completion latency.',
'Read paths list account actions; write paths execute approved simulated checkout actions.',
'No browser, TLS, Redis, model, external provider or background training workload.',
'Fixed offered load is not a measurement of maximum sustainable capacity.'], 'runs': []}
with isolated_database() as url:
store = Store(url)
store.migrate()
identity = Identity(store)
actors = [identity.provision('load_' + str(i), 'isolated-load-password-123') for i in range(16)]
headers = [{'Authorization': 'Bearer ' + identity.issue(actor)['access_token']} for actor in actors]
def deny(request):
raise AssertionError('External provider traffic is prohibited')
provider = Provider(store, identity, httpx.MockTransport(deny))
app = create_app(store, provider)
listener = socket.socket()
listener.bind(('127.0.0.1', 0))
listener.listen(128)
base = 'http://127.0.0.1:' + str(listener.getsockname()[1])
server = uvicorn.Server(uvicorn.Config(app, log_level='warning', access_log=False))
thread = threading.Thread(target=lambda: server.run(sockets=[listener]), daemon=True)
thread.start()
try:
with httpx.Client(base_url=base, timeout=10) as client:
started = time.monotonic()
while not server.started:
if not thread.is_alive() or time.monotonic() - started > 15:
raise RuntimeError('Isolated API failed to start')
time.sleep(.05)
for _ in range(30):
client.get('/health').raise_for_status()
time.sleep(1 if seconds >= 300 else .01)
for run_index in range(runs):
rows = []
epoch = time.monotonic()
def operation(index, due):
actor_index = index % 16
is_write = index % 5 == 4
preparation = time.monotonic()
action_id = None
try:
if is_write:
_, action = checkout(store, actors[actor_index])
action_id = action['id']
prepared = time.monotonic()
if is_write:
response = client.post('/api/v2/actions/' + action_id + '/execute',
headers={**headers[actor_index], 'Idempotency-Key': 'load-' + action_id})
else:
response = client.get('/api/v2/actions', headers=headers[actor_index])
completed = time.monotonic()
state = response.json().get('data', {}).get('state') if is_write and response.status_code == 200 else None
return {'index': index, 'kind': 'action' if is_write else 'read', 'status': response.status_code,
'succeeded': response.status_code == 200 and (not is_write or state == 'SUCCEEDED'),
'action_id': action_id, 'state': state, 'preparation_ms': (prepared-preparation)*1000,
'http_ms': (completed-prepared)*1000, 'scheduled_completion_ms': (completed-due)*1000}
except Exception as exc:
return {'index': index, 'kind': 'action' if is_write else 'read', 'succeeded': False,
'error_type': type(exc).__name__, 'scheduled_completion_ms': (time.monotonic()-due)*1000}
with ThreadPoolExecutor(max_workers=8) as pool:
pending = []
for index in range(seconds * 5):
due = epoch + index / 5
pause = due - time.monotonic()
if pause > 0:
time.sleep(pause)
pending.append(pool.submit(operation, index, due))
for future in as_completed(pending):
rows.append(future.result())
metrics = {}
for kind in ('read', 'action'):
selected = [x for x in rows if x['kind'] == kind]
latencies = [x['scheduled_completion_ms'] for x in selected]
metrics[kind] = {'requests': len(selected), 'failed': sum(not x['succeeded'] for x in selected),
'p50_ms': percentile(latencies, .5), 'p95_ms': percentile(latencies, .95), 'p99_ms': percentile(latencies, .99),
'http_p95_ms': percentile([x['http_ms'] for x in selected if 'http_ms' in x], .95),
'preparation_p95_ms': percentile([x['preparation_ms'] for x in selected if 'preparation_ms' in x], .95)}
report['runs'].append({'run': run_index+1, 'elapsed_seconds': time.monotonic()-epoch,
'requests': len(rows), 'metrics': metrics, 'results': sorted(rows, key=lambda x:x['index'])})
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(report, indent=2, allow_nan=False)+'\n')
print(json.dumps({'run': run_index+1, 'metrics': metrics}), flush=True)
finally:
server.should_exit = True
thread.join(15)
listener.close()
store.engine.dispose()
if thread.is_alive():
raise RuntimeError('Isolated API did not stop')
usage = resource.getrusage(resource.RUSAGE_SELF)
report['resource_usage'] = {'peak_rss_bytes': usage.ru_maxrss if sys.platform == 'darwin' else usage.ru_maxrss * 1024,
'cpu_user_seconds': usage.ru_utime, 'cpu_system_seconds': usage.ru_stime,
'scope': 'Load client and API share one process; PostgreSQL process excluded'}
report['full_protocol'] = seconds == 300 and runs == 3
report['passed'] = report['full_protocol'] and all(
row['metrics']['read']['failed'] == 0 and row['metrics']['action']['failed'] == 0
and row['metrics']['read']['p95_ms'] < 250 and row['metrics']['action']['p95_ms'] < 1000 for row in report['runs'])
paths = [Path(__file__), ROOT/'backend/flagship/api.py', ROOT/'backend/flagship/commerce.py', ROOT/'backend/flagship/provider.py']
report['source_sha256'] = {str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest() for p in paths}
output.write_text(json.dumps(report, indent=2, allow_nan=False)+'\n')
return report
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--seconds', type=int, default=300, choices=range(1,301))
parser.add_argument('--runs', type=int, default=3, choices=range(1,4))
args = parser.parse_args()
result = run(args.output, args.seconds, args.runs)
print(json.dumps({'passed': result['passed'], 'full_protocol': result['full_protocol']}))
raise SystemExit(0 if result['passed'] or not result['full_protocol'] else 1)
|