File size: 9,282 Bytes
2b86c60
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
#!/usr/bin/env python3
"""Scheduled localhost socket load against a disposable PostgreSQL application.

No live provider or model traffic. Write cases include fresh simulation setup
through Commerce before the timed HTTP execution; preparation is reported and
included in scheduled-to-completion latency. This is not production capacity.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor, as_completed
import hashlib
import json
import os
import resource
from pathlib import Path
import socket
import sys
import threading
import time

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
os.environ['HYPERFLOW_ENABLE_LLM'] = '0'
from scripts.verify_runtime import isolated_database, checkout, machine
from backend.flagship.api import create_app
from backend.flagship.identity import Identity
from backend.flagship.provider import Provider
from backend.flagship.store import Store
import httpx
import uvicorn


def percentile(values, fraction):
    ordered = sorted(values)
    return ordered[min(len(ordered)-1, int((len(ordered)-1)*fraction))] if ordered else None


def run(output, seconds=300, runs=3):
    if not os.getenv('PG_TEST_URL'):
        raise ValueError('PG_TEST_URL must point to an isolated PostgreSQL administration database')
    report = {'schema_version': 1, 'protocol': 'hyperflow-socket-load-v1', 'source': 'controlled_simulation',
        'method': 'Scheduled localhost HTTP; real PostgreSQL; no TLS or external provider/model latency',
        'hardware': machine(), 'configuration': {'runs': runs, 'seconds_per_run': seconds,
        'scheduled_requests_per_second': 5, 'accounts': 16, 'read_fraction': .8, 'executor_workers': 8},
        'limitations': ['Simulation writes use fresh client-side Commerce setup before HTTP execution; preparation is measured separately.',
            'Approval preparation contributes database load and scheduled completion latency.',
            'Read paths list account actions; write paths execute approved simulated checkout actions.',
            'No browser, TLS, Redis, model, external provider or background training workload.',
            'Fixed offered load is not a measurement of maximum sustainable capacity.'], 'runs': []}
    with isolated_database() as url:
        store = Store(url)
        store.migrate()
        identity = Identity(store)
        actors = [identity.provision('load_' + str(i), 'isolated-load-password-123') for i in range(16)]
        headers = [{'Authorization': 'Bearer ' + identity.issue(actor)['access_token']} for actor in actors]
        def deny(request):
            raise AssertionError('External provider traffic is prohibited')
        provider = Provider(store, identity, httpx.MockTransport(deny))
        app = create_app(store, provider)
        listener = socket.socket()
        listener.bind(('127.0.0.1', 0))
        listener.listen(128)
        base = 'http://127.0.0.1:' + str(listener.getsockname()[1])
        server = uvicorn.Server(uvicorn.Config(app, log_level='warning', access_log=False))
        thread = threading.Thread(target=lambda: server.run(sockets=[listener]), daemon=True)
        thread.start()
        try:
            with httpx.Client(base_url=base, timeout=10) as client:
                started = time.monotonic()
                while not server.started:
                    if not thread.is_alive() or time.monotonic() - started > 15:
                        raise RuntimeError('Isolated API failed to start')
                    time.sleep(.05)
                for _ in range(30):
                    client.get('/health').raise_for_status()
                    time.sleep(1 if seconds >= 300 else .01)
                for run_index in range(runs):
                    rows = []
                    epoch = time.monotonic()
                    def operation(index, due):
                        actor_index = index % 16
                        is_write = index % 5 == 4
                        preparation = time.monotonic()
                        action_id = None
                        try:
                            if is_write:
                                _, action = checkout(store, actors[actor_index])
                                action_id = action['id']
                            prepared = time.monotonic()
                            if is_write:
                                response = client.post('/api/v2/actions/' + action_id + '/execute',
                                    headers={**headers[actor_index], 'Idempotency-Key': 'load-' + action_id})
                            else:
                                response = client.get('/api/v2/actions', headers=headers[actor_index])
                            completed = time.monotonic()
                            state = response.json().get('data', {}).get('state') if is_write and response.status_code == 200 else None
                            return {'index': index, 'kind': 'action' if is_write else 'read', 'status': response.status_code,
                                'succeeded': response.status_code == 200 and (not is_write or state == 'SUCCEEDED'),
                                'action_id': action_id, 'state': state, 'preparation_ms': (prepared-preparation)*1000,
                                'http_ms': (completed-prepared)*1000, 'scheduled_completion_ms': (completed-due)*1000}
                        except Exception as exc:
                            return {'index': index, 'kind': 'action' if is_write else 'read', 'succeeded': False,
                                    'error_type': type(exc).__name__, 'scheduled_completion_ms': (time.monotonic()-due)*1000}
                    with ThreadPoolExecutor(max_workers=8) as pool:
                        pending = []
                        for index in range(seconds * 5):
                            due = epoch + index / 5
                            pause = due - time.monotonic()
                            if pause > 0:
                                time.sleep(pause)
                            pending.append(pool.submit(operation, index, due))
                        for future in as_completed(pending):
                            rows.append(future.result())
                    metrics = {}
                    for kind in ('read', 'action'):
                        selected = [x for x in rows if x['kind'] == kind]
                        latencies = [x['scheduled_completion_ms'] for x in selected]
                        metrics[kind] = {'requests': len(selected), 'failed': sum(not x['succeeded'] for x in selected),
                            'p50_ms': percentile(latencies, .5), 'p95_ms': percentile(latencies, .95), 'p99_ms': percentile(latencies, .99),
                            'http_p95_ms': percentile([x['http_ms'] for x in selected if 'http_ms' in x], .95),
                            'preparation_p95_ms': percentile([x['preparation_ms'] for x in selected if 'preparation_ms' in x], .95)}
                    report['runs'].append({'run': run_index+1, 'elapsed_seconds': time.monotonic()-epoch,
                        'requests': len(rows), 'metrics': metrics, 'results': sorted(rows, key=lambda x:x['index'])})
                    output.parent.mkdir(parents=True, exist_ok=True)
                    output.write_text(json.dumps(report, indent=2, allow_nan=False)+'\n')
                    print(json.dumps({'run': run_index+1, 'metrics': metrics}), flush=True)
        finally:
            server.should_exit = True
            thread.join(15)
            listener.close()
            store.engine.dispose()
            if thread.is_alive():
                raise RuntimeError('Isolated API did not stop')
    usage = resource.getrusage(resource.RUSAGE_SELF)
    report['resource_usage'] = {'peak_rss_bytes': usage.ru_maxrss if sys.platform == 'darwin' else usage.ru_maxrss * 1024,
        'cpu_user_seconds': usage.ru_utime, 'cpu_system_seconds': usage.ru_stime,
        'scope': 'Load client and API share one process; PostgreSQL process excluded'}
    report['full_protocol'] = seconds == 300 and runs == 3
    report['passed'] = report['full_protocol'] and all(
        row['metrics']['read']['failed'] == 0 and row['metrics']['action']['failed'] == 0
        and row['metrics']['read']['p95_ms'] < 250 and row['metrics']['action']['p95_ms'] < 1000 for row in report['runs'])
    paths = [Path(__file__), ROOT/'backend/flagship/api.py', ROOT/'backend/flagship/commerce.py', ROOT/'backend/flagship/provider.py']
    report['source_sha256'] = {str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest() for p in paths}
    output.write_text(json.dumps(report, indent=2, allow_nan=False)+'\n')
    return report


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--output', type=Path, required=True)
    parser.add_argument('--seconds', type=int, default=300, choices=range(1,301))
    parser.add_argument('--runs', type=int, default=3, choices=range(1,4))
    args = parser.parse_args()
    result = run(args.output, args.seconds, args.runs)
    print(json.dumps({'passed': result['passed'], 'full_protocol': result['full_protocol']}))
    raise SystemExit(0 if result['passed'] or not result['full_protocol'] else 1)