File size: 6,399 Bytes
2b86c60 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | #!/usr/bin/env python3
"""Run versioned commerce acceptance cases using isolated simulations only.
This measures controlled scenario completion, not live Swiggy success or user quality.
Paid model evaluation is separate and must explicitly enable model calls.
"""
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from itertools import product
import json
from pathlib import Path
import platform
import statistics
import sys
import tempfile
import time
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from backend.flagship.basket import parse_intent
from backend.flagship.commerce import Commerce
from backend.flagship.identity import Identity
from backend.flagship.provider import Provider, MUTATIONS, ORDERS
from backend.flagship.store import Store, digest
CORPUS = Path(__file__).resolve().parents[1] / 'evaluation/corpus/commerce-v1.json'
class MeasuredProvider(Provider):
def __init__(self, *args):
super().__init__(*args)
self.attempts = []
def call(self, account, provider, tool, *args, **kwargs):
self.attempts.append((provider, tool))
return super().call(account, provider, tool, *args, **kwargs)
def run_case(case, directory):
started = time.perf_counter()
store = Store('sqlite:///' + str(Path(directory) / (case['id'] + '.db')))
store.migrate()
# An isolated actor is fixture data, never a provisioned live-provider identity.
account = {'id': case['id'], 'role': 'reviewer'}
provider = MeasuredProvider(store, Identity(store))
commerce = Commerce(store, provider)
unauthorized = None
if case['kind'] == 'parse':
result = parse_intent(case['text'])
actual = [(x['target_quantity'], x['unit']) for x in result['items']]
passed = actual == [tuple(x) for x in case['expected']] and not result['clarifications']
state = 'PARSED'
else:
plan = commerce.plan(account, {'text': '1 l milk', 'mode': 'replay', 'address_id': 'scenario-address', 'budget_minor': case['budget_minor']})
assert plan['result']['status'] == 'optimal'
if case['scenario'] in ('lost_response', 'uncertain'):
cart = commerce.propose(account, {'provider': 'im', 'tool': 'update_cart', 'arguments': plan['proposal_arguments'], 'mode': 'replay', 'plan_id': plan['plan_id']})
initial = commerce.approve(account, cart['id'], cart['version'])
assert commerce.execute(account, initial['id'], 'seed-cart')['data']['state'] == 'SUCCEEDED'
tool, arguments = 'checkout', {'addressId': 'scenario-address', 'paymentMethod': 'COD'}
plan_id = None
else:
tool, arguments, plan_id = 'update_cart', plan['proposal_arguments'], plan['plan_id']
proposal = commerce.propose(account, {'provider': 'im', 'tool': tool, 'arguments': arguments,
'mode': 'replay', 'scenario': case['scenario'], 'plan_id': plan_id})
before = len(provider.attempts)
try:
commerce.execute(account, proposal['id'], 'without-approval')
raise AssertionError('Unapproved proposal was executable')
except LookupError:
pass
unauthorized = sum(tool in MUTATIONS[p] for p, tool in provider.attempts[before:])
assert unauthorized == 0
action = commerce.approve(account, proposal['id'], proposal['version'])
if case['scenario'] == 'expired_approval':
action = store.update('action', action['id'], account['id'], {**action['data'], 'expires_at': 0}, action['version'])
first = commerce.execute(account, action['id'], 'execute')
again = commerce.execute(account, action['id'], 'execute')
assert again['data']['state'] == first['data']['state']
state = first['data']['state']
if case['scenario'] == 'lost_response':
state = commerce.reconcile(account, action['id'])['data']['state']
passed = state == case['expected_state']
receipts = store.list('simulation', account['id'])[0]['data']['receipts']
assert len(receipts) <= 1
assert sum(tool in ORDERS for _, tool in provider.attempts) <= 1
elapsed = (time.perf_counter() - started) * 1000
event_rows = store.recent_events(account['id'], 0, 500)
store.engine.dispose()
return {'id': case['id'], 'family': case['family'], 'source': 'simulation', 'passed': passed,
'state': state, 'latency_ms': round(elapsed, 2),
'tool_calls': len(provider.attempts), 'model_calls': 0,
'unauthorized_mutations': unauthorized,
'duplicate_placements': max(0, sum(tool in ORDERS for _, tool in provider.attempts) - 1) if case['kind'] == 'recovery' else None}
def evaluate():
corpus = json.loads(CORPUS.read_text())
with tempfile.TemporaryDirectory(prefix='hyperflow-corpus-') as directory:
results = []
for case in corpus['cases']:
try:
results.append(run_case(case, directory))
except Exception as exc:
results.append({'id': case['id'], 'family': case['family'], 'passed': False, 'error': str(exc)})
latencies = sorted(x['latency_ms'] for x in results if 'latency_ms' in x)
return {'corpus_version': corpus['version'], 'corpus_sha256': digest(corpus),
'source': 'controlled_simulation', 'runtime': platform.python_version(), 'platform': platform.platform(),
'cases': len(results), 'failure_recovery_cases': sum(x['kind'] == 'recovery' for x in corpus['cases']),
'passed': sum(x['passed'] for x in results), 'families': dict(Counter(x['family'] for x in results)),
'p50_ms': statistics.median(latencies), 'p95_ms': latencies[min(len(latencies)-1, int(len(latencies)*.95))],
'llm_comparison': 'NOT_RUN: rules baseline only; no paid provider calls', 'results': results}
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, required=True)
args = parser.parse_args()
report = evaluate()
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + '\n')
print(json.dumps({k: v for k, v in report.items() if k != 'results'}, indent=2))
raise SystemExit(0 if report['passed'] == report['cases'] else 1)
|