#!/usr/bin/env python3 """Run versioned commerce acceptance cases using isolated simulations only. This measures controlled scenario completion, not live Swiggy success or user quality. Paid model evaluation is separate and must explicitly enable model calls. """ import argparse from collections import Counter from concurrent.futures import ThreadPoolExecutor from itertools import product import json from pathlib import Path import platform import statistics import sys import tempfile import time sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from backend.flagship.basket import parse_intent from backend.flagship.commerce import Commerce from backend.flagship.identity import Identity from backend.flagship.provider import Provider, MUTATIONS, ORDERS from backend.flagship.store import Store, digest CORPUS = Path(__file__).resolve().parents[1] / 'evaluation/corpus/commerce-v1.json' class MeasuredProvider(Provider): def __init__(self, *args): super().__init__(*args) self.attempts = [] def call(self, account, provider, tool, *args, **kwargs): self.attempts.append((provider, tool)) return super().call(account, provider, tool, *args, **kwargs) def run_case(case, directory): started = time.perf_counter() store = Store('sqlite:///' + str(Path(directory) / (case['id'] + '.db'))) store.migrate() # An isolated actor is fixture data, never a provisioned live-provider identity. account = {'id': case['id'], 'role': 'reviewer'} provider = MeasuredProvider(store, Identity(store)) commerce = Commerce(store, provider) unauthorized = None if case['kind'] == 'parse': result = parse_intent(case['text']) actual = [(x['target_quantity'], x['unit']) for x in result['items']] passed = actual == [tuple(x) for x in case['expected']] and not result['clarifications'] state = 'PARSED' else: plan = commerce.plan(account, {'text': '1 l milk', 'mode': 'replay', 'address_id': 'scenario-address', 'budget_minor': case['budget_minor']}) assert plan['result']['status'] == 'optimal' if case['scenario'] in ('lost_response', 'uncertain'): cart = commerce.propose(account, {'provider': 'im', 'tool': 'update_cart', 'arguments': plan['proposal_arguments'], 'mode': 'replay', 'plan_id': plan['plan_id']}) initial = commerce.approve(account, cart['id'], cart['version']) assert commerce.execute(account, initial['id'], 'seed-cart')['data']['state'] == 'SUCCEEDED' tool, arguments = 'checkout', {'addressId': 'scenario-address', 'paymentMethod': 'COD'} plan_id = None else: tool, arguments, plan_id = 'update_cart', plan['proposal_arguments'], plan['plan_id'] proposal = commerce.propose(account, {'provider': 'im', 'tool': tool, 'arguments': arguments, 'mode': 'replay', 'scenario': case['scenario'], 'plan_id': plan_id}) before = len(provider.attempts) try: commerce.execute(account, proposal['id'], 'without-approval') raise AssertionError('Unapproved proposal was executable') except LookupError: pass unauthorized = sum(tool in MUTATIONS[p] for p, tool in provider.attempts[before:]) assert unauthorized == 0 action = commerce.approve(account, proposal['id'], proposal['version']) if case['scenario'] == 'expired_approval': action = store.update('action', action['id'], account['id'], {**action['data'], 'expires_at': 0}, action['version']) first = commerce.execute(account, action['id'], 'execute') again = commerce.execute(account, action['id'], 'execute') assert again['data']['state'] == first['data']['state'] state = first['data']['state'] if case['scenario'] == 'lost_response': state = commerce.reconcile(account, action['id'])['data']['state'] passed = state == case['expected_state'] receipts = store.list('simulation', account['id'])[0]['data']['receipts'] assert len(receipts) <= 1 assert sum(tool in ORDERS for _, tool in provider.attempts) <= 1 elapsed = (time.perf_counter() - started) * 1000 event_rows = store.recent_events(account['id'], 0, 500) store.engine.dispose() return {'id': case['id'], 'family': case['family'], 'source': 'simulation', 'passed': passed, 'state': state, 'latency_ms': round(elapsed, 2), 'tool_calls': len(provider.attempts), 'model_calls': 0, 'unauthorized_mutations': unauthorized, 'duplicate_placements': max(0, sum(tool in ORDERS for _, tool in provider.attempts) - 1) if case['kind'] == 'recovery' else None} def evaluate(): corpus = json.loads(CORPUS.read_text()) with tempfile.TemporaryDirectory(prefix='hyperflow-corpus-') as directory: results = [] for case in corpus['cases']: try: results.append(run_case(case, directory)) except Exception as exc: results.append({'id': case['id'], 'family': case['family'], 'passed': False, 'error': str(exc)}) latencies = sorted(x['latency_ms'] for x in results if 'latency_ms' in x) return {'corpus_version': corpus['version'], 'corpus_sha256': digest(corpus), 'source': 'controlled_simulation', 'runtime': platform.python_version(), 'platform': platform.platform(), 'cases': len(results), 'failure_recovery_cases': sum(x['kind'] == 'recovery' for x in corpus['cases']), 'passed': sum(x['passed'] for x in results), 'families': dict(Counter(x['family'] for x in results)), 'p50_ms': statistics.median(latencies), 'p95_ms': latencies[min(len(latencies)-1, int(len(latencies)*.95))], 'llm_comparison': 'NOT_RUN: rules baseline only; no paid provider calls', 'results': results} if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output', type=Path, required=True) args = parser.parse_args() report = evaluate() args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2) + '\n') print(json.dumps({k: v for k, v in report.items() if k != 'results'}, indent=2)) raise SystemExit(0 if report['passed'] == report['cases'] else 1)