File size: 6,399 Bytes
2b86c60
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
#!/usr/bin/env python3
"""Run versioned commerce acceptance cases using isolated simulations only.

This measures controlled scenario completion, not live Swiggy success or user quality.
Paid model evaluation is separate and must explicitly enable model calls.
"""
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from itertools import product
import json
from pathlib import Path
import platform
import statistics
import sys
import tempfile
import time

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from backend.flagship.basket import parse_intent
from backend.flagship.commerce import Commerce
from backend.flagship.identity import Identity
from backend.flagship.provider import Provider, MUTATIONS, ORDERS
from backend.flagship.store import Store, digest

CORPUS = Path(__file__).resolve().parents[1] / 'evaluation/corpus/commerce-v1.json'


class MeasuredProvider(Provider):
    def __init__(self, *args):
        super().__init__(*args)
        self.attempts = []

    def call(self, account, provider, tool, *args, **kwargs):
        self.attempts.append((provider, tool))
        return super().call(account, provider, tool, *args, **kwargs)


def run_case(case, directory):
    started = time.perf_counter()
    store = Store('sqlite:///' + str(Path(directory) / (case['id'] + '.db')))
    store.migrate()
    # An isolated actor is fixture data, never a provisioned live-provider identity.
    account = {'id': case['id'], 'role': 'reviewer'}
    provider = MeasuredProvider(store, Identity(store))
    commerce = Commerce(store, provider)
    unauthorized = None
    if case['kind'] == 'parse':
        result = parse_intent(case['text'])
        actual = [(x['target_quantity'], x['unit']) for x in result['items']]
        passed = actual == [tuple(x) for x in case['expected']] and not result['clarifications']
        state = 'PARSED'
    else:
        plan = commerce.plan(account, {'text': '1 l milk', 'mode': 'replay', 'address_id': 'scenario-address', 'budget_minor': case['budget_minor']})
        assert plan['result']['status'] == 'optimal'
        if case['scenario'] in ('lost_response', 'uncertain'):
            cart = commerce.propose(account, {'provider': 'im', 'tool': 'update_cart', 'arguments': plan['proposal_arguments'], 'mode': 'replay', 'plan_id': plan['plan_id']})
            initial = commerce.approve(account, cart['id'], cart['version'])
            assert commerce.execute(account, initial['id'], 'seed-cart')['data']['state'] == 'SUCCEEDED'
            tool, arguments = 'checkout', {'addressId': 'scenario-address', 'paymentMethod': 'COD'}
            plan_id = None
        else:
            tool, arguments, plan_id = 'update_cart', plan['proposal_arguments'], plan['plan_id']
        proposal = commerce.propose(account, {'provider': 'im', 'tool': tool, 'arguments': arguments,
            'mode': 'replay', 'scenario': case['scenario'], 'plan_id': plan_id})
        before = len(provider.attempts)
        try:
            commerce.execute(account, proposal['id'], 'without-approval')
            raise AssertionError('Unapproved proposal was executable')
        except LookupError:
            pass
        unauthorized = sum(tool in MUTATIONS[p] for p, tool in provider.attempts[before:])
        assert unauthorized == 0
        action = commerce.approve(account, proposal['id'], proposal['version'])
        if case['scenario'] == 'expired_approval':
            action = store.update('action', action['id'], account['id'], {**action['data'], 'expires_at': 0}, action['version'])
        first = commerce.execute(account, action['id'], 'execute')
        again = commerce.execute(account, action['id'], 'execute')
        assert again['data']['state'] == first['data']['state']
        state = first['data']['state']
        if case['scenario'] == 'lost_response':
            state = commerce.reconcile(account, action['id'])['data']['state']
        passed = state == case['expected_state']
        receipts = store.list('simulation', account['id'])[0]['data']['receipts']
        assert len(receipts) <= 1
        assert sum(tool in ORDERS for _, tool in provider.attempts) <= 1
    elapsed = (time.perf_counter() - started) * 1000
    event_rows = store.recent_events(account['id'], 0, 500)
    store.engine.dispose()
    return {'id': case['id'], 'family': case['family'], 'source': 'simulation', 'passed': passed,
            'state': state, 'latency_ms': round(elapsed, 2),
            'tool_calls': len(provider.attempts), 'model_calls': 0,
            'unauthorized_mutations': unauthorized,
            'duplicate_placements': max(0, sum(tool in ORDERS for _, tool in provider.attempts) - 1) if case['kind'] == 'recovery' else None}


def evaluate():
    corpus = json.loads(CORPUS.read_text())
    with tempfile.TemporaryDirectory(prefix='hyperflow-corpus-') as directory:
        results = []
        for case in corpus['cases']:
            try:
                results.append(run_case(case, directory))
            except Exception as exc:
                results.append({'id': case['id'], 'family': case['family'], 'passed': False, 'error': str(exc)})
    latencies = sorted(x['latency_ms'] for x in results if 'latency_ms' in x)
    return {'corpus_version': corpus['version'], 'corpus_sha256': digest(corpus),
            'source': 'controlled_simulation', 'runtime': platform.python_version(), 'platform': platform.platform(),
            'cases': len(results), 'failure_recovery_cases': sum(x['kind'] == 'recovery' for x in corpus['cases']),
            'passed': sum(x['passed'] for x in results), 'families': dict(Counter(x['family'] for x in results)),
            'p50_ms': statistics.median(latencies), 'p95_ms': latencies[min(len(latencies)-1, int(len(latencies)*.95))],
            'llm_comparison': 'NOT_RUN: rules baseline only; no paid provider calls', 'results': results}


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--output', type=Path, required=True)
    args = parser.parse_args()
    report = evaluate()
    args.output.parent.mkdir(parents=True, exist_ok=True)
    args.output.write_text(json.dumps(report, indent=2) + '\n')
    print(json.dumps({k: v for k, v in report.items() if k != 'results'}, indent=2))
    raise SystemExit(0 if report['passed'] == report['cases'] else 1)