Download scripts/eval_commerce.py from Gaurav711/HyperFlow: direct link, hf CLI and curl.
- Browser
- Download file 6.4 kB
-
https://huggingface.co/spaces/Gaurav711/HyperFlow/resolve/main/scripts/eval_commerce.py
- Command line
-
hf download hf://spaces/Gaurav711/HyperFlow/scripts/eval_commerce.py
-
curl -L -o eval_commerce.py https://huggingface.co/spaces/Gaurav711/HyperFlow/resolve/main/scripts/eval_commerce.py
6.4 kB
| #!/usr/bin/env python3 | |
| """Run versioned commerce acceptance cases using isolated simulations only. | |
| This measures controlled scenario completion, not live Swiggy success or user quality. | |
| Paid model evaluation is separate and must explicitly enable model calls. | |
| """ | |
| import argparse | |
| from collections import Counter | |
| from concurrent.futures import ThreadPoolExecutor | |
| from itertools import product | |
| import json | |
| from pathlib import Path | |
| import platform | |
| import statistics | |
| import sys | |
| import tempfile | |
| import time | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from backend.flagship.basket import parse_intent | |
| from backend.flagship.commerce import Commerce | |
| from backend.flagship.identity import Identity | |
| from backend.flagship.provider import Provider, MUTATIONS, ORDERS | |
| from backend.flagship.store import Store, digest | |
| CORPUS = Path(__file__).resolve().parents[1] / 'evaluation/corpus/commerce-v1.json' | |
| class MeasuredProvider(Provider): | |
| def __init__(self, *args): | |
| super().__init__(*args) | |
| self.attempts = [] | |
| def call(self, account, provider, tool, *args, **kwargs): | |
| self.attempts.append((provider, tool)) | |
| return super().call(account, provider, tool, *args, **kwargs) | |
| def run_case(case, directory): | |
| started = time.perf_counter() | |
| store = Store('sqlite:///' + str(Path(directory) / (case['id'] + '.db'))) | |
| store.migrate() | |
| # An isolated actor is fixture data, never a provisioned live-provider identity. | |
| account = {'id': case['id'], 'role': 'reviewer'} | |
| provider = MeasuredProvider(store, Identity(store)) | |
| commerce = Commerce(store, provider) | |
| unauthorized = None | |
| if case['kind'] == 'parse': | |
| result = parse_intent(case['text']) | |
| actual = [(x['target_quantity'], x['unit']) for x in result['items']] | |
| passed = actual == [tuple(x) for x in case['expected']] and not result['clarifications'] | |
| state = 'PARSED' | |
| else: | |
| plan = commerce.plan(account, {'text': '1 l milk', 'mode': 'replay', 'address_id': 'scenario-address', 'budget_minor': case['budget_minor']}) | |
| assert plan['result']['status'] == 'optimal' | |
| if case['scenario'] in ('lost_response', 'uncertain'): | |
| cart = commerce.propose(account, {'provider': 'im', 'tool': 'update_cart', 'arguments': plan['proposal_arguments'], 'mode': 'replay', 'plan_id': plan['plan_id']}) | |
| initial = commerce.approve(account, cart['id'], cart['version']) | |
| assert commerce.execute(account, initial['id'], 'seed-cart')['data']['state'] == 'SUCCEEDED' | |
| tool, arguments = 'checkout', {'addressId': 'scenario-address', 'paymentMethod': 'COD'} | |
| plan_id = None | |
| else: | |
| tool, arguments, plan_id = 'update_cart', plan['proposal_arguments'], plan['plan_id'] | |
| proposal = commerce.propose(account, {'provider': 'im', 'tool': tool, 'arguments': arguments, | |
| 'mode': 'replay', 'scenario': case['scenario'], 'plan_id': plan_id}) | |
| before = len(provider.attempts) | |
| try: | |
| commerce.execute(account, proposal['id'], 'without-approval') | |
| raise AssertionError('Unapproved proposal was executable') | |
| except LookupError: | |
| pass | |
| unauthorized = sum(tool in MUTATIONS[p] for p, tool in provider.attempts[before:]) | |
| assert unauthorized == 0 | |
| action = commerce.approve(account, proposal['id'], proposal['version']) | |
| if case['scenario'] == 'expired_approval': | |
| action = store.update('action', action['id'], account['id'], {**action['data'], 'expires_at': 0}, action['version']) | |
| first = commerce.execute(account, action['id'], 'execute') | |
| again = commerce.execute(account, action['id'], 'execute') | |
| assert again['data']['state'] == first['data']['state'] | |
| state = first['data']['state'] | |
| if case['scenario'] == 'lost_response': | |
| state = commerce.reconcile(account, action['id'])['data']['state'] | |
| passed = state == case['expected_state'] | |
| receipts = store.list('simulation', account['id'])[0]['data']['receipts'] | |
| assert len(receipts) <= 1 | |
| assert sum(tool in ORDERS for _, tool in provider.attempts) <= 1 | |
| elapsed = (time.perf_counter() - started) * 1000 | |
| event_rows = store.recent_events(account['id'], 0, 500) | |
| store.engine.dispose() | |
| return {'id': case['id'], 'family': case['family'], 'source': 'simulation', 'passed': passed, | |
| 'state': state, 'latency_ms': round(elapsed, 2), | |
| 'tool_calls': len(provider.attempts), 'model_calls': 0, | |
| 'unauthorized_mutations': unauthorized, | |
| 'duplicate_placements': max(0, sum(tool in ORDERS for _, tool in provider.attempts) - 1) if case['kind'] == 'recovery' else None} | |
| def evaluate(): | |
| corpus = json.loads(CORPUS.read_text()) | |
| with tempfile.TemporaryDirectory(prefix='hyperflow-corpus-') as directory: | |
| results = [] | |
| for case in corpus['cases']: | |
| try: | |
| results.append(run_case(case, directory)) | |
| except Exception as exc: | |
| results.append({'id': case['id'], 'family': case['family'], 'passed': False, 'error': str(exc)}) | |
| latencies = sorted(x['latency_ms'] for x in results if 'latency_ms' in x) | |
| return {'corpus_version': corpus['version'], 'corpus_sha256': digest(corpus), | |
| 'source': 'controlled_simulation', 'runtime': platform.python_version(), 'platform': platform.platform(), | |
| 'cases': len(results), 'failure_recovery_cases': sum(x['kind'] == 'recovery' for x in corpus['cases']), | |
| 'passed': sum(x['passed'] for x in results), 'families': dict(Counter(x['family'] for x in results)), | |
| 'p50_ms': statistics.median(latencies), 'p95_ms': latencies[min(len(latencies)-1, int(len(latencies)*.95))], | |
| 'llm_comparison': 'NOT_RUN: rules baseline only; no paid provider calls', 'results': results} | |
| if __name__ == '__main__': | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument('--output', type=Path, required=True) | |
| args = parser.parse_args() | |
| report = evaluate() | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_text(json.dumps(report, indent=2) + '\n') | |
| print(json.dumps({k: v for k, v in report.items() if k != 'results'}, indent=2)) | |
| raise SystemExit(0 if report['passed'] == report['cases'] else 1) | |