#!/usr/bin/env python3 """Measure rules and optionally Gemini on the same bounded corpus slice. Offline default: python scripts/eval_parsing.py --output /tmp/parsing.json Paid opt-in: set GEMINI_API_KEY and GEMINI_PARSING_MODEL, then add --llm --max-cases 20 --timeout-seconds 15 --max-output-tokens 2048. There are no retries, provider actions, purchases, or model calls without --llm. """ import argparse import hashlib import json import os from pathlib import Path import platform import statistics import sys import time sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from backend.flagship.parsing import extract_intent, ExtractionUnavailable ROOT = Path(__file__).resolve().parents[1] CORPUS = ROOT / 'evaluation/corpus/commerce-v1.json' def _run(cases, *, use_llm=False, extractor=None, **options): results = [] for case in cases: started = time.perf_counter() try: intent = extract_intent(case['text'], use_llm=use_llm, extractor=extractor, **options) actual = [(x['target_quantity'], x['unit']) for x in intent['items']] passed = actual == [tuple(x) for x in case['expected']] and not intent['clarifications'] record = {'passed': passed, 'status': 'PARSED' if not intent['clarifications'] else 'NEEDS_CLARIFICATION', 'actual': actual, 'constraint_violations': 0, 'token_usage': intent['extraction']['usage']} except ExtractionUnavailable as exc: record = {'passed': False, 'status': 'UNAVAILABLE', 'error': str(exc), 'constraint_violations': None, 'token_usage': None} except ValueError as exc: record = {'passed': False, 'status': 'REJECTED', 'error': str(exc), 'constraint_violations': 1, 'token_usage': None} except Exception: record = {'passed': False, 'status': 'FAILED', 'error': 'Extractor failed; no intent accepted', 'constraint_violations': None, 'token_usage': None} results.append({'id': case['id'], 'family': case['family'], **record, 'latency_ms': (time.perf_counter() - started) * 1000, 'model_calls': int(use_llm), 'provider_action_calls': 0}) latencies = sorted(case['latency_ms'] for case in results) return {'status': 'RUN', 'cases': len(results), 'passed': sum(case['passed'] for case in results), 'completion_rate': sum(case['passed'] for case in results) / len(results), 'constraint_violations': sum(case['constraint_violations'] or 0 for case in results), 'model_calls': sum(case['model_calls'] for case in results), 'provider_action_calls': 0, 'p50_ms': statistics.median(latencies), 'p95_ms': latencies[min(len(latencies) - 1, int(len(latencies) * .95))], 'results': results} def evaluate(*, corpus_path=CORPUS, use_llm=False, max_cases=20, extractor=None, model=None, timeout_seconds=15, max_output_tokens=2048): if type(max_cases) is not int or not 1 <= max_cases <= 20: raise ValueError('max_cases must be between 1 and 20 per run') if type(use_llm) is not bool: raise ValueError('use_llm must be a boolean') # Validate bounds before starting the measured corpus, without any model call. extract_intent('1 l milk', timeout_seconds=timeout_seconds, max_output_tokens=max_output_tokens) if use_llm and extractor is None and not (os.environ.get('GEMINI_API_KEY') and (model or os.environ.get('GEMINI_PARSING_MODEL') or os.environ.get('GEMINI_MODEL'))): raise ValueError('Explicit --llm requires GEMINI_API_KEY and GEMINI_PARSING_MODEL or --model') content = Path(corpus_path).read_bytes() corpus = json.loads(content) cases = [case for case in corpus['cases'] if case['kind'] == 'parse'][:max_cases] if not cases: raise ValueError('Corpus has no parse cases') options = {'model': model, 'timeout_seconds': timeout_seconds, 'max_output_tokens': max_output_tokens} rules = _run(cases, **options) llm = _run(cases, use_llm=True, extractor=extractor, **options) if use_llm else { 'status': 'NOT_RUN', 'reason': 'Model calls require explicit --llm opt-in', 'model_calls': 0} return {'schema_version': 1, 'corpus_version': corpus['version'], 'corpus_sha256': hashlib.sha256(content).hexdigest(), 'case_ids': [case['id'] for case in cases], 'source': 'controlled_versioned_corpus', 'runtime': platform.python_version(), 'source_sha256': {str(path.relative_to(ROOT)): hashlib.sha256(path.read_bytes()).hexdigest() for path in [ROOT / 'backend/flagship/parsing.py', ROOT / 'backend/flagship/basket.py', Path(__file__)]}, 'bounds': {'max_model_calls': max_cases if use_llm else 0, 'timeout_seconds_per_call': timeout_seconds, 'max_output_tokens_per_call': max_output_tokens, 'retries': 0}, 'rules': rules, 'llm_assisted': llm, 'limitations': ['This corpus labels quantity/unit extraction; it does not establish brand or budget accuracy.', 'Model latency and call counts include failed attempts; missing token usage is not zero usage.', 'Controlled parsing cases are not live commerce completion or safety certification.']} if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output', type=Path, required=True) parser.add_argument('--corpus', type=Path, default=CORPUS) parser.add_argument('--llm', action='store_true', help='Explicitly opt in to potentially paid Gemini extraction') parser.add_argument('--max-cases', type=int, default=20) parser.add_argument('--model', help='Explicit Gemini model; otherwise GEMINI_PARSING_MODEL/GEMINI_MODEL') parser.add_argument('--timeout-seconds', type=float, default=15) parser.add_argument('--max-output-tokens', type=int, default=2048) args = parser.parse_args() try: report = evaluate(corpus_path=args.corpus, use_llm=args.llm, max_cases=args.max_cases, model=args.model, timeout_seconds=args.timeout_seconds, max_output_tokens=args.max_output_tokens) except ValueError as exc: parser.error(str(exc)) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2, allow_nan=False) + '\n') print(json.dumps({key: {'status': report[key]['status'], 'passed': report[key].get('passed'), 'cases': report[key].get('cases')} for key in ('rules', 'llm_assisted')}, indent=2)) raise SystemExit(0 if report['rules']['passed'] == report['rules']['cases'] and (not args.llm or report['llm_assisted']['passed'] == report['llm_assisted']['cases']) else 1)