File size: 6,964 Bytes
2b86c60 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 | #!/usr/bin/env python3
"""Measure rules and optionally Gemini on the same bounded corpus slice.
Offline default: python scripts/eval_parsing.py --output /tmp/parsing.json
Paid opt-in: set GEMINI_API_KEY and GEMINI_PARSING_MODEL, then add --llm
--max-cases 20 --timeout-seconds 15 --max-output-tokens 2048. There are no
retries, provider actions, purchases, or model calls without --llm.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path
import platform
import statistics
import sys
import time
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from backend.flagship.parsing import extract_intent, ExtractionUnavailable
ROOT = Path(__file__).resolve().parents[1]
CORPUS = ROOT / 'evaluation/corpus/commerce-v1.json'
def _run(cases, *, use_llm=False, extractor=None, **options):
results = []
for case in cases:
started = time.perf_counter()
try:
intent = extract_intent(case['text'], use_llm=use_llm, extractor=extractor, **options)
actual = [(x['target_quantity'], x['unit']) for x in intent['items']]
passed = actual == [tuple(x) for x in case['expected']] and not intent['clarifications']
record = {'passed': passed, 'status': 'PARSED' if not intent['clarifications'] else 'NEEDS_CLARIFICATION',
'actual': actual, 'constraint_violations': 0,
'token_usage': intent['extraction']['usage']}
except ExtractionUnavailable as exc:
record = {'passed': False, 'status': 'UNAVAILABLE', 'error': str(exc),
'constraint_violations': None, 'token_usage': None}
except ValueError as exc:
record = {'passed': False, 'status': 'REJECTED', 'error': str(exc),
'constraint_violations': 1, 'token_usage': None}
except Exception:
record = {'passed': False, 'status': 'FAILED', 'error': 'Extractor failed; no intent accepted',
'constraint_violations': None, 'token_usage': None}
results.append({'id': case['id'], 'family': case['family'], **record,
'latency_ms': (time.perf_counter() - started) * 1000,
'model_calls': int(use_llm), 'provider_action_calls': 0})
latencies = sorted(case['latency_ms'] for case in results)
return {'status': 'RUN', 'cases': len(results), 'passed': sum(case['passed'] for case in results),
'completion_rate': sum(case['passed'] for case in results) / len(results),
'constraint_violations': sum(case['constraint_violations'] or 0 for case in results),
'model_calls': sum(case['model_calls'] for case in results), 'provider_action_calls': 0,
'p50_ms': statistics.median(latencies),
'p95_ms': latencies[min(len(latencies) - 1, int(len(latencies) * .95))], 'results': results}
def evaluate(*, corpus_path=CORPUS, use_llm=False, max_cases=20, extractor=None,
model=None, timeout_seconds=15, max_output_tokens=2048):
if type(max_cases) is not int or not 1 <= max_cases <= 20:
raise ValueError('max_cases must be between 1 and 20 per run')
if type(use_llm) is not bool:
raise ValueError('use_llm must be a boolean')
# Validate bounds before starting the measured corpus, without any model call.
extract_intent('1 l milk', timeout_seconds=timeout_seconds, max_output_tokens=max_output_tokens)
if use_llm and extractor is None and not (os.environ.get('GEMINI_API_KEY') and
(model or os.environ.get('GEMINI_PARSING_MODEL') or os.environ.get('GEMINI_MODEL'))):
raise ValueError('Explicit --llm requires GEMINI_API_KEY and GEMINI_PARSING_MODEL or --model')
content = Path(corpus_path).read_bytes()
corpus = json.loads(content)
cases = [case for case in corpus['cases'] if case['kind'] == 'parse'][:max_cases]
if not cases:
raise ValueError('Corpus has no parse cases')
options = {'model': model, 'timeout_seconds': timeout_seconds, 'max_output_tokens': max_output_tokens}
rules = _run(cases, **options)
llm = _run(cases, use_llm=True, extractor=extractor, **options) if use_llm else {
'status': 'NOT_RUN', 'reason': 'Model calls require explicit --llm opt-in', 'model_calls': 0}
return {'schema_version': 1, 'corpus_version': corpus['version'],
'corpus_sha256': hashlib.sha256(content).hexdigest(), 'case_ids': [case['id'] for case in cases],
'source': 'controlled_versioned_corpus', 'runtime': platform.python_version(),
'source_sha256': {str(path.relative_to(ROOT)): hashlib.sha256(path.read_bytes()).hexdigest()
for path in [ROOT / 'backend/flagship/parsing.py', ROOT / 'backend/flagship/basket.py', Path(__file__)]},
'bounds': {'max_model_calls': max_cases if use_llm else 0,
'timeout_seconds_per_call': timeout_seconds, 'max_output_tokens_per_call': max_output_tokens,
'retries': 0},
'rules': rules, 'llm_assisted': llm,
'limitations': ['This corpus labels quantity/unit extraction; it does not establish brand or budget accuracy.',
'Model latency and call counts include failed attempts; missing token usage is not zero usage.',
'Controlled parsing cases are not live commerce completion or safety certification.']}
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--corpus', type=Path, default=CORPUS)
parser.add_argument('--llm', action='store_true', help='Explicitly opt in to potentially paid Gemini extraction')
parser.add_argument('--max-cases', type=int, default=20)
parser.add_argument('--model', help='Explicit Gemini model; otherwise GEMINI_PARSING_MODEL/GEMINI_MODEL')
parser.add_argument('--timeout-seconds', type=float, default=15)
parser.add_argument('--max-output-tokens', type=int, default=2048)
args = parser.parse_args()
try:
report = evaluate(corpus_path=args.corpus, use_llm=args.llm, max_cases=args.max_cases,
model=args.model, timeout_seconds=args.timeout_seconds, max_output_tokens=args.max_output_tokens)
except ValueError as exc:
parser.error(str(exc))
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2, allow_nan=False) + '\n')
print(json.dumps({key: {'status': report[key]['status'], 'passed': report[key].get('passed'),
'cases': report[key].get('cases')} for key in ('rules', 'llm_assisted')}, indent=2))
raise SystemExit(0 if report['rules']['passed'] == report['rules']['cases'] and
(not args.llm or report['llm_assisted']['passed'] == report['llm_assisted']['cases']) else 1)
|