File size: 6,964 Bytes
2b86c60
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
#!/usr/bin/env python3
"""Measure rules and optionally Gemini on the same bounded corpus slice.

Offline default: python scripts/eval_parsing.py --output /tmp/parsing.json
Paid opt-in: set GEMINI_API_KEY and GEMINI_PARSING_MODEL, then add --llm
--max-cases 20 --timeout-seconds 15 --max-output-tokens 2048. There are no
retries, provider actions, purchases, or model calls without --llm.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path
import platform
import statistics
import sys
import time

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from backend.flagship.parsing import extract_intent, ExtractionUnavailable

ROOT = Path(__file__).resolve().parents[1]
CORPUS = ROOT / 'evaluation/corpus/commerce-v1.json'


def _run(cases, *, use_llm=False, extractor=None, **options):
    results = []
    for case in cases:
        started = time.perf_counter()
        try:
            intent = extract_intent(case['text'], use_llm=use_llm, extractor=extractor, **options)
            actual = [(x['target_quantity'], x['unit']) for x in intent['items']]
            passed = actual == [tuple(x) for x in case['expected']] and not intent['clarifications']
            record = {'passed': passed, 'status': 'PARSED' if not intent['clarifications'] else 'NEEDS_CLARIFICATION',
                      'actual': actual, 'constraint_violations': 0,
                      'token_usage': intent['extraction']['usage']}
        except ExtractionUnavailable as exc:
            record = {'passed': False, 'status': 'UNAVAILABLE', 'error': str(exc),
                      'constraint_violations': None, 'token_usage': None}
        except ValueError as exc:
            record = {'passed': False, 'status': 'REJECTED', 'error': str(exc),
                      'constraint_violations': 1, 'token_usage': None}
        except Exception:
            record = {'passed': False, 'status': 'FAILED', 'error': 'Extractor failed; no intent accepted',
                      'constraint_violations': None, 'token_usage': None}
        results.append({'id': case['id'], 'family': case['family'], **record,
                        'latency_ms': (time.perf_counter() - started) * 1000,
                        'model_calls': int(use_llm), 'provider_action_calls': 0})
    latencies = sorted(case['latency_ms'] for case in results)
    return {'status': 'RUN', 'cases': len(results), 'passed': sum(case['passed'] for case in results),
            'completion_rate': sum(case['passed'] for case in results) / len(results),
            'constraint_violations': sum(case['constraint_violations'] or 0 for case in results),
            'model_calls': sum(case['model_calls'] for case in results), 'provider_action_calls': 0,
            'p50_ms': statistics.median(latencies),
            'p95_ms': latencies[min(len(latencies) - 1, int(len(latencies) * .95))], 'results': results}


def evaluate(*, corpus_path=CORPUS, use_llm=False, max_cases=20, extractor=None,
             model=None, timeout_seconds=15, max_output_tokens=2048):
    if type(max_cases) is not int or not 1 <= max_cases <= 20:
        raise ValueError('max_cases must be between 1 and 20 per run')
    if type(use_llm) is not bool:
        raise ValueError('use_llm must be a boolean')
    # Validate bounds before starting the measured corpus, without any model call.
    extract_intent('1 l milk', timeout_seconds=timeout_seconds, max_output_tokens=max_output_tokens)
    if use_llm and extractor is None and not (os.environ.get('GEMINI_API_KEY') and
            (model or os.environ.get('GEMINI_PARSING_MODEL') or os.environ.get('GEMINI_MODEL'))):
        raise ValueError('Explicit --llm requires GEMINI_API_KEY and GEMINI_PARSING_MODEL or --model')
    content = Path(corpus_path).read_bytes()
    corpus = json.loads(content)
    cases = [case for case in corpus['cases'] if case['kind'] == 'parse'][:max_cases]
    if not cases:
        raise ValueError('Corpus has no parse cases')
    options = {'model': model, 'timeout_seconds': timeout_seconds, 'max_output_tokens': max_output_tokens}
    rules = _run(cases, **options)
    llm = _run(cases, use_llm=True, extractor=extractor, **options) if use_llm else {
        'status': 'NOT_RUN', 'reason': 'Model calls require explicit --llm opt-in', 'model_calls': 0}
    return {'schema_version': 1, 'corpus_version': corpus['version'],
            'corpus_sha256': hashlib.sha256(content).hexdigest(), 'case_ids': [case['id'] for case in cases],
            'source': 'controlled_versioned_corpus', 'runtime': platform.python_version(),
            'source_sha256': {str(path.relative_to(ROOT)): hashlib.sha256(path.read_bytes()).hexdigest()
                              for path in [ROOT / 'backend/flagship/parsing.py', ROOT / 'backend/flagship/basket.py', Path(__file__)]},
            'bounds': {'max_model_calls': max_cases if use_llm else 0,
                       'timeout_seconds_per_call': timeout_seconds, 'max_output_tokens_per_call': max_output_tokens,
                       'retries': 0},
            'rules': rules, 'llm_assisted': llm,
            'limitations': ['This corpus labels quantity/unit extraction; it does not establish brand or budget accuracy.',
                            'Model latency and call counts include failed attempts; missing token usage is not zero usage.',
                            'Controlled parsing cases are not live commerce completion or safety certification.']}


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--output', type=Path, required=True)
    parser.add_argument('--corpus', type=Path, default=CORPUS)
    parser.add_argument('--llm', action='store_true', help='Explicitly opt in to potentially paid Gemini extraction')
    parser.add_argument('--max-cases', type=int, default=20)
    parser.add_argument('--model', help='Explicit Gemini model; otherwise GEMINI_PARSING_MODEL/GEMINI_MODEL')
    parser.add_argument('--timeout-seconds', type=float, default=15)
    parser.add_argument('--max-output-tokens', type=int, default=2048)
    args = parser.parse_args()
    try:
        report = evaluate(corpus_path=args.corpus, use_llm=args.llm, max_cases=args.max_cases,
                          model=args.model, timeout_seconds=args.timeout_seconds, max_output_tokens=args.max_output_tokens)
    except ValueError as exc:
        parser.error(str(exc))
    args.output.parent.mkdir(parents=True, exist_ok=True)
    args.output.write_text(json.dumps(report, indent=2, allow_nan=False) + '\n')
    print(json.dumps({key: {'status': report[key]['status'], 'passed': report[key].get('passed'),
                           'cases': report[key].get('cases')} for key in ('rules', 'llm_assisted')}, indent=2))
    raise SystemExit(0 if report['rules']['passed'] == report['rules']['cases'] and
                     (not args.llm or report['llm_assisted']['passed'] == report['llm_assisted']['cases']) else 1)