File size: 12,423 Bytes
2b86c60 64b0986 2b86c60 64b0986 2b86c60 7ff807e 2b86c60 7ff807e 2b86c60 64b0986 2b86c60 64b0986 7ff807e 64b0986 7ff807e 64b0986 2b86c60 64b0986 7ff807e 64b0986 7ff807e 64b0986 7ff807e 64b0986 7ff807e 64b0986 2b86c60 64b0986 7ff807e 64b0986 2b86c60 64b0986 7ff807e 2b86c60 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 | #!/usr/bin/env python3
"""Evaluate complete shopping intents against public synthetic author labels.
Default runs development and validation only. Reserved labels are public, not an
independent unseen test. Without --llm-model this command never calls a model; it never
calls a commerce provider.
"""
import argparse
from collections import Counter
import hashlib
import json
from pathlib import Path
import platform
import statistics
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from backend.flagship.parsing import ExtractionUnavailable, extract_intent
CORPUS = ROOT / 'evaluation/corpus/intents-v1.json'
FIELDS = ('name', 'target_quantity', 'unit', 'preferred_brands', 'disallowed_brands', 'allow_substitution')
OUTCOMES = {'intent', 'clarification', 'rejection', 'clarification_or_rejection'}
def load_corpus(path):
data = json.loads(Path(path).read_text())
cases = data.get('cases')
if not isinstance(cases, list) or len(cases) != 100:
raise ValueError('Corpus requires exactly 100 labelled cases')
counts, groups, identities = Counter(), {}, set()
for case in cases:
if not isinstance(case, dict) or not isinstance(case.get('text'), str) or not case['text'].strip():
raise ValueError('Case requires nonempty text')
if not case.get('id') or case['id'] in identities:
raise ValueError('Case identities must be unique')
identities.add(case['id'])
split = case.get('split')
if split not in {'development', 'validation', 'reserved_test'}:
raise ValueError('Invalid case split')
counts[split] += 1
for name in ('family', 'paraphrase_group'):
value = case.get(name)
if not isinstance(value, str) or not value:
raise ValueError('Case requires family and paraphrase group')
key = (name, value)
if key in groups and groups[key] != split:
raise ValueError('Related groups cannot cross splits')
groups[key] = split
expected = case.get('expected', {})
if expected.get('outcome') not in OUTCOMES:
raise ValueError('Missing expected outcome')
if expected['outcome'] == 'intent':
if not isinstance(expected.get('items'), list) or not expected['items'] or 'budget_minor' not in expected:
raise ValueError('Intent label needs complete items and budget')
for item in expected['items']:
if not isinstance(item, dict) or not set(FIELDS).issubset(item):
raise ValueError('Intent label omitted a constraint')
if dict(counts) != {'development': 60, 'validation': 20, 'reserved_test': 20}:
raise ValueError('Corpus split must be 60/20/20')
return data
def comparable(items, name=None):
result = []
for item in items:
value = {field: item[field] for field in FIELDS}
value['name'] = (name or str.strip)(value['name'].casefold())
for field in ('preferred_brands', 'disallowed_brands'):
value[field] = sorted(x.casefold().strip() for x in value[field])
result.append(value)
return sorted(result, key=lambda value: json.dumps(value, sort_keys=True))
def grade(expected, actual=None, error=None):
observed = 'rejection' if error else ('clarification' if actual.get('clarifications') else 'intent')
wanted = expected['outcome']
if wanted != 'intent':
return observed in ({'clarification', 'rejection'} if wanted == 'clarification_or_rejection' else {wanted}), observed
if observed != 'intent':
return False, observed
try:
correct = (actual.get('budget_minor') == expected['budget_minor']
and comparable(actual.get('items', [])) == comparable(expected['items']))
except (KeyError, TypeError, AttributeError):
correct = False
return correct, observed
def run_cases(corpus, include_reserved, extractor, pace_seconds=0.0, retry_after_seconds=30.0):
results = []
for case in corpus['cases']:
if case['split'] == 'reserved_test' and not include_reserved:
continue
actual, error, unavailable = None, None, None
for attempt in range(2): # one retry after a provider error such as a rate limit
if pace_seconds:
time.sleep(pace_seconds if attempt == 0 else retry_after_seconds)
started = time.perf_counter()
try:
actual, error, unavailable = extractor(case['text']), None, None
break
except ExtractionUnavailable as exc: # provider failure, not a model judgment
unavailable = str(exc)
except ValueError as exc:
error, unavailable = str(exc), None
break
except Exception as exc:
raise RuntimeError('Evaluator execution failed for ' + case['id']) from exc
row = {'id': case['id'], 'split': case['split'], 'family': case['family'], 'expected': case['expected'],
'actual': actual, 'error': error, 'provider_unavailable': unavailable,
'latency_ms': (time.perf_counter() - started) * 1000}
if unavailable:
row.update(passed=None, observed_outcome='provider_unavailable', accepted_constraint_violation=False)
else:
passed, outcome = grade(case['expected'], actual, error)
row.update(passed=passed, observed_outcome=outcome, accepted_constraint_violation=outcome == 'intent' and not passed)
results.append(row)
return results
def name_tolerant_pass(row):
"""Secondary, post-hoc score: every field exact except that the labelled product name only has to
appear inside the extracted name ('rice' in '1 kg rice'). The strict score stays primary."""
if row['passed'] or row['observed_outcome'] != 'intent' or row['expected']['outcome'] != 'intent':
return bool(row['passed'])
actual, expected = row['actual'], row['expected']
if actual.get('budget_minor') != expected['budget_minor'] or len(actual['items']) != len(expected['items']):
return False
strip_name = lambda value: '' if value is None else value
got, want = comparable(actual['items'], name=strip_name), comparable(expected['items'], name=strip_name)
key = lambda v: json.dumps({k: v[k] for k in v if k != 'name'}, sort_keys=True)
got, want = sorted(got, key=key), sorted(want, key=key)
return all(key(g) == key(w) and w['name'] in g['name'] for g, w in zip(got, want))
def summary(rows):
scored = [x for x in rows if x['passed'] is not None]
return {'cases': len(rows), 'scored': len(scored), 'provider_unavailable': len(rows) - len(scored),
'passed': sum(x['passed'] for x in scored),
'passed_name_tolerant': sum(name_tolerant_pass(x) for x in scored),
'exact_completion_rate': sum(x['passed'] for x in scored) / len(scored) if scored else None,
'accepted_constraint_violations': sum(x['accepted_constraint_violation'] for x in scored)}
def evaluate(path=CORPUS, include_reserved=False, extractor=extract_intent):
corpus = load_corpus(path)
results = run_cases(corpus, include_reserved, extractor)
files = [Path(path).resolve(), Path(__file__).resolve(), ROOT / 'backend/flagship/parsing.py', ROOT / 'backend/flagship/basket.py']
durations = sorted(x['latency_ms'] for x in results)
return {'schema_version': 1, 'corpus_id': corpus['corpus_id'], 'source': corpus['provenance'],
'runtime': platform.python_version(), 'captured_at_unix': time.time(),
'source_sha256': {str(p.relative_to(ROOT)) if p.is_relative_to(ROOT) else p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in files},
'rules': summary(results), 'splits': {split: summary([x for x in results if x['split'] == split]) for split in ('development', 'validation', 'reserved_test')},
'reserved_test_status': 'RUN_PUBLIC_SYNTHETIC_LABELS' if include_reserved else 'NOT_RUN',
'llm_assisted': {'status': 'NOT_RUN', 'model_calls': 0}, 'provider_calls': 0,
'latency_ms': {'p50': statistics.median(durations), 'p95': durations[min(len(durations)-1, int(len(durations)*.95))]},
'limitations': ['Public synthetic author labels; no independent human label validation.',
'Unnecessary clarification is not counted as exact intent completion.',
'Parsing quality is distinct from solver correctness and transaction authorization.',
'No model improvement, live commerce success, or general safety claim.'], 'results': results}
SPLIT_PRIORITY = {'reserved_test': 0, 'validation': 1, 'development': 2}
def evaluate_llm(model, path=CORPUS, include_reserved=False, pace_seconds=6.0, previous=None, provider='gemini'):
"""Score model-assisted extraction on the same cases and grader; provider errors are reported separately.
Cases run held-out first, so a daily quota is spent where it matters. Rows already scored in
`previous` (an earlier report with the same model) are reused, so runs can resume across days.
"""
corpus = load_corpus(path)
done = {}
earlier = (previous or {}).get('llm_assisted', {})
if earlier.get('model') == model and earlier.get('provider', 'gemini') == provider:
done = {r['id']: r for r in previous['llm_assisted']['results'] if r['passed'] is not None}
pending = sorted((c for c in corpus['cases'] if c['id'] not in done), key=lambda c: SPLIT_PRIORITY[c['split']])
fresh = run_cases({**corpus, 'cases': pending}, include_reserved,
lambda text: extract_intent(text, use_llm=True, model=model, provider=provider), pace_seconds)
order = {c['id']: n for n, c in enumerate(corpus['cases'])}
rows = sorted([r for r in done.values() if include_reserved or r['split'] != 'reserved_test'] + fresh,
key=lambda r: order[r['id']])
usage = Counter()
for row in rows:
for key, value in ((row['actual'] or {}).get('extraction', {}).get('usage') or {}).items():
usage[key] += value
durations = sorted(x['latency_ms'] for x in rows if not x['provider_unavailable'])
return {'status': 'RUN', 'provider': provider, 'model': model, 'model_calls_scored': len(durations), 'usage_totals': dict(usage),
**summary(rows), 'splits': {split: summary([x for x in rows if x['split'] == split])
for split in ('development', 'validation', 'reserved_test')},
'latency_ms': {'p50': statistics.median(durations), 'p95': durations[min(len(durations) - 1, int(len(durations) * .95))]} if durations else None,
'results': rows}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--corpus', type=Path, default=CORPUS)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--include-reserved', action='store_true')
parser.add_argument('--require-perfect', action='store_true', help='Fail when any scored case fails; default measures baseline honestly')
parser.add_argument('--llm-model', help='Also score Gemini-assisted extraction with this pinned model (needs GEMINI_API_KEY)')
parser.add_argument('--llm-provider', choices=['gemini', 'nvidia'], default='gemini')
parser.add_argument('--llm-pace', type=float, default=6.0, help='Seconds between model calls (free-tier rate limits)')
parser.add_argument('--resume', type=Path, help='Earlier report whose scored model rows are reused')
args = parser.parse_args()
report = evaluate(args.corpus, args.include_reserved)
if args.llm_model:
previous = json.loads(args.resume.read_text()) if args.resume and args.resume.exists() else None
report['llm_assisted'] = evaluate_llm(args.llm_model, args.corpus, args.include_reserved, args.llm_pace, previous, args.llm_provider)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2, allow_nan=False) + '\n')
print(json.dumps({key: report[key] for key in ('rules', 'splits', 'reserved_test_status', 'llm_assisted')}, indent=2))
if args.require_perfect and report['rules']['passed'] != report['rules']['cases']:
raise SystemExit(1)
if __name__ == '__main__':
main()
|