#!/usr/bin/env python3 """Evaluate complete shopping intents against public synthetic author labels. Default runs development and validation only. Reserved labels are public, not an independent unseen test. Without --llm-model this command never calls a model; it never calls a commerce provider. """ import argparse from collections import Counter import hashlib import json from pathlib import Path import platform import statistics import sys import time ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from backend.flagship.parsing import ExtractionUnavailable, extract_intent CORPUS = ROOT / 'evaluation/corpus/intents-v1.json' FIELDS = ('name', 'target_quantity', 'unit', 'preferred_brands', 'disallowed_brands', 'allow_substitution') OUTCOMES = {'intent', 'clarification', 'rejection', 'clarification_or_rejection'} def load_corpus(path): data = json.loads(Path(path).read_text()) cases = data.get('cases') if not isinstance(cases, list) or len(cases) != 100: raise ValueError('Corpus requires exactly 100 labelled cases') counts, groups, identities = Counter(), {}, set() for case in cases: if not isinstance(case, dict) or not isinstance(case.get('text'), str) or not case['text'].strip(): raise ValueError('Case requires nonempty text') if not case.get('id') or case['id'] in identities: raise ValueError('Case identities must be unique') identities.add(case['id']) split = case.get('split') if split not in {'development', 'validation', 'reserved_test'}: raise ValueError('Invalid case split') counts[split] += 1 for name in ('family', 'paraphrase_group'): value = case.get(name) if not isinstance(value, str) or not value: raise ValueError('Case requires family and paraphrase group') key = (name, value) if key in groups and groups[key] != split: raise ValueError('Related groups cannot cross splits') groups[key] = split expected = case.get('expected', {}) if expected.get('outcome') not in OUTCOMES: raise ValueError('Missing expected outcome') if expected['outcome'] == 'intent': if not isinstance(expected.get('items'), list) or not expected['items'] or 'budget_minor' not in expected: raise ValueError('Intent label needs complete items and budget') for item in expected['items']: if not isinstance(item, dict) or not set(FIELDS).issubset(item): raise ValueError('Intent label omitted a constraint') if dict(counts) != {'development': 60, 'validation': 20, 'reserved_test': 20}: raise ValueError('Corpus split must be 60/20/20') return data def comparable(items, name=None): result = [] for item in items: value = {field: item[field] for field in FIELDS} value['name'] = (name or str.strip)(value['name'].casefold()) for field in ('preferred_brands', 'disallowed_brands'): value[field] = sorted(x.casefold().strip() for x in value[field]) result.append(value) return sorted(result, key=lambda value: json.dumps(value, sort_keys=True)) def grade(expected, actual=None, error=None): observed = 'rejection' if error else ('clarification' if actual.get('clarifications') else 'intent') wanted = expected['outcome'] if wanted != 'intent': return observed in ({'clarification', 'rejection'} if wanted == 'clarification_or_rejection' else {wanted}), observed if observed != 'intent': return False, observed try: correct = (actual.get('budget_minor') == expected['budget_minor'] and comparable(actual.get('items', [])) == comparable(expected['items'])) except (KeyError, TypeError, AttributeError): correct = False return correct, observed def run_cases(corpus, include_reserved, extractor, pace_seconds=0.0, retry_after_seconds=30.0): results = [] for case in corpus['cases']: if case['split'] == 'reserved_test' and not include_reserved: continue actual, error, unavailable = None, None, None for attempt in range(2): # one retry after a provider error such as a rate limit if pace_seconds: time.sleep(pace_seconds if attempt == 0 else retry_after_seconds) started = time.perf_counter() try: actual, error, unavailable = extractor(case['text']), None, None break except ExtractionUnavailable as exc: # provider failure, not a model judgment unavailable = str(exc) except ValueError as exc: error, unavailable = str(exc), None break except Exception as exc: raise RuntimeError('Evaluator execution failed for ' + case['id']) from exc row = {'id': case['id'], 'split': case['split'], 'family': case['family'], 'expected': case['expected'], 'actual': actual, 'error': error, 'provider_unavailable': unavailable, 'latency_ms': (time.perf_counter() - started) * 1000} if unavailable: row.update(passed=None, observed_outcome='provider_unavailable', accepted_constraint_violation=False) else: passed, outcome = grade(case['expected'], actual, error) row.update(passed=passed, observed_outcome=outcome, accepted_constraint_violation=outcome == 'intent' and not passed) results.append(row) return results def name_tolerant_pass(row): """Secondary, post-hoc score: every field exact except that the labelled product name only has to appear inside the extracted name ('rice' in '1 kg rice'). The strict score stays primary.""" if row['passed'] or row['observed_outcome'] != 'intent' or row['expected']['outcome'] != 'intent': return bool(row['passed']) actual, expected = row['actual'], row['expected'] if actual.get('budget_minor') != expected['budget_minor'] or len(actual['items']) != len(expected['items']): return False strip_name = lambda value: '' if value is None else value got, want = comparable(actual['items'], name=strip_name), comparable(expected['items'], name=strip_name) key = lambda v: json.dumps({k: v[k] for k in v if k != 'name'}, sort_keys=True) got, want = sorted(got, key=key), sorted(want, key=key) return all(key(g) == key(w) and w['name'] in g['name'] for g, w in zip(got, want)) def summary(rows): scored = [x for x in rows if x['passed'] is not None] return {'cases': len(rows), 'scored': len(scored), 'provider_unavailable': len(rows) - len(scored), 'passed': sum(x['passed'] for x in scored), 'passed_name_tolerant': sum(name_tolerant_pass(x) for x in scored), 'exact_completion_rate': sum(x['passed'] for x in scored) / len(scored) if scored else None, 'accepted_constraint_violations': sum(x['accepted_constraint_violation'] for x in scored)} def evaluate(path=CORPUS, include_reserved=False, extractor=extract_intent): corpus = load_corpus(path) results = run_cases(corpus, include_reserved, extractor) files = [Path(path).resolve(), Path(__file__).resolve(), ROOT / 'backend/flagship/parsing.py', ROOT / 'backend/flagship/basket.py'] durations = sorted(x['latency_ms'] for x in results) return {'schema_version': 1, 'corpus_id': corpus['corpus_id'], 'source': corpus['provenance'], 'runtime': platform.python_version(), 'captured_at_unix': time.time(), 'source_sha256': {str(p.relative_to(ROOT)) if p.is_relative_to(ROOT) else p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in files}, 'rules': summary(results), 'splits': {split: summary([x for x in results if x['split'] == split]) for split in ('development', 'validation', 'reserved_test')}, 'reserved_test_status': 'RUN_PUBLIC_SYNTHETIC_LABELS' if include_reserved else 'NOT_RUN', 'llm_assisted': {'status': 'NOT_RUN', 'model_calls': 0}, 'provider_calls': 0, 'latency_ms': {'p50': statistics.median(durations), 'p95': durations[min(len(durations)-1, int(len(durations)*.95))]}, 'limitations': ['Public synthetic author labels; no independent human label validation.', 'Unnecessary clarification is not counted as exact intent completion.', 'Parsing quality is distinct from solver correctness and transaction authorization.', 'No model improvement, live commerce success, or general safety claim.'], 'results': results} SPLIT_PRIORITY = {'reserved_test': 0, 'validation': 1, 'development': 2} def evaluate_llm(model, path=CORPUS, include_reserved=False, pace_seconds=6.0, previous=None, provider='gemini'): """Score model-assisted extraction on the same cases and grader; provider errors are reported separately. Cases run held-out first, so a daily quota is spent where it matters. Rows already scored in `previous` (an earlier report with the same model) are reused, so runs can resume across days. """ corpus = load_corpus(path) done = {} earlier = (previous or {}).get('llm_assisted', {}) if earlier.get('model') == model and earlier.get('provider', 'gemini') == provider: done = {r['id']: r for r in previous['llm_assisted']['results'] if r['passed'] is not None} pending = sorted((c for c in corpus['cases'] if c['id'] not in done), key=lambda c: SPLIT_PRIORITY[c['split']]) fresh = run_cases({**corpus, 'cases': pending}, include_reserved, lambda text: extract_intent(text, use_llm=True, model=model, provider=provider), pace_seconds) order = {c['id']: n for n, c in enumerate(corpus['cases'])} rows = sorted([r for r in done.values() if include_reserved or r['split'] != 'reserved_test'] + fresh, key=lambda r: order[r['id']]) usage = Counter() for row in rows: for key, value in ((row['actual'] or {}).get('extraction', {}).get('usage') or {}).items(): usage[key] += value durations = sorted(x['latency_ms'] for x in rows if not x['provider_unavailable']) return {'status': 'RUN', 'provider': provider, 'model': model, 'model_calls_scored': len(durations), 'usage_totals': dict(usage), **summary(rows), 'splits': {split: summary([x for x in rows if x['split'] == split]) for split in ('development', 'validation', 'reserved_test')}, 'latency_ms': {'p50': statistics.median(durations), 'p95': durations[min(len(durations) - 1, int(len(durations) * .95))]} if durations else None, 'results': rows} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--corpus', type=Path, default=CORPUS) parser.add_argument('--output', type=Path, required=True) parser.add_argument('--include-reserved', action='store_true') parser.add_argument('--require-perfect', action='store_true', help='Fail when any scored case fails; default measures baseline honestly') parser.add_argument('--llm-model', help='Also score Gemini-assisted extraction with this pinned model (needs GEMINI_API_KEY)') parser.add_argument('--llm-provider', choices=['gemini', 'nvidia'], default='gemini') parser.add_argument('--llm-pace', type=float, default=6.0, help='Seconds between model calls (free-tier rate limits)') parser.add_argument('--resume', type=Path, help='Earlier report whose scored model rows are reused') args = parser.parse_args() report = evaluate(args.corpus, args.include_reserved) if args.llm_model: previous = json.loads(args.resume.read_text()) if args.resume and args.resume.exists() else None report['llm_assisted'] = evaluate_llm(args.llm_model, args.corpus, args.include_reserved, args.llm_pace, previous, args.llm_provider) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2, allow_nan=False) + '\n') print(json.dumps({key: report[key] for key in ('rules', 'splits', 'reserved_test_status', 'llm_assisted')}, indent=2)) if args.require_perfect and report['rules']['passed'] != report['rules']['cases']: raise SystemExit(1) if __name__ == '__main__': main()