HyperFlow / scripts /eval_intents.py
Gaurav711's picture
Deploy 16191e8
7ff807e verified
Raw History Blame Contribute Delete
12.4 kB
#!/usr/bin/env python3
"""Evaluate complete shopping intents against public synthetic author labels.
Default runs development and validation only. Reserved labels are public, not an
independent unseen test. Without --llm-model this command never calls a model; it never
calls a commerce provider.
"""
import argparse
from collections import Counter
import hashlib
import json
from pathlib import Path
import platform
import statistics
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from backend.flagship.parsing import ExtractionUnavailable, extract_intent
CORPUS = ROOT / 'evaluation/corpus/intents-v1.json'
FIELDS = ('name', 'target_quantity', 'unit', 'preferred_brands', 'disallowed_brands', 'allow_substitution')
OUTCOMES = {'intent', 'clarification', 'rejection', 'clarification_or_rejection'}
def load_corpus(path):
data = json.loads(Path(path).read_text())
cases = data.get('cases')
if not isinstance(cases, list) or len(cases) != 100:
raise ValueError('Corpus requires exactly 100 labelled cases')
counts, groups, identities = Counter(), {}, set()
for case in cases:
if not isinstance(case, dict) or not isinstance(case.get('text'), str) or not case['text'].strip():
raise ValueError('Case requires nonempty text')
if not case.get('id') or case['id'] in identities:
raise ValueError('Case identities must be unique')
identities.add(case['id'])
split = case.get('split')
if split not in {'development', 'validation', 'reserved_test'}:
raise ValueError('Invalid case split')
counts[split] += 1
for name in ('family', 'paraphrase_group'):
value = case.get(name)
if not isinstance(value, str) or not value:
raise ValueError('Case requires family and paraphrase group')
key = (name, value)
if key in groups and groups[key] != split:
raise ValueError('Related groups cannot cross splits')
groups[key] = split
expected = case.get('expected', {})
if expected.get('outcome') not in OUTCOMES:
raise ValueError('Missing expected outcome')
if expected['outcome'] == 'intent':
if not isinstance(expected.get('items'), list) or not expected['items'] or 'budget_minor' not in expected:
raise ValueError('Intent label needs complete items and budget')
for item in expected['items']:
if not isinstance(item, dict) or not set(FIELDS).issubset(item):
raise ValueError('Intent label omitted a constraint')
if dict(counts) != {'development': 60, 'validation': 20, 'reserved_test': 20}:
raise ValueError('Corpus split must be 60/20/20')
return data
def comparable(items, name=None):
result = []
for item in items:
value = {field: item[field] for field in FIELDS}
value['name'] = (name or str.strip)(value['name'].casefold())
for field in ('preferred_brands', 'disallowed_brands'):
value[field] = sorted(x.casefold().strip() for x in value[field])
result.append(value)
return sorted(result, key=lambda value: json.dumps(value, sort_keys=True))
def grade(expected, actual=None, error=None):
observed = 'rejection' if error else ('clarification' if actual.get('clarifications') else 'intent')
wanted = expected['outcome']
if wanted != 'intent':
return observed in ({'clarification', 'rejection'} if wanted == 'clarification_or_rejection' else {wanted}), observed
if observed != 'intent':
return False, observed
try:
correct = (actual.get('budget_minor') == expected['budget_minor']
and comparable(actual.get('items', [])) == comparable(expected['items']))
except (KeyError, TypeError, AttributeError):
correct = False
return correct, observed
def run_cases(corpus, include_reserved, extractor, pace_seconds=0.0, retry_after_seconds=30.0):
results = []
for case in corpus['cases']:
if case['split'] == 'reserved_test' and not include_reserved:
continue
actual, error, unavailable = None, None, None
for attempt in range(2): # one retry after a provider error such as a rate limit
if pace_seconds:
time.sleep(pace_seconds if attempt == 0 else retry_after_seconds)
started = time.perf_counter()
try:
actual, error, unavailable = extractor(case['text']), None, None
break
except ExtractionUnavailable as exc: # provider failure, not a model judgment
unavailable = str(exc)
except ValueError as exc:
error, unavailable = str(exc), None
break
except Exception as exc:
raise RuntimeError('Evaluator execution failed for ' + case['id']) from exc
row = {'id': case['id'], 'split': case['split'], 'family': case['family'], 'expected': case['expected'],
'actual': actual, 'error': error, 'provider_unavailable': unavailable,
'latency_ms': (time.perf_counter() - started) * 1000}
if unavailable:
row.update(passed=None, observed_outcome='provider_unavailable', accepted_constraint_violation=False)
else:
passed, outcome = grade(case['expected'], actual, error)
row.update(passed=passed, observed_outcome=outcome, accepted_constraint_violation=outcome == 'intent' and not passed)
results.append(row)
return results
def name_tolerant_pass(row):
"""Secondary, post-hoc score: every field exact except that the labelled product name only has to
appear inside the extracted name ('rice' in '1 kg rice'). The strict score stays primary."""
if row['passed'] or row['observed_outcome'] != 'intent' or row['expected']['outcome'] != 'intent':
return bool(row['passed'])
actual, expected = row['actual'], row['expected']
if actual.get('budget_minor') != expected['budget_minor'] or len(actual['items']) != len(expected['items']):
return False
strip_name = lambda value: '' if value is None else value
got, want = comparable(actual['items'], name=strip_name), comparable(expected['items'], name=strip_name)
key = lambda v: json.dumps({k: v[k] for k in v if k != 'name'}, sort_keys=True)
got, want = sorted(got, key=key), sorted(want, key=key)
return all(key(g) == key(w) and w['name'] in g['name'] for g, w in zip(got, want))
def summary(rows):
scored = [x for x in rows if x['passed'] is not None]
return {'cases': len(rows), 'scored': len(scored), 'provider_unavailable': len(rows) - len(scored),
'passed': sum(x['passed'] for x in scored),
'passed_name_tolerant': sum(name_tolerant_pass(x) for x in scored),
'exact_completion_rate': sum(x['passed'] for x in scored) / len(scored) if scored else None,
'accepted_constraint_violations': sum(x['accepted_constraint_violation'] for x in scored)}
def evaluate(path=CORPUS, include_reserved=False, extractor=extract_intent):
corpus = load_corpus(path)
results = run_cases(corpus, include_reserved, extractor)
files = [Path(path).resolve(), Path(__file__).resolve(), ROOT / 'backend/flagship/parsing.py', ROOT / 'backend/flagship/basket.py']
durations = sorted(x['latency_ms'] for x in results)
return {'schema_version': 1, 'corpus_id': corpus['corpus_id'], 'source': corpus['provenance'],
'runtime': platform.python_version(), 'captured_at_unix': time.time(),
'source_sha256': {str(p.relative_to(ROOT)) if p.is_relative_to(ROOT) else p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in files},
'rules': summary(results), 'splits': {split: summary([x for x in results if x['split'] == split]) for split in ('development', 'validation', 'reserved_test')},
'reserved_test_status': 'RUN_PUBLIC_SYNTHETIC_LABELS' if include_reserved else 'NOT_RUN',
'llm_assisted': {'status': 'NOT_RUN', 'model_calls': 0}, 'provider_calls': 0,
'latency_ms': {'p50': statistics.median(durations), 'p95': durations[min(len(durations)-1, int(len(durations)*.95))]},
'limitations': ['Public synthetic author labels; no independent human label validation.',
'Unnecessary clarification is not counted as exact intent completion.',
'Parsing quality is distinct from solver correctness and transaction authorization.',
'No model improvement, live commerce success, or general safety claim.'], 'results': results}
SPLIT_PRIORITY = {'reserved_test': 0, 'validation': 1, 'development': 2}
def evaluate_llm(model, path=CORPUS, include_reserved=False, pace_seconds=6.0, previous=None, provider='gemini'):
"""Score model-assisted extraction on the same cases and grader; provider errors are reported separately.
Cases run held-out first, so a daily quota is spent where it matters. Rows already scored in
`previous` (an earlier report with the same model) are reused, so runs can resume across days.
"""
corpus = load_corpus(path)
done = {}
earlier = (previous or {}).get('llm_assisted', {})
if earlier.get('model') == model and earlier.get('provider', 'gemini') == provider:
done = {r['id']: r for r in previous['llm_assisted']['results'] if r['passed'] is not None}
pending = sorted((c for c in corpus['cases'] if c['id'] not in done), key=lambda c: SPLIT_PRIORITY[c['split']])
fresh = run_cases({**corpus, 'cases': pending}, include_reserved,
lambda text: extract_intent(text, use_llm=True, model=model, provider=provider), pace_seconds)
order = {c['id']: n for n, c in enumerate(corpus['cases'])}
rows = sorted([r for r in done.values() if include_reserved or r['split'] != 'reserved_test'] + fresh,
key=lambda r: order[r['id']])
usage = Counter()
for row in rows:
for key, value in ((row['actual'] or {}).get('extraction', {}).get('usage') or {}).items():
usage[key] += value
durations = sorted(x['latency_ms'] for x in rows if not x['provider_unavailable'])
return {'status': 'RUN', 'provider': provider, 'model': model, 'model_calls_scored': len(durations), 'usage_totals': dict(usage),
**summary(rows), 'splits': {split: summary([x for x in rows if x['split'] == split])
for split in ('development', 'validation', 'reserved_test')},
'latency_ms': {'p50': statistics.median(durations), 'p95': durations[min(len(durations) - 1, int(len(durations) * .95))]} if durations else None,
'results': rows}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--corpus', type=Path, default=CORPUS)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--include-reserved', action='store_true')
parser.add_argument('--require-perfect', action='store_true', help='Fail when any scored case fails; default measures baseline honestly')
parser.add_argument('--llm-model', help='Also score Gemini-assisted extraction with this pinned model (needs GEMINI_API_KEY)')
parser.add_argument('--llm-provider', choices=['gemini', 'nvidia'], default='gemini')
parser.add_argument('--llm-pace', type=float, default=6.0, help='Seconds between model calls (free-tier rate limits)')
parser.add_argument('--resume', type=Path, help='Earlier report whose scored model rows are reused')
args = parser.parse_args()
report = evaluate(args.corpus, args.include_reserved)
if args.llm_model:
previous = json.loads(args.resume.read_text()) if args.resume and args.resume.exists() else None
report['llm_assisted'] = evaluate_llm(args.llm_model, args.corpus, args.include_reserved, args.llm_pace, previous, args.llm_provider)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2, allow_nan=False) + '\n')
print(json.dumps({key: report[key] for key in ('rules', 'splits', 'reserved_test_status', 'llm_assisted')}, indent=2))
if args.require_perfect and report['rules']['passed'] != report['rules']['cases']:
raise SystemExit(1)
if __name__ == '__main__':
main()