Download scripts/eval_intents.py from Gaurav711/HyperFlow: direct link, hf CLI and curl.
- Browser
- Download file 12.4 kB
-
https://huggingface.co/spaces/Gaurav711/HyperFlow/resolve/main/scripts/eval_intents.py
- Command line
-
hf download hf://spaces/Gaurav711/HyperFlow/scripts/eval_intents.py
-
curl -L -o eval_intents.py https://huggingface.co/spaces/Gaurav711/HyperFlow/resolve/main/scripts/eval_intents.py
12.4 kB
| #!/usr/bin/env python3 | |
| """Evaluate complete shopping intents against public synthetic author labels. | |
| Default runs development and validation only. Reserved labels are public, not an | |
| independent unseen test. Without --llm-model this command never calls a model; it never | |
| calls a commerce provider. | |
| """ | |
| import argparse | |
| from collections import Counter | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import platform | |
| import statistics | |
| import sys | |
| import time | |
| ROOT = Path(__file__).resolve().parents[1] | |
| sys.path.insert(0, str(ROOT)) | |
| from backend.flagship.parsing import ExtractionUnavailable, extract_intent | |
| CORPUS = ROOT / 'evaluation/corpus/intents-v1.json' | |
| FIELDS = ('name', 'target_quantity', 'unit', 'preferred_brands', 'disallowed_brands', 'allow_substitution') | |
| OUTCOMES = {'intent', 'clarification', 'rejection', 'clarification_or_rejection'} | |
| def load_corpus(path): | |
| data = json.loads(Path(path).read_text()) | |
| cases = data.get('cases') | |
| if not isinstance(cases, list) or len(cases) != 100: | |
| raise ValueError('Corpus requires exactly 100 labelled cases') | |
| counts, groups, identities = Counter(), {}, set() | |
| for case in cases: | |
| if not isinstance(case, dict) or not isinstance(case.get('text'), str) or not case['text'].strip(): | |
| raise ValueError('Case requires nonempty text') | |
| if not case.get('id') or case['id'] in identities: | |
| raise ValueError('Case identities must be unique') | |
| identities.add(case['id']) | |
| split = case.get('split') | |
| if split not in {'development', 'validation', 'reserved_test'}: | |
| raise ValueError('Invalid case split') | |
| counts[split] += 1 | |
| for name in ('family', 'paraphrase_group'): | |
| value = case.get(name) | |
| if not isinstance(value, str) or not value: | |
| raise ValueError('Case requires family and paraphrase group') | |
| key = (name, value) | |
| if key in groups and groups[key] != split: | |
| raise ValueError('Related groups cannot cross splits') | |
| groups[key] = split | |
| expected = case.get('expected', {}) | |
| if expected.get('outcome') not in OUTCOMES: | |
| raise ValueError('Missing expected outcome') | |
| if expected['outcome'] == 'intent': | |
| if not isinstance(expected.get('items'), list) or not expected['items'] or 'budget_minor' not in expected: | |
| raise ValueError('Intent label needs complete items and budget') | |
| for item in expected['items']: | |
| if not isinstance(item, dict) or not set(FIELDS).issubset(item): | |
| raise ValueError('Intent label omitted a constraint') | |
| if dict(counts) != {'development': 60, 'validation': 20, 'reserved_test': 20}: | |
| raise ValueError('Corpus split must be 60/20/20') | |
| return data | |
| def comparable(items, name=None): | |
| result = [] | |
| for item in items: | |
| value = {field: item[field] for field in FIELDS} | |
| value['name'] = (name or str.strip)(value['name'].casefold()) | |
| for field in ('preferred_brands', 'disallowed_brands'): | |
| value[field] = sorted(x.casefold().strip() for x in value[field]) | |
| result.append(value) | |
| return sorted(result, key=lambda value: json.dumps(value, sort_keys=True)) | |
| def grade(expected, actual=None, error=None): | |
| observed = 'rejection' if error else ('clarification' if actual.get('clarifications') else 'intent') | |
| wanted = expected['outcome'] | |
| if wanted != 'intent': | |
| return observed in ({'clarification', 'rejection'} if wanted == 'clarification_or_rejection' else {wanted}), observed | |
| if observed != 'intent': | |
| return False, observed | |
| try: | |
| correct = (actual.get('budget_minor') == expected['budget_minor'] | |
| and comparable(actual.get('items', [])) == comparable(expected['items'])) | |
| except (KeyError, TypeError, AttributeError): | |
| correct = False | |
| return correct, observed | |
| def run_cases(corpus, include_reserved, extractor, pace_seconds=0.0, retry_after_seconds=30.0): | |
| results = [] | |
| for case in corpus['cases']: | |
| if case['split'] == 'reserved_test' and not include_reserved: | |
| continue | |
| actual, error, unavailable = None, None, None | |
| for attempt in range(2): # one retry after a provider error such as a rate limit | |
| if pace_seconds: | |
| time.sleep(pace_seconds if attempt == 0 else retry_after_seconds) | |
| started = time.perf_counter() | |
| try: | |
| actual, error, unavailable = extractor(case['text']), None, None | |
| break | |
| except ExtractionUnavailable as exc: # provider failure, not a model judgment | |
| unavailable = str(exc) | |
| except ValueError as exc: | |
| error, unavailable = str(exc), None | |
| break | |
| except Exception as exc: | |
| raise RuntimeError('Evaluator execution failed for ' + case['id']) from exc | |
| row = {'id': case['id'], 'split': case['split'], 'family': case['family'], 'expected': case['expected'], | |
| 'actual': actual, 'error': error, 'provider_unavailable': unavailable, | |
| 'latency_ms': (time.perf_counter() - started) * 1000} | |
| if unavailable: | |
| row.update(passed=None, observed_outcome='provider_unavailable', accepted_constraint_violation=False) | |
| else: | |
| passed, outcome = grade(case['expected'], actual, error) | |
| row.update(passed=passed, observed_outcome=outcome, accepted_constraint_violation=outcome == 'intent' and not passed) | |
| results.append(row) | |
| return results | |
| def name_tolerant_pass(row): | |
| """Secondary, post-hoc score: every field exact except that the labelled product name only has to | |
| appear inside the extracted name ('rice' in '1 kg rice'). The strict score stays primary.""" | |
| if row['passed'] or row['observed_outcome'] != 'intent' or row['expected']['outcome'] != 'intent': | |
| return bool(row['passed']) | |
| actual, expected = row['actual'], row['expected'] | |
| if actual.get('budget_minor') != expected['budget_minor'] or len(actual['items']) != len(expected['items']): | |
| return False | |
| strip_name = lambda value: '' if value is None else value | |
| got, want = comparable(actual['items'], name=strip_name), comparable(expected['items'], name=strip_name) | |
| key = lambda v: json.dumps({k: v[k] for k in v if k != 'name'}, sort_keys=True) | |
| got, want = sorted(got, key=key), sorted(want, key=key) | |
| return all(key(g) == key(w) and w['name'] in g['name'] for g, w in zip(got, want)) | |
| def summary(rows): | |
| scored = [x for x in rows if x['passed'] is not None] | |
| return {'cases': len(rows), 'scored': len(scored), 'provider_unavailable': len(rows) - len(scored), | |
| 'passed': sum(x['passed'] for x in scored), | |
| 'passed_name_tolerant': sum(name_tolerant_pass(x) for x in scored), | |
| 'exact_completion_rate': sum(x['passed'] for x in scored) / len(scored) if scored else None, | |
| 'accepted_constraint_violations': sum(x['accepted_constraint_violation'] for x in scored)} | |
| def evaluate(path=CORPUS, include_reserved=False, extractor=extract_intent): | |
| corpus = load_corpus(path) | |
| results = run_cases(corpus, include_reserved, extractor) | |
| files = [Path(path).resolve(), Path(__file__).resolve(), ROOT / 'backend/flagship/parsing.py', ROOT / 'backend/flagship/basket.py'] | |
| durations = sorted(x['latency_ms'] for x in results) | |
| return {'schema_version': 1, 'corpus_id': corpus['corpus_id'], 'source': corpus['provenance'], | |
| 'runtime': platform.python_version(), 'captured_at_unix': time.time(), | |
| 'source_sha256': {str(p.relative_to(ROOT)) if p.is_relative_to(ROOT) else p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in files}, | |
| 'rules': summary(results), 'splits': {split: summary([x for x in results if x['split'] == split]) for split in ('development', 'validation', 'reserved_test')}, | |
| 'reserved_test_status': 'RUN_PUBLIC_SYNTHETIC_LABELS' if include_reserved else 'NOT_RUN', | |
| 'llm_assisted': {'status': 'NOT_RUN', 'model_calls': 0}, 'provider_calls': 0, | |
| 'latency_ms': {'p50': statistics.median(durations), 'p95': durations[min(len(durations)-1, int(len(durations)*.95))]}, | |
| 'limitations': ['Public synthetic author labels; no independent human label validation.', | |
| 'Unnecessary clarification is not counted as exact intent completion.', | |
| 'Parsing quality is distinct from solver correctness and transaction authorization.', | |
| 'No model improvement, live commerce success, or general safety claim.'], 'results': results} | |
| SPLIT_PRIORITY = {'reserved_test': 0, 'validation': 1, 'development': 2} | |
| def evaluate_llm(model, path=CORPUS, include_reserved=False, pace_seconds=6.0, previous=None, provider='gemini'): | |
| """Score model-assisted extraction on the same cases and grader; provider errors are reported separately. | |
| Cases run held-out first, so a daily quota is spent where it matters. Rows already scored in | |
| `previous` (an earlier report with the same model) are reused, so runs can resume across days. | |
| """ | |
| corpus = load_corpus(path) | |
| done = {} | |
| earlier = (previous or {}).get('llm_assisted', {}) | |
| if earlier.get('model') == model and earlier.get('provider', 'gemini') == provider: | |
| done = {r['id']: r for r in previous['llm_assisted']['results'] if r['passed'] is not None} | |
| pending = sorted((c for c in corpus['cases'] if c['id'] not in done), key=lambda c: SPLIT_PRIORITY[c['split']]) | |
| fresh = run_cases({**corpus, 'cases': pending}, include_reserved, | |
| lambda text: extract_intent(text, use_llm=True, model=model, provider=provider), pace_seconds) | |
| order = {c['id']: n for n, c in enumerate(corpus['cases'])} | |
| rows = sorted([r for r in done.values() if include_reserved or r['split'] != 'reserved_test'] + fresh, | |
| key=lambda r: order[r['id']]) | |
| usage = Counter() | |
| for row in rows: | |
| for key, value in ((row['actual'] or {}).get('extraction', {}).get('usage') or {}).items(): | |
| usage[key] += value | |
| durations = sorted(x['latency_ms'] for x in rows if not x['provider_unavailable']) | |
| return {'status': 'RUN', 'provider': provider, 'model': model, 'model_calls_scored': len(durations), 'usage_totals': dict(usage), | |
| **summary(rows), 'splits': {split: summary([x for x in rows if x['split'] == split]) | |
| for split in ('development', 'validation', 'reserved_test')}, | |
| 'latency_ms': {'p50': statistics.median(durations), 'p95': durations[min(len(durations) - 1, int(len(durations) * .95))]} if durations else None, | |
| 'results': rows} | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument('--corpus', type=Path, default=CORPUS) | |
| parser.add_argument('--output', type=Path, required=True) | |
| parser.add_argument('--include-reserved', action='store_true') | |
| parser.add_argument('--require-perfect', action='store_true', help='Fail when any scored case fails; default measures baseline honestly') | |
| parser.add_argument('--llm-model', help='Also score Gemini-assisted extraction with this pinned model (needs GEMINI_API_KEY)') | |
| parser.add_argument('--llm-provider', choices=['gemini', 'nvidia'], default='gemini') | |
| parser.add_argument('--llm-pace', type=float, default=6.0, help='Seconds between model calls (free-tier rate limits)') | |
| parser.add_argument('--resume', type=Path, help='Earlier report whose scored model rows are reused') | |
| args = parser.parse_args() | |
| report = evaluate(args.corpus, args.include_reserved) | |
| if args.llm_model: | |
| previous = json.loads(args.resume.read_text()) if args.resume and args.resume.exists() else None | |
| report['llm_assisted'] = evaluate_llm(args.llm_model, args.corpus, args.include_reserved, args.llm_pace, previous, args.llm_provider) | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_text(json.dumps(report, indent=2, allow_nan=False) + '\n') | |
| print(json.dumps({key: report[key] for key in ('rules', 'splits', 'reserved_test_status', 'llm_assisted')}, indent=2)) | |
| if args.require_perfect and report['rules']['passed'] != report['rules']['cases']: | |
| raise SystemExit(1) | |
| if __name__ == '__main__': | |
| main() | |