Download scripts/eval_parsing.py from Gaurav711/HyperFlow: direct link, hf CLI and curl.
- Browser
- Download file 6.96 kB
-
https://huggingface.co/spaces/Gaurav711/HyperFlow/resolve/main/scripts/eval_parsing.py
- Command line
-
hf download hf://spaces/Gaurav711/HyperFlow/scripts/eval_parsing.py
-
curl -L -o eval_parsing.py https://huggingface.co/spaces/Gaurav711/HyperFlow/resolve/main/scripts/eval_parsing.py
6.96 kB
| #!/usr/bin/env python3 | |
| """Measure rules and optionally Gemini on the same bounded corpus slice. | |
| Offline default: python scripts/eval_parsing.py --output /tmp/parsing.json | |
| Paid opt-in: set GEMINI_API_KEY and GEMINI_PARSING_MODEL, then add --llm | |
| --max-cases 20 --timeout-seconds 15 --max-output-tokens 2048. There are no | |
| retries, provider actions, purchases, or model calls without --llm. | |
| """ | |
| import argparse | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import platform | |
| import statistics | |
| import sys | |
| import time | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from backend.flagship.parsing import extract_intent, ExtractionUnavailable | |
| ROOT = Path(__file__).resolve().parents[1] | |
| CORPUS = ROOT / 'evaluation/corpus/commerce-v1.json' | |
| def _run(cases, *, use_llm=False, extractor=None, **options): | |
| results = [] | |
| for case in cases: | |
| started = time.perf_counter() | |
| try: | |
| intent = extract_intent(case['text'], use_llm=use_llm, extractor=extractor, **options) | |
| actual = [(x['target_quantity'], x['unit']) for x in intent['items']] | |
| passed = actual == [tuple(x) for x in case['expected']] and not intent['clarifications'] | |
| record = {'passed': passed, 'status': 'PARSED' if not intent['clarifications'] else 'NEEDS_CLARIFICATION', | |
| 'actual': actual, 'constraint_violations': 0, | |
| 'token_usage': intent['extraction']['usage']} | |
| except ExtractionUnavailable as exc: | |
| record = {'passed': False, 'status': 'UNAVAILABLE', 'error': str(exc), | |
| 'constraint_violations': None, 'token_usage': None} | |
| except ValueError as exc: | |
| record = {'passed': False, 'status': 'REJECTED', 'error': str(exc), | |
| 'constraint_violations': 1, 'token_usage': None} | |
| except Exception: | |
| record = {'passed': False, 'status': 'FAILED', 'error': 'Extractor failed; no intent accepted', | |
| 'constraint_violations': None, 'token_usage': None} | |
| results.append({'id': case['id'], 'family': case['family'], **record, | |
| 'latency_ms': (time.perf_counter() - started) * 1000, | |
| 'model_calls': int(use_llm), 'provider_action_calls': 0}) | |
| latencies = sorted(case['latency_ms'] for case in results) | |
| return {'status': 'RUN', 'cases': len(results), 'passed': sum(case['passed'] for case in results), | |
| 'completion_rate': sum(case['passed'] for case in results) / len(results), | |
| 'constraint_violations': sum(case['constraint_violations'] or 0 for case in results), | |
| 'model_calls': sum(case['model_calls'] for case in results), 'provider_action_calls': 0, | |
| 'p50_ms': statistics.median(latencies), | |
| 'p95_ms': latencies[min(len(latencies) - 1, int(len(latencies) * .95))], 'results': results} | |
| def evaluate(*, corpus_path=CORPUS, use_llm=False, max_cases=20, extractor=None, | |
| model=None, timeout_seconds=15, max_output_tokens=2048): | |
| if type(max_cases) is not int or not 1 <= max_cases <= 20: | |
| raise ValueError('max_cases must be between 1 and 20 per run') | |
| if type(use_llm) is not bool: | |
| raise ValueError('use_llm must be a boolean') | |
| # Validate bounds before starting the measured corpus, without any model call. | |
| extract_intent('1 l milk', timeout_seconds=timeout_seconds, max_output_tokens=max_output_tokens) | |
| if use_llm and extractor is None and not (os.environ.get('GEMINI_API_KEY') and | |
| (model or os.environ.get('GEMINI_PARSING_MODEL') or os.environ.get('GEMINI_MODEL'))): | |
| raise ValueError('Explicit --llm requires GEMINI_API_KEY and GEMINI_PARSING_MODEL or --model') | |
| content = Path(corpus_path).read_bytes() | |
| corpus = json.loads(content) | |
| cases = [case for case in corpus['cases'] if case['kind'] == 'parse'][:max_cases] | |
| if not cases: | |
| raise ValueError('Corpus has no parse cases') | |
| options = {'model': model, 'timeout_seconds': timeout_seconds, 'max_output_tokens': max_output_tokens} | |
| rules = _run(cases, **options) | |
| llm = _run(cases, use_llm=True, extractor=extractor, **options) if use_llm else { | |
| 'status': 'NOT_RUN', 'reason': 'Model calls require explicit --llm opt-in', 'model_calls': 0} | |
| return {'schema_version': 1, 'corpus_version': corpus['version'], | |
| 'corpus_sha256': hashlib.sha256(content).hexdigest(), 'case_ids': [case['id'] for case in cases], | |
| 'source': 'controlled_versioned_corpus', 'runtime': platform.python_version(), | |
| 'source_sha256': {str(path.relative_to(ROOT)): hashlib.sha256(path.read_bytes()).hexdigest() | |
| for path in [ROOT / 'backend/flagship/parsing.py', ROOT / 'backend/flagship/basket.py', Path(__file__)]}, | |
| 'bounds': {'max_model_calls': max_cases if use_llm else 0, | |
| 'timeout_seconds_per_call': timeout_seconds, 'max_output_tokens_per_call': max_output_tokens, | |
| 'retries': 0}, | |
| 'rules': rules, 'llm_assisted': llm, | |
| 'limitations': ['This corpus labels quantity/unit extraction; it does not establish brand or budget accuracy.', | |
| 'Model latency and call counts include failed attempts; missing token usage is not zero usage.', | |
| 'Controlled parsing cases are not live commerce completion or safety certification.']} | |
| if __name__ == '__main__': | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument('--output', type=Path, required=True) | |
| parser.add_argument('--corpus', type=Path, default=CORPUS) | |
| parser.add_argument('--llm', action='store_true', help='Explicitly opt in to potentially paid Gemini extraction') | |
| parser.add_argument('--max-cases', type=int, default=20) | |
| parser.add_argument('--model', help='Explicit Gemini model; otherwise GEMINI_PARSING_MODEL/GEMINI_MODEL') | |
| parser.add_argument('--timeout-seconds', type=float, default=15) | |
| parser.add_argument('--max-output-tokens', type=int, default=2048) | |
| args = parser.parse_args() | |
| try: | |
| report = evaluate(corpus_path=args.corpus, use_llm=args.llm, max_cases=args.max_cases, | |
| model=args.model, timeout_seconds=args.timeout_seconds, max_output_tokens=args.max_output_tokens) | |
| except ValueError as exc: | |
| parser.error(str(exc)) | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_text(json.dumps(report, indent=2, allow_nan=False) + '\n') | |
| print(json.dumps({key: {'status': report[key]['status'], 'passed': report[key].get('passed'), | |
| 'cases': report[key].get('cases')} for key in ('rules', 'llm_assisted')}, indent=2)) | |
| raise SystemExit(0 if report['rules']['passed'] == report['rules']['cases'] and | |
| (not args.llm or report['llm_assisted']['passed'] == report['llm_assisted']['cases']) else 1) | |