Spaces:
Sleeping
Sleeping
| from __future__ import annotations | |
| import argparse | |
| import asyncio | |
| import json | |
| from datetime import datetime | |
| from elbench.config import load_project_config | |
| from elbench.execution import BenchmarkRunner, PreflightError, PreflightRunner, RunOptions, run_campaign_sync | |
| from elbench.execution.campaign import build_default_api_pools, clean_model_artifacts | |
| from elbench.registry import FileRegistry | |
| def build_parser() -> argparse.ArgumentParser: | |
| parser = argparse.ArgumentParser(description="ELBench benchmark runner") | |
| subparsers = parser.add_subparsers(dest="command", required=True) | |
| inspect_parser = subparsers.add_parser("inspect", help="Inspect resolved benchmark files") | |
| inspect_parser.add_argument("--config-root", default="configs") | |
| preflight_parser = subparsers.add_parser("preflight", help="Validate whether a run can produce official scores") | |
| preflight_parser.add_argument("--config-root", default="configs") | |
| preflight_parser.add_argument("--model-id", required=True) | |
| preflight_parser.add_argument("--module", action="append") | |
| preflight_parser.add_argument("--subset", action="append") | |
| preflight_parser.add_argument("--source-file", action="append") | |
| preflight_parser.add_argument("--dimension", action="append") | |
| preflight_parser.add_argument("--max-samples", type=int) | |
| preflight_parser.add_argument("--no-judge", action="store_true") | |
| preflight_parser.add_argument("--allow-mock-judge", action="store_true") | |
| preflight_parser.add_argument("--json", action="store_true") | |
| run_parser = subparsers.add_parser("run", help="Run benchmark") | |
| run_parser.add_argument("--config-root", default="configs") | |
| run_parser.add_argument("--model-id", required=True) | |
| run_parser.add_argument("--run-id", default=None) | |
| run_parser.add_argument("--module", action="append") | |
| run_parser.add_argument("--subset", action="append") | |
| run_parser.add_argument("--source-file", action="append") | |
| run_parser.add_argument("--dimension", action="append") | |
| run_parser.add_argument("--max-samples", type=int) | |
| run_parser.add_argument("--max-concurrency", type=int) | |
| run_parser.add_argument("--no-resume", action="store_true") | |
| run_parser.add_argument("--no-judge", action="store_true") | |
| run_parser.add_argument("--skip-preflight", action="store_true") | |
| run_parser.add_argument("--allow-mock-judge", action="store_true") | |
| run_parser.add_argument("--progress", action="store_true") | |
| campaign_parser = subparsers.add_parser("campaign", help="Run the default two-API benchmark campaign") | |
| campaign_parser.add_argument("--config-root", default="configs") | |
| campaign_parser.add_argument("--run-prefix", default=None) | |
| campaign_parser.add_argument("--max-concurrency", type=int) | |
| campaign_parser.add_argument("--module", action="append") | |
| campaign_parser.add_argument("--progress", action="store_true") | |
| campaign_parser.add_argument("--list-models", action="store_true") | |
| clean_parser = subparsers.add_parser("clean-model-results", help="Remove local result artifacts for models") | |
| clean_parser.add_argument("--config-root", default="configs") | |
| clean_parser.add_argument("--model-id", action="append", required=True) | |
| return parser | |
| def main() -> None: | |
| parser = build_parser() | |
| args = parser.parse_args() | |
| config = load_project_config(args.config_root) | |
| if args.command == "inspect": | |
| registry = FileRegistry(config) | |
| resolved = registry.resolve() | |
| for item in resolved: | |
| print( | |
| f"module={item.entry.module} subset={item.entry.subset} file={item.entry.canonical_name} path={item.path}" | |
| ) | |
| return | |
| if args.command == "preflight": | |
| runner = PreflightRunner(config) | |
| report = runner.run( | |
| model_id=args.model_id, | |
| modules=set(args.module) if args.module else None, | |
| subsets=set(args.subset) if args.subset else None, | |
| source_files=set(args.source_file) if args.source_file else None, | |
| dimensions=set(args.dimension) if args.dimension else None, | |
| max_samples=args.max_samples, | |
| judge_enabled=not args.no_judge, | |
| require_real_judges=not args.allow_mock_judge, | |
| ) | |
| if args.json: | |
| print(json.dumps(report.to_dict(), ensure_ascii=False, indent=2)) | |
| else: | |
| print(f"ok={report.ok}") | |
| print(f"selected_modules={','.join(sorted(report.selection.selected_modules))}") | |
| print(f"total_samples={report.total_samples}") | |
| for module_name, count in sorted(report.by_module.items()): | |
| print(f"module[{module_name}]={count}") | |
| for issue in report.issues: | |
| print(f"{issue.level.upper()} {issue.code}: {issue.message}") | |
| raise SystemExit(0 if report.ok else 2) | |
| if args.command == "campaign": | |
| if args.list_models: | |
| for pool in build_default_api_pools(config): | |
| print(f"{pool.pool_id}: {', '.join(pool.model_ids)}") | |
| return | |
| result = run_campaign_sync( | |
| config_root=args.config_root, | |
| run_prefix=args.run_prefix, | |
| max_concurrency=args.max_concurrency, | |
| modules=set(args.module) if args.module else None, | |
| progress_enabled=args.progress, | |
| ) | |
| print(json.dumps(result.to_dict(), ensure_ascii=False, indent=2)) | |
| return | |
| if args.command == "clean-model-results": | |
| deleted = clean_model_artifacts(config.app.output_root, args.model_id) | |
| for path in deleted: | |
| print(path) | |
| print(f"deleted={len(deleted)}") | |
| return | |
| run_id = args.run_id or datetime.now().strftime("%Y%m%d-%H%M%S") | |
| if not args.skip_preflight: | |
| runner = PreflightRunner(config) | |
| try: | |
| runner.ensure_ok( | |
| model_id=args.model_id, | |
| modules=set(args.module) if args.module else None, | |
| subsets=set(args.subset) if args.subset else None, | |
| source_files=set(args.source_file) if args.source_file else None, | |
| dimensions=set(args.dimension) if args.dimension else None, | |
| max_samples=args.max_samples, | |
| judge_enabled=not args.no_judge, | |
| require_real_judges=not args.allow_mock_judge, | |
| ) | |
| except PreflightError as exc: | |
| raise SystemExit(f"Preflight failed: {exc}") from exc | |
| runner = BenchmarkRunner(config) | |
| result = asyncio.run( | |
| runner.run( | |
| RunOptions( | |
| model_id=args.model_id, | |
| run_id=run_id, | |
| modules=set(args.module) if args.module else None, | |
| subsets=set(args.subset) if args.subset else None, | |
| source_files=set(args.source_file) if args.source_file else None, | |
| dimensions=set(args.dimension) if args.dimension else None, | |
| max_samples=args.max_samples, | |
| max_concurrency=args.max_concurrency, | |
| resume=not args.no_resume, | |
| judge_enabled=not args.no_judge, | |
| progress_enabled=args.progress, | |
| ) | |
| ) | |
| ) | |
| print(result) | |
| if __name__ == "__main__": | |
| main() | |