Spaces:
Sleeping
Sleeping
File size: 7,276 Bytes
5db30ac | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 | from __future__ import annotations
import argparse
import asyncio
import json
from datetime import datetime
from elbench.config import load_project_config
from elbench.execution import BenchmarkRunner, PreflightError, PreflightRunner, RunOptions, run_campaign_sync
from elbench.execution.campaign import build_default_api_pools, clean_model_artifacts
from elbench.registry import FileRegistry
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="ELBench benchmark runner")
subparsers = parser.add_subparsers(dest="command", required=True)
inspect_parser = subparsers.add_parser("inspect", help="Inspect resolved benchmark files")
inspect_parser.add_argument("--config-root", default="configs")
preflight_parser = subparsers.add_parser("preflight", help="Validate whether a run can produce official scores")
preflight_parser.add_argument("--config-root", default="configs")
preflight_parser.add_argument("--model-id", required=True)
preflight_parser.add_argument("--module", action="append")
preflight_parser.add_argument("--subset", action="append")
preflight_parser.add_argument("--source-file", action="append")
preflight_parser.add_argument("--dimension", action="append")
preflight_parser.add_argument("--max-samples", type=int)
preflight_parser.add_argument("--no-judge", action="store_true")
preflight_parser.add_argument("--allow-mock-judge", action="store_true")
preflight_parser.add_argument("--json", action="store_true")
run_parser = subparsers.add_parser("run", help="Run benchmark")
run_parser.add_argument("--config-root", default="configs")
run_parser.add_argument("--model-id", required=True)
run_parser.add_argument("--run-id", default=None)
run_parser.add_argument("--module", action="append")
run_parser.add_argument("--subset", action="append")
run_parser.add_argument("--source-file", action="append")
run_parser.add_argument("--dimension", action="append")
run_parser.add_argument("--max-samples", type=int)
run_parser.add_argument("--max-concurrency", type=int)
run_parser.add_argument("--no-resume", action="store_true")
run_parser.add_argument("--no-judge", action="store_true")
run_parser.add_argument("--skip-preflight", action="store_true")
run_parser.add_argument("--allow-mock-judge", action="store_true")
run_parser.add_argument("--progress", action="store_true")
campaign_parser = subparsers.add_parser("campaign", help="Run the default two-API benchmark campaign")
campaign_parser.add_argument("--config-root", default="configs")
campaign_parser.add_argument("--run-prefix", default=None)
campaign_parser.add_argument("--max-concurrency", type=int)
campaign_parser.add_argument("--module", action="append")
campaign_parser.add_argument("--progress", action="store_true")
campaign_parser.add_argument("--list-models", action="store_true")
clean_parser = subparsers.add_parser("clean-model-results", help="Remove local result artifacts for models")
clean_parser.add_argument("--config-root", default="configs")
clean_parser.add_argument("--model-id", action="append", required=True)
return parser
def main() -> None:
parser = build_parser()
args = parser.parse_args()
config = load_project_config(args.config_root)
if args.command == "inspect":
registry = FileRegistry(config)
resolved = registry.resolve()
for item in resolved:
print(
f"module={item.entry.module} subset={item.entry.subset} file={item.entry.canonical_name} path={item.path}"
)
return
if args.command == "preflight":
runner = PreflightRunner(config)
report = runner.run(
model_id=args.model_id,
modules=set(args.module) if args.module else None,
subsets=set(args.subset) if args.subset else None,
source_files=set(args.source_file) if args.source_file else None,
dimensions=set(args.dimension) if args.dimension else None,
max_samples=args.max_samples,
judge_enabled=not args.no_judge,
require_real_judges=not args.allow_mock_judge,
)
if args.json:
print(json.dumps(report.to_dict(), ensure_ascii=False, indent=2))
else:
print(f"ok={report.ok}")
print(f"selected_modules={','.join(sorted(report.selection.selected_modules))}")
print(f"total_samples={report.total_samples}")
for module_name, count in sorted(report.by_module.items()):
print(f"module[{module_name}]={count}")
for issue in report.issues:
print(f"{issue.level.upper()} {issue.code}: {issue.message}")
raise SystemExit(0 if report.ok else 2)
if args.command == "campaign":
if args.list_models:
for pool in build_default_api_pools(config):
print(f"{pool.pool_id}: {', '.join(pool.model_ids)}")
return
result = run_campaign_sync(
config_root=args.config_root,
run_prefix=args.run_prefix,
max_concurrency=args.max_concurrency,
modules=set(args.module) if args.module else None,
progress_enabled=args.progress,
)
print(json.dumps(result.to_dict(), ensure_ascii=False, indent=2))
return
if args.command == "clean-model-results":
deleted = clean_model_artifacts(config.app.output_root, args.model_id)
for path in deleted:
print(path)
print(f"deleted={len(deleted)}")
return
run_id = args.run_id or datetime.now().strftime("%Y%m%d-%H%M%S")
if not args.skip_preflight:
runner = PreflightRunner(config)
try:
runner.ensure_ok(
model_id=args.model_id,
modules=set(args.module) if args.module else None,
subsets=set(args.subset) if args.subset else None,
source_files=set(args.source_file) if args.source_file else None,
dimensions=set(args.dimension) if args.dimension else None,
max_samples=args.max_samples,
judge_enabled=not args.no_judge,
require_real_judges=not args.allow_mock_judge,
)
except PreflightError as exc:
raise SystemExit(f"Preflight failed: {exc}") from exc
runner = BenchmarkRunner(config)
result = asyncio.run(
runner.run(
RunOptions(
model_id=args.model_id,
run_id=run_id,
modules=set(args.module) if args.module else None,
subsets=set(args.subset) if args.subset else None,
source_files=set(args.source_file) if args.source_file else None,
dimensions=set(args.dimension) if args.dimension else None,
max_samples=args.max_samples,
max_concurrency=args.max_concurrency,
resume=not args.no_resume,
judge_enabled=not args.no_judge,
progress_enabled=args.progress,
)
)
)
print(result)
if __name__ == "__main__":
main()
|