Spaces:
Running on Zero
Running on Zero
File size: 1,923 Bytes
de8702b 536758e de8702b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | """Run every preset (or a chosen subset) through runner.py, one after another.
Presets whose API key is missing are skipped. Extra arguments after ``--`` are
forwarded to runner.py, e.g. ``python scripts/run_all.py kev-4b -- --limit 5``.
"""
import argparse
import os
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from benchmark.adapters import ADAPTERS # noqa: E402
from benchmark.models import PRESETS, load_dotenv # noqa: E402
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("presets", nargs="*", help=f"Default: all ({', '.join(PRESETS)})")
parser.add_argument("--skip-local", action="store_true", help="Skip local checkpoints (laya, julia, d1, gliner).")
argv = sys.argv[1:]
split = argv.index("--") if "--" in argv else len(argv)
args, forwarded = parser.parse_args(argv[:split]), argv[split + 1:]
unknown = set(args.presets) - PRESETS.keys()
if unknown:
raise SystemExit(f"Unknown preset(s): {sorted(unknown)}")
load_dotenv()
summary = []
for name in args.presets or PRESETS:
provider, _model = PRESETS[name]
key_env = getattr(ADAPTERS[provider], "api_key_env", None)
if key_env and not os.getenv(key_env):
summary.append((name, f"skipped ({key_env} missing)"))
continue
if not key_env and args.skip_local:
summary.append((name, "skipped (local)"))
continue
print(f"\n######## {name} ########", flush=True)
cmd = [sys.executable, "runner.py", "--preset", name, *forwarded]
code = subprocess.call(cmd, cwd=ROOT)
summary.append((name, "ok" if code == 0 else f"failed (exit {code})"))
print("\n" + "=" * 60)
for name, status in summary:
print(f"{name:22} {status}")
if __name__ == "__main__":
main()
|