File size: 2,551 Bytes
29a2ba2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | """
band_edge_detector/cli.py
=========================
Command-line interface.
Usage:
band-edge-detector --input scores.csv --output report/
band-edge-detector --input scores.csv --method pca --n-bins 7
band-edge-detector --input long_scores.csv --format long
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
from .data import BenchmarkMatrix
from .report import analyze
def main(argv=None):
parser = argparse.ArgumentParser(
prog="band-edge-detector",
description="Detect band-edge compression in benchmark scores.",
)
parser.add_argument("--input", required=True,
help="CSV with models as rows, benchmarks as columns.")
parser.add_argument("--format", choices=["wide", "long"], default="wide",
help="'wide' = models × benchmarks; "
"'long' = (model, benchmark, score).")
parser.add_argument("--output", default="band_edge_report",
help="Directory for figures.")
parser.add_argument("--method", choices=["mean_z", "pca", "irt"],
default="mean_z",
help="Capability proxy method.")
parser.add_argument("--n-bins", type=int, default=5,
help="Quantile bins.")
parser.add_argument("--threshold", type=float, default=0.5,
help="Compression threshold.")
parser.add_argument("--min-coverage", type=float, default=0.5,
help="Drop benchmarks with less coverage than this.")
parser.add_argument("--min-benchmarks", type=int, default=5,
help="Drop models with fewer than this many scores.")
args = parser.parse_args(argv)
if args.format == "wide":
matrix = BenchmarkMatrix.from_csv(args.input)
else:
matrix = BenchmarkMatrix.from_long_csv(args.input)
matrix = matrix.filter_benchmarks(min_coverage=args.min_coverage)
matrix = matrix.filter_models(min_benchmarks=args.min_benchmarks)
print(f"Loaded {matrix.n_models} models × {matrix.n_benchmarks} benchmarks")
report = analyze(
matrix,
method=args.method,
n_bins=args.n_bins,
compression_threshold=args.threshold,
output_dir=args.output,
)
print(report.summary())
print(f"\nFigures written to: {args.output}/")
return 0
if __name__ == "__main__":
sys.exit(main()) |