Spaces:
Running
Running
File size: 4,150 Bytes
9c84f9d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 | """Compare runs side by side."""
import json
import click
from tabulate import tabulate
@click.command("compare")
@click.argument("project")
@click.option("--task", "-t", default=None, help="Filter by task name")
@click.option(
"--runs",
"run_ids",
default=None,
help="Comma-separated run IDs to compare",
)
@click.option("--last", "-n", default=None, type=int, help="Compare last N runs for a task")
@click.pass_context
def compare_command(ctx, project, task, run_ids, last):
"""Compare evaluation scores across runs.
Examples:
solar-eval compare my-project --task ci-v1-all --last 5
solar-eval compare my-project --runs "run1,run2"
"""
cfg = ctx.obj["config"]
if cfg.is_remote:
raise click.ClickException("Compare is local-mode only")
artifacts_dir = cfg.artifacts_dir(project)
if not artifacts_dir.exists():
click.secho("No artifacts found.", fg="yellow")
return
# Collect runs to compare
if run_ids:
# Explicit run IDs
run_dirs = []
for rid in run_ids.split(","):
rid = rid.strip()
d = artifacts_dir / rid
if d.is_dir() and (d / "run.json").exists():
run_dirs.append(d)
else:
click.secho(f" Run not found: {rid}, skipping", fg="yellow")
else:
# Scan all runs, optionally filtered by task
run_dirs = []
for d in sorted(artifacts_dir.iterdir()):
if not d.is_dir() or not (d / "run.json").exists():
continue
if task:
run_meta = json.loads((d / "run.json").read_text())
if run_meta.get("task") != task:
continue
run_dirs.append(d)
if last and last < len(run_dirs):
run_dirs = run_dirs[-last:]
if not run_dirs:
click.secho("No runs found to compare.", fg="yellow")
return
# Load run metadata and evaluation data
runs_info = []
all_categories = set()
for d in run_dirs:
run_meta = json.loads((d / "run.json").read_text())
eval_file = d / "evaluation.json"
if not eval_file.exists():
continue
eval_data = json.loads(eval_file.read_text())
runs_info.append(
{
"run_id": d.name,
"task": run_meta.get("task", "-"),
"model": run_meta.get("model", "-"),
"prompt": run_meta.get("prompt", run_meta.get("prompt_version", "-")),
"pipeline": run_meta.get("pipeline", "-"),
"eval_data": eval_data,
}
)
all_categories.update(eval_data.get("scores", {}).keys())
if not runs_info:
click.secho("No evaluation data available for comparison.", fg="yellow")
return
# Build comparison table
headers = ["Category"] + [f"{r['prompt']}/{r['model']}" for r in runs_info]
rows = []
for cat in sorted(all_categories):
row = [cat]
for r in runs_info:
score = r["eval_data"].get("scores", {}).get(cat)
row.append(f"{score:.4f}" if score is not None else "-")
rows.append(row)
# Separator and overall row
rows.append(["─" * 20] + ["─" * 8] * len(runs_info))
overall_row = ["Overall"]
for r in runs_info:
overall = r["eval_data"].get("overall_score")
overall_row.append(f"{overall:.4f}" if overall is not None else "-")
rows.append(overall_row)
task_label = task or "all tasks"
click.secho(f"\nComparison: {project}/{task_label}", fg="blue", bold=True)
# Show run details
for r in runs_info:
click.echo(f" {r['prompt']} ({r['model']}, {r['pipeline']})")
click.echo()
click.echo(tabulate(rows, headers=headers, tablefmt="simple"))
# Highlight best
best_run = max(runs_info, key=lambda r: r["eval_data"].get("overall_score", 0))
best_score = best_run["eval_data"].get("overall_score", 0)
click.secho(
f"\n Best: {best_run['prompt']}/{best_run['model']} ({best_score:.4f})",
fg="green",
bold=True,
)
|