solar_eval/cli/commands/compare.py

"""Compare runs side by side."""

import json

import click
from tabulate import tabulate


@click.command("compare")
@click.argument("project")
@click.option("--task", "-t", default=None, help="Filter by task name")
@click.option(
    "--runs",
    "run_ids",
    default=None,
    help="Comma-separated run IDs to compare",
)
@click.option("--last", "-n", default=None, type=int, help="Compare last N runs for a task")
@click.pass_context
def compare_command(ctx, project, task, run_ids, last):
    """Compare evaluation scores across runs.

    Examples:
        solar-eval compare my-project --task ci-v1-all --last 5
        solar-eval compare my-project --runs "run1,run2"
    """
    cfg = ctx.obj["config"]
    if cfg.is_remote:
        raise click.ClickException("Compare is local-mode only")

    artifacts_dir = cfg.artifacts_dir(project)
    if not artifacts_dir.exists():
        click.secho("No artifacts found.", fg="yellow")
        return

    # Collect runs to compare
    if run_ids:
        # Explicit run IDs
        run_dirs = []
        for rid in run_ids.split(","):
            rid = rid.strip()
            d = artifacts_dir / rid
            if d.is_dir() and (d / "run.json").exists():
                run_dirs.append(d)
            else:
                click.secho(f"  Run not found: {rid}, skipping", fg="yellow")
    else:
        # Scan all runs, optionally filtered by task
        run_dirs = []
        for d in sorted(artifacts_dir.iterdir()):
            if not d.is_dir() or not (d / "run.json").exists():
                continue
            if task:
                run_meta = json.loads((d / "run.json").read_text())
                if run_meta.get("task") != task:
                    continue
            run_dirs.append(d)

    if last and last < len(run_dirs):
        run_dirs = run_dirs[-last:]

    if not run_dirs:
        click.secho("No runs found to compare.", fg="yellow")
        return

    # Load run metadata and evaluation data
    runs_info = []
    all_categories = set()
    for d in run_dirs:
        run_meta = json.loads((d / "run.json").read_text())
        eval_file = d / "evaluation.json"
        if not eval_file.exists():
            continue
        eval_data = json.loads(eval_file.read_text())
        runs_info.append(
            {
                "run_id": d.name,
                "task": run_meta.get("task", "-"),
                "model": run_meta.get("model", "-"),
                "prompt": run_meta.get("prompt", run_meta.get("prompt_version", "-")),
                "pipeline": run_meta.get("pipeline", "-"),
                "eval_data": eval_data,
            }
        )
        all_categories.update(eval_data.get("scores", {}).keys())

    if not runs_info:
        click.secho("No evaluation data available for comparison.", fg="yellow")
        return

    # Build comparison table
    headers = ["Category"] + [f"{r['prompt']}/{r['model']}" for r in runs_info]

    rows = []
    for cat in sorted(all_categories):
        row = [cat]
        for r in runs_info:
            score = r["eval_data"].get("scores", {}).get(cat)
            row.append(f"{score:.4f}" if score is not None else "-")
        rows.append(row)

    # Separator and overall row
    rows.append(["─" * 20] + ["─" * 8] * len(runs_info))
    overall_row = ["Overall"]
    for r in runs_info:
        overall = r["eval_data"].get("overall_score")
        overall_row.append(f"{overall:.4f}" if overall is not None else "-")
    rows.append(overall_row)

    task_label = task or "all tasks"
    click.secho(f"\nComparison: {project}/{task_label}", fg="blue", bold=True)

    # Show run details
    for r in runs_info:
        click.echo(f"  {r['prompt']} ({r['model']}, {r['pipeline']})")
    click.echo()

    click.echo(tabulate(rows, headers=headers, tablefmt="simple"))

    # Highlight best
    best_run = max(runs_info, key=lambda r: r["eval_data"].get("overall_score", 0))
    best_score = best_run["eval_data"].get("overall_score", 0)
    click.secho(
        f"\n  Best: {best_run['prompt']}/{best_run['model']} ({best_score:.4f})",
        fg="green",
        bold=True,
    )