Spaces:
Sleeping
Sleeping
File size: 3,257 Bytes
9c84f9d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | """Evaluation result commands — local JSONL or remote API."""
import json
import click
from solar_eval.cli.config import config
from solar_eval.cli.commands.projects import _resolve_project_id
from solar_eval.cli.formatters import format_evaluation_detail
@click.group("eval")
def eval_group() -> None:
"""View evaluation results."""
@eval_group.command("show")
@click.argument("run_ref")
@click.pass_context
def show_evaluation(ctx: click.Context, run_ref: str) -> None:
"""Show evaluation summary. Local: run dir name. Remote: 'project/run_id'."""
cfg = ctx.obj["config"]
if cfg.is_remote:
from solar_eval.cli.client import EvalClient
client = EvalClient(cfg.remote_url, cfg.timeout)
parts = run_ref.split("/")
if len(parts) != 2:
raise click.ClickException("Remote: use 'project-name/run-id' format")
project_id = _resolve_project_id(client, parts[0])
evaluation = client.get(f"/api/projects/{project_id}/runs/{parts[1]}/evaluation")
else:
parts = run_ref.split("/")
if len(parts) != 2:
raise click.ClickException("Local mode: use 'project-name/run-name' format")
eval_file = cfg.artifacts_dir(parts[0]) / parts[1] / "evaluation.json"
if not eval_file.exists():
raise click.ClickException(f"Evaluation not found: {eval_file}")
evaluation = json.loads(eval_file.read_text())
click.secho(format_evaluation_detail(evaluation))
@eval_group.command("details")
@click.argument("run_ref")
@click.option("--limit", "-n", default=None, type=int, help="Limit results")
@click.pass_context
def show_eval_details(ctx: click.Context, run_ref: str, limit: int | None) -> None:
"""Show per-sample evaluation details."""
cfg = ctx.obj["config"]
if cfg.is_remote:
from solar_eval.cli.client import EvalClient
client = EvalClient(cfg.remote_url, cfg.timeout)
parts = run_ref.split("/")
if len(parts) != 2:
raise click.ClickException("Remote: use 'project-name/run-id' format")
project_id = _resolve_project_id(client, parts[0])
details = client.get(f"/api/projects/{project_id}/runs/{parts[1]}/evaluation/details")
else:
parts = run_ref.split("/")
if len(parts) != 2:
raise click.ClickException("Local mode: use 'project-name/run-name' format")
details_file = cfg.artifacts_dir(parts[0]) / parts[1] / "eval_details.jsonl"
if not details_file.exists():
raise click.ClickException(f"Details not found: {details_file}")
details = [json.loads(line) for line in details_file.read_text().strip().split("\n") if line]
if not details:
click.secho("No evaluation details found.", fg="yellow")
return
if limit:
details = details[:limit]
for d in details:
click.secho(f"Sample #{d.get('sample_idx', '?')}", fg="blue", bold=True)
if d.get("category_scores"):
for cat, score in d["category_scores"].items():
click.echo(f" {cat}: {score:.2f}")
if d.get("severity"):
click.echo(f" Severity: {d['severity']}")
click.echo()
click.echo(f"Showing {len(details)} sample(s)")
|