solar_eval/cli/commands/quickrun.py

"""Quick-run: run + evaluate + compare in one command."""

import click


@click.command("quick-run")
@click.argument("project")
@click.option("--task", "-t", required=True, help="Task name")
@click.option("--prompt", "-p", "prompt_name", required=True, help="Prompt name")
@click.option("--model", "-m", required=True, help="Model name")
@click.option(
    "--pipeline",
    "-P",
    "pipeline_name",
    default=None,
    help="Pipeline template name (overrides project default)",
)
@click.option("--reasoning-effort", "-re", default=None, help="Reasoning effort")
@click.option("--max-workers", "-w", default=5, type=int, help="Max parallel workers")
@click.pass_context
def quick_run(ctx, project, task, prompt_name, model, pipeline_name, reasoning_effort, max_workers):
    """Run evaluation and compare with previous runs for the same task."""
    cfg = ctx.obj["config"]
    if cfg.is_remote:
        raise click.ClickException("Quick-run is local-mode only")

    # Run evaluation
    from solar_eval.cli.commands.runs import _start_local

    click.secho("\nRunning evaluation...", fg="blue")
    _start_local(
        cfg,
        project=project,
        task=task,
        prompt_name=prompt_name,
        model=model,
        pipeline_name=pipeline_name,
        reasoning_effort=reasoning_effort,
        temperature=0.0,
        max_tokens=4000,
        max_workers=max_workers,
        limit=None,
    )

    # Compare with previous runs for same task
    from solar_eval.cli.commands.compare import compare_command

    click.secho("\nComparing with previous runs:", fg="blue", bold=True)
    ctx.invoke(compare_command, project=project, task=task, run_ids=None, last=5)