src/chemgraph/eval/cli.py

"""CLI entry point for ChemGraph evaluation benchmarks.

Usage::

    # Quick local evaluation using a profile
    chemgraph eval --profile quick --models gpt-4o-mini --judge-model gpt-4o

    # Standard evaluation with LLM judge
    chemgraph eval --profile standard --models gpt-4o-mini gemini-2.5-flash

    # Minimal invocation (uses bundled default dataset)
    chemgraph-eval --models gpt-4o-mini --judge-model gpt-4o

    # Explicit dataset override
    chemgraph-eval \\
        --models gpt-4o-mini gemini-2.5-flash \\
        --dataset path/to/custom_ground_truth.json \\
        --judge-model gpt-4o \\
        --workflows single_agent \\
        --output-dir eval_results

    # Profile + override
    chemgraph eval --profile quick --models gpt-4o --max-queries 3
"""

import argparse
import asyncio
import sys
from typing import Optional

from chemgraph.eval.config import BenchmarkConfig
from chemgraph.eval.runner import ModelBenchmarkRunner


def add_eval_args(parser: argparse.ArgumentParser) -> None:
    """Add evaluation-specific arguments to an argument parser.

    This function is used by both the standalone ``chemgraph-eval``
    entry point and the ``chemgraph eval`` subcommand so that the
    argument interface is consistent.

    Parameters
    ----------
    parser : argparse.ArgumentParser
        Parser or subparser to receive evaluation arguments.
    """
    parser.add_argument(
        "--models",
        nargs="+",
        required=True,
        help="LLM model names to evaluate.",
    )
    parser.add_argument(
        "--judge-model",
        type=str,
        default=None,
        help=(
            "LLM model name for the judge. Required when "
            "--judge-type is 'llm' or 'both'."
        ),
    )
    parser.add_argument(
        "--profile",
        type=str,
        default=None,
        help=(
            "Evaluation profile name from config.toml [eval.profiles.*] "
            "(e.g. 'quick', 'standard'). Requires --config. "
            "CLI arguments override profile values."
        ),
    )
    parser.add_argument(
        "--dataset",
        type=str,
        default=None,
        help=(
            "Path to ground-truth JSON file. "
            "Defaults to the bundled dataset shipped with the package."
        ),
    )
    parser.add_argument(
        "--workflows",
        nargs="+",
        default=None,
        help="Workflow types to test (default: single_agent).",
    )
    parser.add_argument(
        "--output-dir",
        type=str,
        default="eval_results",
        help="Output directory for results (default: eval_results).",
    )
    parser.add_argument(
        "--report",
        choices=["json", "markdown", "console", "all"],
        default="all",
        help="Report format (default: all).",
    )
    parser.add_argument(
        "--no-structured-output",
        action="store_true",
        help="Disable structured output on the agent.",
    )
    parser.add_argument(
        "--judge-type",
        type=str,
        choices=["llm", "structured", "both"],
        default=None,
        help=(
            "Judge strategy: 'llm' (LLM-as-judge), 'structured' "
            "(deterministic structured-output comparison), or 'both' "
            "(run both judges). Default: llm."
        ),
    )
    parser.add_argument(
        "--recursion-limit",
        type=int,
        default=None,
        help="Max LangGraph recursion steps per query (default: 50).",
    )
    parser.add_argument(
        "--max-queries",
        type=int,
        default=None,
        help="Max number of queries to evaluate (0 = all, default: all).",
    )
    parser.add_argument(
        "--tags",
        nargs="*",
        default=[],
        help="Optional tags for the run metadata.",
    )
    parser.add_argument(
        "--resume",
        action="store_true",
        help=(
            "Resume from per-query checkpoint files, skipping "
            "already-completed (model, workflow, query) combinations."
        ),
    )
    parser.add_argument(
        "--config",
        type=str,
        default=None,
        help=(
            "Path to a TOML configuration file (e.g. config.toml). "
            "Provides model base_url, argo_user, and eval profiles."
        ),
    )


def _resolve_profile(args: argparse.Namespace) -> Optional[str]:
    """Resolve the eval profile name from CLI args and config file.

    If ``--profile`` is explicitly set, use it.  Otherwise, if
    ``--config`` is provided and the config file defines
    ``[eval] default_profile``, use that as the profile name.

    Returns ``None`` if no profile should be used.

    Parameters
    ----------
    args : argparse.Namespace
        Parsed evaluation arguments.

    Returns
    -------
    str or None
        Selected profile name, or ``None`` when no profile applies.
    """
    if args.profile:
        return args.profile

    if args.config:
        import toml
        from pathlib import Path

        p = Path(args.config)
        if p.exists():
            with open(p) as fh:
                raw = toml.load(fh)
            default = raw.get("eval", {}).get("default_profile")
            if default:
                profiles = raw.get("eval", {}).get("profiles", {})
                if default in profiles:
                    return default

    return None


def build_config_from_args(args: argparse.Namespace) -> BenchmarkConfig:
    """Build a ``BenchmarkConfig`` from parsed CLI arguments.

    Handles both profile-based and explicit-argument construction.
    When ``--config`` is provided without ``--profile``, the
    ``[eval] default_profile`` from the config file is used
    automatically if it exists.

    Parameters
    ----------
    args : argparse.Namespace
        Parsed evaluation arguments.

    Returns
    -------
    BenchmarkConfig
        Validated benchmark configuration.
    """
    profile = _resolve_profile(args)

    if profile:
        # Profile mode: requires --config
        config_file = args.config
        if not config_file:
            print(
                "Error: --config is required when using --profile.",
                file=sys.stderr,
            )
            sys.exit(1)

        # Collect CLI overrides (None values will be skipped by from_profile)
        overrides = {
            "output_dir": args.output_dir,
            "tags": args.tags or None,
        }
        if args.dataset is not None:
            overrides["dataset"] = args.dataset
        if args.workflows is not None:
            overrides["workflow_types"] = args.workflows
        if args.judge_model is not None:
            overrides["judge_model"] = args.judge_model
        if args.recursion_limit is not None:
            overrides["recursion_limit"] = args.recursion_limit
        if args.max_queries is not None:
            overrides["max_queries"] = args.max_queries
        if args.no_structured_output:
            overrides["structured_output"] = False
        if args.judge_type is not None:
            overrides["judge_type"] = args.judge_type
        if args.resume:
            overrides["resume"] = True

        config = BenchmarkConfig.from_profile(
            profile_name=profile,
            models=args.models,
            config_file=config_file,
            **overrides,
        )
    else:
        # Explicit mode: dataset defaults to the bundled ground truth
        # when --dataset is not provided.
        kwargs: dict = {
            "models": args.models,
            "workflow_types": args.workflows or ["single_agent"],
            "output_dir": args.output_dir,
            "structured_output": not args.no_structured_output,
            "recursion_limit": args.recursion_limit or 50,
            "tags": args.tags or [],
            "max_queries": args.max_queries or 0,
            "config_file": args.config,
            "judge_type": args.judge_type or "llm",
            "resume": args.resume,
        }
        if args.judge_model is not None:
            kwargs["judge_model"] = args.judge_model
        if args.dataset is not None:
            kwargs["dataset"] = args.dataset

        config = BenchmarkConfig(**kwargs)

    return config


def run_eval(args: argparse.Namespace) -> None:
    """Execute an evaluation benchmark from parsed CLI arguments.

    Parameters
    ----------
    args : argparse.Namespace
        Parsed evaluation arguments.
    """
    config = build_config_from_args(args)
    runner = ModelBenchmarkRunner(config)

    print("ChemGraph Evaluation Benchmark")
    if args.profile:
        print(f"  Profile:      {args.profile}")
    print(f"  Models:       {config.models}")
    print(f"  Workflows:    {config.workflow_types}")
    print(f"  Dataset:      {config.dataset}")
    print(f"  Judge Type:   {config.judge_type}")
    if config.judge_model:
        print(f"  Judge Model:  {config.judge_model}")
    if config.max_queries > 0:
        print(f"  Max Queries:  {config.max_queries}")
    if config.resume:
        print("  Resume:       enabled")
    if config.config_file:
        print(f"  Config:       {config.config_file}")
    print(f"  Output:       {config.output_dir}")
    print()

    asyncio.run(runner.run_all())
    runner.report(format=args.report)


def parse_args(argv=None) -> argparse.Namespace:
    """Parse arguments for the standalone ``chemgraph-eval`` command.

    Parameters
    ----------
    argv : list[str], optional
        Argument list to parse. Uses ``sys.argv`` when omitted.

    Returns
    -------
    argparse.Namespace
        Parsed command-line arguments.
    """
    parser = argparse.ArgumentParser(
        prog="chemgraph-eval",
        description="Run ChemGraph multi-model evaluation benchmarks.",
    )
    add_eval_args(parser)
    return parser.parse_args(argv)


def main(argv=None) -> None:
    """Standalone entry point for ``chemgraph-eval``.

    Parameters
    ----------
    argv : list[str], optional
        Argument list to parse. Uses ``sys.argv`` when omitted.
    """
    args = parse_args(argv)
    run_eval(args)


if __name__ == "__main__":
    main()