File size: 7,752 Bytes
22087d0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 | """
Kortix Agent Evaluation Script.
Usage:
# Run with default dataset (3 complex tests)
uv run python evals/agent_eval.py
# Run specific dataset with count
uv run python evals/agent_eval.py --dataset gaia-level1 --count 5
uv run python evals/agent_eval.py --dataset complex 3
uv run python evals/agent_eval.py -d math -n 2
# List available datasets
uv run python evals/agent_eval.py --list
# Run with Braintrust CLI (for production)
braintrust eval evals/agent_eval.py
braintrust eval --no-send-logs evals/agent_eval.py
Available datasets:
- basic: Simple greeting/capability tests
- coding: Code generation tests
- tools: Tool usage tests
- reasoning: Complex reasoning tests
- math: Math problems
- complex: Complex tool-using tasks (default)
- real_world: Real-world scenarios
- gaia-level1: GAIA benchmark level 1 (easiest)
- gaia-level2: GAIA benchmark level 2 (medium)
- gaia-level3: GAIA benchmark level 3 (hardest)
"""
import os
import sys
import argparse
from pathlib import Path
# Add backend to path for imports
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# Load config first to get env vars
from core.utils.config import config
from braintrust import Eval
from evals.runner import create_agent_task
from evals.scorers import (
AnswerCorrectness,
TaskCompletionScorer,
ToolUsageScorer,
ResponseTimeScorer,
)
from evals.datasets import get_dataset, list_available_datasets
# ============================================================================
# CLI ARGUMENT PARSING
# ============================================================================
def parse_args():
"""Parse command line arguments."""
parser = argparse.ArgumentParser(
description="Run Kortix Agent evaluations",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
%(prog)s --dataset gaia-level1 --count 5
%(prog)s -d complex -n 3
%(prog)s --list
%(prog)s gaia-level3 10 # positional: dataset count
"""
)
parser.add_argument(
"positional_dataset",
nargs="?",
default=None,
help="Dataset name (positional)"
)
parser.add_argument(
"positional_count",
nargs="?",
type=int,
default=None,
help="Number of tests (positional)"
)
parser.add_argument(
"-d", "--dataset",
type=str,
default=os.getenv("EVAL_DATASET", "complex"),
help="Dataset to use (default: complex)"
)
parser.add_argument(
"-n", "--count",
type=int,
default=int(os.getenv("EVAL_COUNT", "3")),
help="Number of test cases to run (default: 3)"
)
parser.add_argument(
"-m", "--model",
type=str,
default=os.getenv("EVAL_MODEL", "kortix/basic"),
help="Model to use (default: kortix/basic)"
)
parser.add_argument(
"--timeout",
type=float,
default=float(os.getenv("EVAL_TIMEOUT", "300")),
help="Timeout per test in seconds (default: 300 = 5 minutes)"
)
parser.add_argument(
"--max-iterations",
type=int,
default=int(os.getenv("EVAL_MAX_ITERATIONS", "25")),
help="Max agent iterations (default: 25)"
)
parser.add_argument(
"-l", "--list",
action="store_true",
help="List available datasets and exit"
)
parser.add_argument(
"--experiment",
type=str,
default=os.getenv("EVAL_EXPERIMENT_NAME"),
help="Custom experiment name"
)
parser.add_argument(
"-p", "--project-id",
type=str,
default=os.getenv("EVAL_PROJECT_ID"),
help="Project ID with sandbox (enables web_search and other sandbox tools). "
"Create a project in the web UI first, then use its ID here."
)
args = parser.parse_args()
# Handle positional arguments
if args.positional_dataset:
args.dataset = args.positional_dataset
if args.positional_count:
args.count = args.positional_count
return args
# ============================================================================
# DATASET LOADING
# ============================================================================
# Parse args (global so get_eval_data can access)
ARGS = parse_args()
def get_eval_data():
"""
Return evaluation dataset based on CLI arguments.
"""
dataset_name = ARGS.dataset
count = ARGS.count
print(f"π Loading dataset: {dataset_name} (count={count})")
try:
dataset = get_dataset(dataset_name, count=count)
print(f"β
Loaded {len(dataset)} test cases")
# Print first test preview
if dataset:
first = dataset[0]
preview = first.get("input", str(first))[:80]
print(f"π First test: {preview}...")
return dataset
except ValueError as e:
print(f"β Error: {e}")
print(f"\nπ Available datasets:")
for name in list_available_datasets():
print(f" - {name}")
sys.exit(1)
# ============================================================================
# MAIN
# ============================================================================
if __name__ == "__main__" or "braintrust" in sys.argv[0].lower():
# Handle --list flag
if ARGS.list:
print("\nπ Available datasets:\n")
for name in list_available_datasets():
print(f" - {name}")
print("\nπ‘ Usage: python evals/agent_eval.py --dataset <name> --count <n>")
print(" Example: python evals/agent_eval.py --dataset gaia-level1 --count 5")
sys.exit(0)
print(f"""
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
β Kortix Agent Evaluation β
β βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ£
β Dataset: {ARGS.dataset:<45} β
β Count: {ARGS.count:<45} β
β Model: {ARGS.model:<45} β
β Timeout: {ARGS.timeout}s{' ' * (44 - len(str(ARGS.timeout)))} β
β Project: {(ARGS.project_id or 'None (no sandbox)'):<45} β
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
""")
if ARGS.project_id:
print("π§ Sandbox tools ENABLED (web_search, file tools, browser, etc.)")
else:
print("β οΈ No project ID - agent will run WITHOUT sandbox tools (web_search, etc.)")
print(" To enable: --project-id <your-project-id> or set EVAL_PROJECT_ID env var")
print()
# Run evaluation
Eval(
"Kortix Agent", # Project name in Braintrust
data=get_eval_data,
task=create_agent_task(
model_name=ARGS.model,
max_iterations=ARGS.max_iterations,
timeout_seconds=ARGS.timeout,
project_id=ARGS.project_id,
),
scores=[
AnswerCorrectness,
TaskCompletionScorer,
ToolUsageScorer,
ResponseTimeScorer,
],
experiment_name=ARGS.experiment,
max_concurrency=1, # CRITICAL: Run tests SEQUENTIALLY
)
|