File size: 7,752 Bytes
22087d0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
"""
Kortix Agent Evaluation Script.

Usage:
    # Run with default dataset (3 complex tests)
    uv run python evals/agent_eval.py
    
    # Run specific dataset with count
    uv run python evals/agent_eval.py --dataset gaia-level1 --count 5
    uv run python evals/agent_eval.py --dataset complex 3
    uv run python evals/agent_eval.py -d math -n 2
    
    # List available datasets
    uv run python evals/agent_eval.py --list
    
    # Run with Braintrust CLI (for production)
    braintrust eval evals/agent_eval.py
    braintrust eval --no-send-logs evals/agent_eval.py

Available datasets:
    - basic: Simple greeting/capability tests
    - coding: Code generation tests
    - tools: Tool usage tests
    - reasoning: Complex reasoning tests
    - math: Math problems
    - complex: Complex tool-using tasks (default)
    - real_world: Real-world scenarios
    - gaia-level1: GAIA benchmark level 1 (easiest)
    - gaia-level2: GAIA benchmark level 2 (medium)  
    - gaia-level3: GAIA benchmark level 3 (hardest)
"""

import os
import sys
import argparse
from pathlib import Path

# Add backend to path for imports
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

# Load config first to get env vars
from core.utils.config import config

from braintrust import Eval

from evals.runner import create_agent_task
from evals.scorers import (
    AnswerCorrectness,
    TaskCompletionScorer,
    ToolUsageScorer,
    ResponseTimeScorer,
)
from evals.datasets import get_dataset, list_available_datasets


# ============================================================================
# CLI ARGUMENT PARSING
# ============================================================================

def parse_args():
    """Parse command line arguments."""
    parser = argparse.ArgumentParser(
        description="Run Kortix Agent evaluations",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
    %(prog)s --dataset gaia-level1 --count 5
    %(prog)s -d complex -n 3
    %(prog)s --list
    %(prog)s gaia-level3 10  # positional: dataset count
        """
    )
    
    parser.add_argument(
        "positional_dataset",
        nargs="?",
        default=None,
        help="Dataset name (positional)"
    )
    
    parser.add_argument(
        "positional_count",
        nargs="?",
        type=int,
        default=None,
        help="Number of tests (positional)"
    )
    
    parser.add_argument(
        "-d", "--dataset",
        type=str,
        default=os.getenv("EVAL_DATASET", "complex"),
        help="Dataset to use (default: complex)"
    )
    
    parser.add_argument(
        "-n", "--count",
        type=int,
        default=int(os.getenv("EVAL_COUNT", "3")),
        help="Number of test cases to run (default: 3)"
    )
    
    parser.add_argument(
        "-m", "--model",
        type=str,
        default=os.getenv("EVAL_MODEL", "kortix/basic"),
        help="Model to use (default: kortix/basic)"
    )
    
    parser.add_argument(
        "--timeout",
        type=float,
        default=float(os.getenv("EVAL_TIMEOUT", "300")),
        help="Timeout per test in seconds (default: 300 = 5 minutes)"
    )
    
    parser.add_argument(
        "--max-iterations",
        type=int,
        default=int(os.getenv("EVAL_MAX_ITERATIONS", "25")),
        help="Max agent iterations (default: 25)"
    )
    
    parser.add_argument(
        "-l", "--list",
        action="store_true",
        help="List available datasets and exit"
    )
    
    parser.add_argument(
        "--experiment",
        type=str,
        default=os.getenv("EVAL_EXPERIMENT_NAME"),
        help="Custom experiment name"
    )
    
    parser.add_argument(
        "-p", "--project-id",
        type=str,
        default=os.getenv("EVAL_PROJECT_ID"),
        help="Project ID with sandbox (enables web_search and other sandbox tools). "
             "Create a project in the web UI first, then use its ID here."
    )
    
    args = parser.parse_args()
    
    # Handle positional arguments
    if args.positional_dataset:
        args.dataset = args.positional_dataset
    if args.positional_count:
        args.count = args.positional_count
    
    return args


# ============================================================================
# DATASET LOADING
# ============================================================================

# Parse args (global so get_eval_data can access)
ARGS = parse_args()


def get_eval_data():
    """
    Return evaluation dataset based on CLI arguments.
    """
    dataset_name = ARGS.dataset
    count = ARGS.count
    
    print(f"πŸ“‚ Loading dataset: {dataset_name} (count={count})")
    
    try:
        dataset = get_dataset(dataset_name, count=count)
        print(f"βœ… Loaded {len(dataset)} test cases")
        
        # Print first test preview
        if dataset:
            first = dataset[0]
            preview = first.get("input", str(first))[:80]
            print(f"πŸ“ First test: {preview}...")
        
        return dataset
        
    except ValueError as e:
        print(f"❌ Error: {e}")
        print(f"\nπŸ“‹ Available datasets:")
        for name in list_available_datasets():
            print(f"   - {name}")
        sys.exit(1)


# ============================================================================
# MAIN
# ============================================================================

if __name__ == "__main__" or "braintrust" in sys.argv[0].lower():
    
    # Handle --list flag
    if ARGS.list:
        print("\nπŸ“‹ Available datasets:\n")
        for name in list_available_datasets():
            print(f"   - {name}")
        print("\nπŸ’‘ Usage: python evals/agent_eval.py --dataset <name> --count <n>")
        print("   Example: python evals/agent_eval.py --dataset gaia-level1 --count 5")
        sys.exit(0)
    
    print(f"""
╔══════════════════════════════════════════════════════════════╗
β•‘                  Kortix Agent Evaluation                      β•‘
╠══════════════════════════════════════════════════════════════╣
β•‘  Dataset:     {ARGS.dataset:<45} β•‘
β•‘  Count:       {ARGS.count:<45} β•‘
β•‘  Model:       {ARGS.model:<45} β•‘
β•‘  Timeout:     {ARGS.timeout}s{' ' * (44 - len(str(ARGS.timeout)))} β•‘
β•‘  Project:     {(ARGS.project_id or 'None (no sandbox)'):<45} β•‘
β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•
""")
    
    if ARGS.project_id:
        print("πŸ”§ Sandbox tools ENABLED (web_search, file tools, browser, etc.)")
    else:
        print("⚠️  No project ID - agent will run WITHOUT sandbox tools (web_search, etc.)")
        print("   To enable: --project-id <your-project-id> or set EVAL_PROJECT_ID env var")
    print()
    
    # Run evaluation
    Eval(
        "Kortix Agent",  # Project name in Braintrust
        data=get_eval_data,
        task=create_agent_task(
            model_name=ARGS.model,
            max_iterations=ARGS.max_iterations,
            timeout_seconds=ARGS.timeout,
            project_id=ARGS.project_id,
        ),
        scores=[
            AnswerCorrectness,
            TaskCompletionScorer,
            ToolUsageScorer,
            ResponseTimeScorer,
        ],
        experiment_name=ARGS.experiment,
        max_concurrency=1,  # CRITICAL: Run tests SEQUENTIALLY
    )