scripts/evaluate.py

#!/usr/init/env python3
"""
Evaluation and Benchmarking Script for Gemma Developer Agent
Measures task completion rate, tool accuracy, multi-file refactoring, and test healing.
"""

import sys
import os
from pathlib import Path

# Ensure project root is in sys.path so 'scripts.agent' can be imported when run directly
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from scripts.agent import process_query, run_file_write

# Expanded Benchmark Test Suite
BENCHMARK_TASKS = [
    {
        "id": "task_01_refactor",
        "prompt": "Refactor math_utils.py: rename multiply to multiply_values and add type hints.",
        "setup_files": {"math_utils.py": "def multiply(a, b):\n    return a * b\n"},
        "expected_substring": "multiply_values"
    },
    {
        "id": "task_02_syntax_fix",
        "prompt": "Fix the syntax error in syntax_bug.py.",
        "setup_files": {"syntax_bug.py": "def broken_func(x)\n    return x + 1\n"},
        "expected_substring": ":"
    },
    {
        "id": "task_03_multi_file_dependency",
        "prompt": "Update module_b.py to import and call process_data from module_a.py instead of old_func.",
        "setup_files": {
            "module_a.py": "def process_data(x):\n    return x * 10\n",
            "module_b.py": "from module_a import old_func\n\ndef run():\n    return old_func(5)\n"
        },
        "expected_substring": "process_data"
    },
    {
        "id": "task_04_pytest_healing",
        "prompt": "Run pytest and fix math_calc.py so the unit test passes.",
        "setup_files": {
            "math_calc.py": "def add(a, b):\n    return a - b\n",
            "test_math_calc.py": "from math_calc import add\n\ndef test_add():\n    assert add(2, 3) == 5\n"
        },
        "expected_substring": "return a + b"
    }
]

def run_evaluation():
    print("=== Starting Gemma Agent Advanced Benchmark Evaluation ===")
    passed = 0
    total = len(BENCHMARK_TASKS)
    
    workspace = Path("./benchmark_workspace").resolve()
    workspace.mkdir(exist_ok=True)
    original_cwd = os.getcwd()

    try:
        for task in BENCHMARK_TASKS:
            print(f"\nRunning {task['id']}...")
            
            # Clean workspace before each task
            for existing_file in workspace.glob("*"):
                if existing_file.is_file():
                    existing_file.unlink()

            # Setup test environment inside workspace
            for filename, content in task["setup_files"].items():
                file_path = workspace / filename
                file_path.write_text(content)
                
            # Execute agent query inside the workspace directory
            try:
                os.chdir(workspace)
                response = process_query(task["prompt"])
                os.chdir(original_cwd)
                
                # Verify expected output across workspace files
                success = False
                for file_path in workspace.glob("*.py"):
                    file_content = file_path.read_text()
                    if task["expected_substring"] in file_content:
                        success = True
                        break
                
                if success:
                    print(f"[{task['id']}] PASSED")
                    passed += 1
                else:
                    print(f"[{task['id']}] FAILED (Expected pattern '{task['expected_substring']}' not found)")
                    
            except Exception as e:
                os.chdir(original_cwd)
                print(f"[{task['id']}] ERROR: {e}")

    finally:
        os.chdir(original_cwd)

    print(f"\n=== Evaluation Summary ===")
    print(f"Passed: {passed}/{total} ({(passed/total)*100:.1f}%)")

if __name__ == "__main__":
    run_evaluation()