File size: 3,801 Bytes
c85c557 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 | #!/usr/init/env python3
"""
Evaluation and Benchmarking Script for Gemma Developer Agent
Measures task completion rate, tool accuracy, multi-file refactoring, and test healing.
"""
import sys
import os
from pathlib import Path
# Ensure project root is in sys.path so 'scripts.agent' can be imported when run directly
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from scripts.agent import process_query, run_file_write
# Expanded Benchmark Test Suite
BENCHMARK_TASKS = [
{
"id": "task_01_refactor",
"prompt": "Refactor math_utils.py: rename multiply to multiply_values and add type hints.",
"setup_files": {"math_utils.py": "def multiply(a, b):\n return a * b\n"},
"expected_substring": "multiply_values"
},
{
"id": "task_02_syntax_fix",
"prompt": "Fix the syntax error in syntax_bug.py.",
"setup_files": {"syntax_bug.py": "def broken_func(x)\n return x + 1\n"},
"expected_substring": ":"
},
{
"id": "task_03_multi_file_dependency",
"prompt": "Update module_b.py to import and call process_data from module_a.py instead of old_func.",
"setup_files": {
"module_a.py": "def process_data(x):\n return x * 10\n",
"module_b.py": "from module_a import old_func\n\ndef run():\n return old_func(5)\n"
},
"expected_substring": "process_data"
},
{
"id": "task_04_pytest_healing",
"prompt": "Run pytest and fix math_calc.py so the unit test passes.",
"setup_files": {
"math_calc.py": "def add(a, b):\n return a - b\n",
"test_math_calc.py": "from math_calc import add\n\ndef test_add():\n assert add(2, 3) == 5\n"
},
"expected_substring": "return a + b"
}
]
def run_evaluation():
print("=== Starting Gemma Agent Advanced Benchmark Evaluation ===")
passed = 0
total = len(BENCHMARK_TASKS)
workspace = Path("./benchmark_workspace").resolve()
workspace.mkdir(exist_ok=True)
original_cwd = os.getcwd()
try:
for task in BENCHMARK_TASKS:
print(f"\nRunning {task['id']}...")
# Clean workspace before each task
for existing_file in workspace.glob("*"):
if existing_file.is_file():
existing_file.unlink()
# Setup test environment inside workspace
for filename, content in task["setup_files"].items():
file_path = workspace / filename
file_path.write_text(content)
# Execute agent query inside the workspace directory
try:
os.chdir(workspace)
response = process_query(task["prompt"])
os.chdir(original_cwd)
# Verify expected output across workspace files
success = False
for file_path in workspace.glob("*.py"):
file_content = file_path.read_text()
if task["expected_substring"] in file_content:
success = True
break
if success:
print(f"[{task['id']}] PASSED")
passed += 1
else:
print(f"[{task['id']}] FAILED (Expected pattern '{task['expected_substring']}' not found)")
except Exception as e:
os.chdir(original_cwd)
print(f"[{task['id']}] ERROR: {e}")
finally:
os.chdir(original_cwd)
print(f"\n=== Evaluation Summary ===")
print(f"Passed: {passed}/{total} ({(passed/total)*100:.1f}%)")
if __name__ == "__main__":
run_evaluation()
|