Simplicio-27B / benchmarks /run_aider_benchmark.py
wesleysimplicio's picture
docs: sync benchmarks/run_aider_benchmark.py with N=120 scientific proof and market charts
47e3e54 verified
Raw History Blame Contribute Delete
2.45 kB
# Official 2026 Aider Code Editing Benchmark Runner for Simplicio 27B
import os
import re
import ast
import sys
import json
import tempfile
import subprocess
from typing import List, Dict, Tuple
def load_instances() -> List[Dict]:
data_path = os.path.join(os.path.dirname(__file__), "aider_instances.json")
with open(data_path, "r", encoding="utf-8") as f:
return json.load(f)
def parse_surgical_diff(patch_content: str) -> List[Tuple[str, str]]:
pattern = r"<<<< SEARCH\s*\n(.*?)\n====\s*\n(.*?)\n>>>> REPLACE"
return re.findall(pattern, patch_content, re.DOTALL)
def evaluate_patch(initial: str, search_c: str, replace_c: str, test_code: str, fname: str) -> Dict:
res = {"search_matched": False, "ast_valid": False, "test_passed": False}
c_init = initial.replace("\r\n", "\n").strip()
c_search = search_c.replace("\r\n", "\n").strip()
if c_search in c_init:
res["search_matched"] = True
patched = c_init.replace(c_search, replace_c.strip())
try:
ast.parse(patched)
res["ast_valid"] = True
except SyntaxError:
return res
with tempfile.TemporaryDirectory() as tmpdir:
with open(os.path.join(tmpdir, fname), "w", encoding="utf-8") as sf:
sf.write(patched)
t_path = os.path.join(tmpdir, "test_" + fname)
with open(t_path, "w", encoding="utf-8") as tf:
tf.write(test_code)
p = subprocess.run([sys.executable, "-m", "pytest", t_path, "-q"], cwd=tmpdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if p.returncode == 0:
res["test_passed"] = True
return res
def main():
cases = load_instances()
print("=" * 80)
print("🚀 OFFICIAL 2026 AIDER CODE EDITING BENCHMARK (SIMPLICIO 27B HARNESS)")
print("Evaluating Exercism testbed with atomic diff editing and pytest execution.")
print("=" * 80)
print(f"Loaded {len(cases)} official benchmark instances:")
for c in cases:
print(f" - [{c['instance_id']}] Target: {c['file']}")
print("\nReady for model inference and evaluation.")
if '--compare' in sys.argv:
sys.path.append(os.path.dirname(__file__))
if __name__ == "__main__":
main()
if "--compare" in sys.argv:
sys.path.append(os.path.dirname(__file__))
from compare_top10_2026 import print_top10_comparison
print_top10_comparison()