| |
| """Klasik (arama tabanlı) taban çizgisi: Pynguin ile Python öğeleri için test üretip ajanla AYNI ölçüm |
| hattından (kapsama, mutasyon, yerleştirilmiş hatalar) geçirir. |
| |
| PYNGUIN=/yol/pynguin python scripts/pynguin_baseline.py --seeds 1 2 3 --budget 120 |
| python scripts/pynguin_baseline.py --measure-only # üretilmiş testleri yeniden ölç |
| |
| Pynguin Python 3.10 ister ve PYNGUIN_DANGER_AWARE ortam değişkeniyle çalışır (kodu çalıştırır). |
| Üretilen testler benchmarks/baselines/pynguin/<öğe>_s<tohum>.py olarak saklanır. |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import os |
| import shutil |
| import subprocess |
| import sys |
| import tempfile |
| import time |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| sys.path.insert(0, str(ROOT / "backend")) |
|
|
| from app.eval import benchmark |
| from app.eval.report import render |
| from app.llm.scripted import ScriptedLLM, text, tool |
|
|
| OUT = ROOT / "benchmarks" / "baselines" / "pynguin" |
|
|
|
|
| def generate(item: dict, seed: int, budget: int, pynguin: str) -> dict: |
| with tempfile.TemporaryDirectory() as tmp: |
| proj, out = Path(tmp) / "proj", Path(tmp) / "out" |
| proj.mkdir() |
| (proj / "solution.py").write_text(item["code"], encoding="utf-8") |
| t0 = time.time() |
| p = subprocess.run([pynguin, "--project-path", str(proj), "--module-name", "solution", |
| "--output-path", str(out), "--maximum-search-time", str(budget), "--seed", str(seed)], |
| capture_output=True, text=True, env=os.environ | {"PYNGUIN_DANGER_AWARE": "1"}, |
| timeout=budget * 4 + 300) |
| dur = round(time.time() - t0, 1) |
| gen = out / "test_solution.py" |
| if not gen.exists(): |
| return {"item": item["id"], "seed": seed, "ok": False, "duration_s": dur, "log": p.stdout[-1500:]} |
| dest = OUT / f"{item['id']}_s{seed}.py" |
| shutil.copy(gen, dest) |
| return {"item": item["id"], "seed": seed, "ok": True, "duration_s": dur, "file": dest.name} |
|
|
|
|
| def main() -> None: |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) |
| ap.add_argument("--seeds", type=int, nargs="*", default=[1, 2, 3]) |
| ap.add_argument("--budget", type=int, default=120, help="Pynguin arama süresi (sn)") |
| ap.add_argument("--max-mutants", type=int, default=30) |
| ap.add_argument("--measure-only", action="store_true") |
| ap.add_argument("--split", choices=["dev", "test", "bagimsiz", "all"], default="all") |
| ap.add_argument("--out", default=str(ROOT / "reports" / "PYNGUIN_TABAN_CIZGISI.md")) |
| args = ap.parse_args() |
| items = [i for i in benchmark.load_manifest() if i["language"] == "python" |
| and (args.split == "all" or i.get("split", "dev") == args.split)] |
| OUT.mkdir(parents=True, exist_ok=True) |
| meta_path = OUT / "generation.json" |
| meta = json.loads(meta_path.read_text()) if meta_path.exists() else [] |
|
|
| if not args.measure_only: |
| pynguin = os.environ.get("PYNGUIN", "pynguin") |
| for seed in args.seeds: |
| for it in items: |
| print(f" üretiliyor: {it['id']} tohum={seed} …", flush=True) |
| r = generate(it, seed, args.budget, pynguin) |
| meta = [m for m in meta if not (m["item"] == r["item"] and m["seed"] == seed)] + [r] |
| meta_path.write_text(json.dumps(meta, ensure_ascii=False, indent=1)) |
| print(f" {'tamam' if r['ok'] else 'BAŞARISIZ'} ({r['duration_s']} sn)", flush=True) |
|
|
| evals = [] |
| for seed in args.seeds: |
| files = {it["id"]: OUT / f"{it['id']}_s{seed}.py" for it in items} |
| ids = [k for k, f in files.items() if f.exists()] |
|
|
| def factory(item, files=files): |
| return ScriptedLLM([text('{"summary": "Pynguin taban çizgisi"}'), |
| tool("write_tests", code=files[item["id"]].read_text(encoding="utf-8")), |
| tool("finish", summary="pynguin")], model="pynguin") |
|
|
| cfg = {"max_iterations": 1, "max_mutants": args.max_mutants, "coverage_target": 85, "run_mutation": True} |
| ev = benchmark.run_eval(f"Pynguin 0.47 (tohum {seed}, {args.budget} sn)", cfg, ids, None, background=False, |
| llm_factory=factory, label=f"pynguin-s{seed}") |
| for it in ev.items: |
| print(f" s{seed} {it.id:<14} geçen=%{it.final_pass_rate} kapsama={it.final_coverage} " |
| f"mutasyon={it.mutation_score} yerleştirilmiş={it.seeded_caught}/{it.seeded_total} test={it.tests}") |
| evals.append(ev) |
| out = Path(args.out) |
| out.parent.mkdir(parents=True, exist_ok=True) |
| out.write_text(render(evals, title="Pynguin (arama tabanlı) taban çizgisi", |
| note="Pynguin 0.47.0, Python 3.10; varsayılan DynaMOSA algoritması ve varsayılan " |
| "doğrulama üretimi. Testler ajanla aynı ölçüm hattında değerlendirilmiştir."), |
| encoding="utf-8") |
| out.with_suffix(".json").write_text(json.dumps([e.model_dump() for e in evals], ensure_ascii=False, indent=2)) |
| print(f"Rapor: {out}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
