| |
| """Hakem ajanın doğruluğunu ölçmek için etiketli iddia kümesi üretir (LLM gerekmez). |
| |
| * code_bug (gerçek hata): her yerleştirilmiş hata için hatalı kod + referans paketinde o hata yüzünden |
| başarısız olan ilk test. |
| * test_bug (yanlış iddia), iki tür: |
| - yanlış değer: orijinal (doğru) kod + beklenen değeri bir birim kaydırılmış bir doğrulama; |
| - kasıtlı doğrulama: istisna beklemesi gereken çağrının istisna beklentisi kaldırılmış (test başarı |
| bekliyor, kod bilerek fırlatıyor) — küçük modellerin gerçek koşularda en sık yaptığı yanlış bildirim. |
| İki sınıfta da iddia metni aynı şablondur; hakem yalnızca kod, test ve hata çıktısına bakarak karar verir. |
| |
| Çıktı: benchmarks/judge/claims.json |
| """ |
| from __future__ import annotations |
|
|
| import json |
| import re |
| import sys |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| sys.path.insert(0, str(ROOT / "backend")) |
|
|
| from app.eval.benchmark import load_manifest |
| from app.sandbox.registry import get_runner |
|
|
| CLAIM = ("The test encodes the behaviour documented in the code (docstring/comments/names), but the code " |
| "produces a different result. This is a bug in the code under test, not in the test.") |
| MAX_NEG_PER_ITEM = 3 |
|
|
| |
| NUM_PATTERNS = { |
| "python": re.compile(r"(==\s*)(-?\d+)(?![\d.])"), |
| "javascript": re.compile(r"(\.toBe\(|\.toEqual\()(-?\d+)(?=\))"), |
| "typescript": re.compile(r"(\.toBe\(|\.toEqual\()(-?\d+)(?=\))"), |
| "java": re.compile(r"(assertEquals\()(-?\d+)(?=,)"), |
| } |
|
|
|
|
| RAISE_PATTERNS = { |
| "python": (re.compile(r"^( *)with pytest\.raises\([^)]*\):\n +(.+)$", re.M), r"\1\2"), |
| "javascript": (re.compile(r"expect\(\(\) => (.+?)\)\.toThrow\([^)]*\);"), r"\1;"), |
| "typescript": (re.compile(r"expect\(\(\) => (.+?)\)\.toThrow\([^)]*\);"), r"\1;"), |
| "java": (re.compile(r"assertThrows\(\w+\.class, \(\) -> (.+?)\);"), r"\1;"), |
| } |
| MAX_RAISE_PER_ITEM = 2 |
|
|
|
|
| def ref_key(item, junit4): |
| return "reference_tests_junit4" if junit4 and item.get("reference_tests_junit4") else "reference_tests" |
|
|
|
|
| def main() -> None: |
| junit4 = get_runner("java").framework() == "junit4" |
| claims: list[dict] = [] |
| for it in load_manifest(): |
| |
| if not it.get("reference_tests") or it.get("kind") == "ui" or it.get("split", "dev") not in ("dev", "test"): |
| continue |
| runner = get_runner(it["language"], it.get("kind", "unit")) |
| test_code = (ROOT / "benchmarks" / it[ref_key(it, junit4)]).read_text(encoding="utf-8") |
| base = runner.run(it["code"], test_code, it.get("kind", "unit"), None, with_coverage=False) |
| if not base.all_green: |
| print(f" ! {it['id']}: referans paket yeşil değil, atlandı") |
| continue |
| |
| for k, bug in enumerate(it.get("seeded_bugs", [])): |
| buggy = it["code"].replace(bug["from"], bug["to"], 1) |
| r = runner.run(buggy, test_code, it.get("kind", "unit"), None, with_coverage=False) |
| bad = [c for c in r.cases if c.status in ("failed", "error")] if r.compiled else [] |
| if not bad: |
| continue |
| c = bad[0] |
| claims.append({"id": f"{it['id']}-bug{k + 1}", "item": it["id"], "split": it.get("split", "dev"), |
| "language": it["language"], "label": "code_bug", "code": buggy, "test_code": test_code, |
| "test_name": c.name, "failure": c.message[:2000], "claim": CLAIM, |
| "note": bug["description"]}) |
| |
| pat = NUM_PATTERNS[it["language"]] |
| neg = 0 |
| for m in pat.finditer(test_code): |
| if neg >= MAX_NEG_PER_ITEM: |
| break |
| v = int(m.group(2)) |
| mutated = test_code[:m.start(2)] + str(v + 1) + test_code[m.end(2):] |
| r = runner.run(it["code"], mutated, it.get("kind", "unit"), None, with_coverage=False) |
| bad = [c for c in r.cases if c.status in ("failed", "error")] if r.compiled else [] |
| if len(bad) != 1: |
| continue |
| c = bad[0] |
| line = test_code.count("\n", 0, m.start()) + 1 |
| claims.append({"id": f"{it['id']}-wrong{neg + 1}", "item": it["id"], "split": it.get("split", "dev"), |
| "language": it["language"], "label": "test_bug", "code": it["code"], |
| "test_code": mutated, "test_name": c.name, "failure": c.message[:2000], "claim": CLAIM, |
| "note": f"test satırı {line}: beklenen {v} → {v + 1}"}) |
| neg += 1 |
| rpat, repl = RAISE_PATTERNS[it["language"]] |
| nr = 0 |
| for m in rpat.finditer(test_code): |
| if nr >= MAX_RAISE_PER_ITEM: |
| break |
| mutated = test_code[:m.start()] + m.expand(repl) + test_code[m.end():] |
| r = runner.run(it["code"], mutated, it.get("kind", "unit"), None, with_coverage=False) |
| bad = [c for c in r.cases if c.status in ("failed", "error")] if r.compiled else [] |
| if len(bad) != 1: |
| continue |
| c = bad[0] |
| line = test_code.count("\n", 0, m.start()) + 1 |
| claims.append({"id": f"{it['id']}-raise{nr + 1}", "item": it["id"], "split": it.get("split", "dev"), |
| "language": it["language"], "label": "test_bug", "code": it["code"], |
| "test_code": mutated, "test_name": c.name, "failure": c.message[:2000], "claim": CLAIM, |
| "note": f"test satırı {line}: istisna beklentisi kaldırıldı"}) |
| nr += 1 |
| print(f" {it['id']:<16} toplam iddia: {sum(1 for c in claims if c['item'] == it['id'])}") |
| out = ROOT / "benchmarks" / "judge" / "claims.json" |
| out.write_text(json.dumps(claims, ensure_ascii=False, indent=1), encoding="utf-8") |
| pos = sum(c["label"] == "code_bug" for c in claims) |
| nraise = sum("-raise" in c["id"] for c in claims) |
| print(f"{len(claims)} iddia ({pos} gerçek hata, {len(claims) - pos - nraise} yanlış değer, " |
| f"{nraise} kasıtlı doğrulama) → {out}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
