| |
| """Hakem ajanın doğruluğu: benchmarks/judge/claims.json üzerindeki etiketli iddialarla ölçülür. |
| |
| python scripts/judge_accuracy.py --model qwen2.5-coder:14b # kural + hakem modeli |
| python scripts/judge_accuracy.py --model qwen2.5-coder:14b --rule-mode hint # kural ipucu olarak |
| python scripts/judge_accuracy.py --model qwen2.5-coder:14b --rule-mode off # yalnızca hakem modeli |
| python scripts/judge_accuracy.py --rule-only # yalnızca kural aşaması (LLM gerekmez) |
| |
| "Pozitif" = iddia kabul edildi (code_bug). Gerçek hatalarda kabul doğru (TP), yanlış iddialarda kabul |
| yanlış pozitiftir (FP). Çıktı: reports/deney/hakem_<model>.json ve konsolda özet. |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import sys |
| import time |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| sys.path.insert(0, str(ROOT / "backend")) |
|
|
| from app.agents.judge import Verdict, deliberate_raise_line, judge_bug_report |
| from app.sandbox.registry import get_runner |
|
|
|
|
| def metrics(rows: list[dict]) -> dict: |
| tp = sum(r["label"] == "code_bug" and r["accepted"] for r in rows) |
| fn = sum(r["label"] == "code_bug" and not r["accepted"] for r in rows) |
| fp = sum(r["label"] == "test_bug" and r["accepted"] for r in rows) |
| tn = sum(r["label"] == "test_bug" and not r["accepted"] for r in rows) |
| from decimal import ROUND_HALF_UP, Decimal |
|
|
| def r1(x): |
| return float(Decimal(str(x)).quantize(Decimal("0.1"), rounding=ROUND_HALF_UP)) if x is not None else None |
|
|
| raw = lambda a, b: 100.0 * a / b if b else None |
| rec, spe = raw(tp, tp + fn), raw(tn, tn + fp) |
| return {"n": len(rows), "tp": tp, "fn": fn, "fp": fp, "tn": tn, |
| "accuracy": r1(raw(tp + tn, len(rows))), "precision": r1(raw(tp, tp + fp)), |
| "recall": r1(rec), "specificity": r1(spe), |
| "balanced_accuracy": r1((rec + spe) / 2) if None not in (rec, spe) else None} |
|
|
|
|
| def main() -> None: |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) |
| ap.add_argument("--model") |
| ap.add_argument("--rule-only", action="store_true") |
| ap.add_argument("--rule-mode", choices=["veto", "hint", "off"], default="veto") |
| ap.add_argument("--out-dir", default=str(ROOT / "reports" / "deney")) |
| args = ap.parse_args() |
| claims = json.loads((ROOT / "benchmarks" / "judge" / "claims.json").read_text(encoding="utf-8")) |
|
|
| chat = None |
| if not args.rule_only: |
| from app.llm.providers import make_llm, is_local |
| client = make_llm(args.model) |
| if is_local(args.model) and not client.health()["ok"]: |
| sys.exit("Ollama'ya ulaşılamadı.") |
| chat = lambda msgs, **kw: client.chat(msgs, **kw) |
|
|
| rows = [] |
| t0 = time.time() |
| for c in claims: |
| src_file = get_runner(c["language"]).source_filename(c["code"]) |
| if args.rule_only: |
| ln = deliberate_raise_line(c["code"], src_file, c["failure"]) |
| v = Verdict(False, f"raise/throw satırı {ln}", "kural") if ln else Verdict(True, "kural karar vermedi", "-") |
| else: |
| v = judge_bug_report(chat, c["code"], src_file, c["language"], c["test_code"], c["test_name"], |
| c["failure"], c["claim"], rule_mode=args.rule_mode) |
| kind = "gerçek hata" if c["label"] == "code_bug" else ("kasıtlı doğrulama" if "-raise" in c["id"] else "yanlış değer") |
| rows.append({"id": c["id"], "split": c["split"], "label": c["label"], "kind": kind, "accepted": v.accepted, |
| "by": v.by, "reason": v.reason}) |
| mark = "✓" if v.accepted == (c["label"] == "code_bug") else "✗" |
| print(f" {mark} {c['id']:<22} {c['label']:<9} kabul={v.accepted!s:<5} ({v.by})") |
| res = {"model": "kural" if args.rule_only else (args.model or "varsayılan"), |
| "rule_mode": "yalnız kural" if args.rule_only else args.rule_mode, |
| "overall": metrics(rows), |
| "by_stage": {"kural_reddi": sum(r["by"] == "kural" for r in rows)}, |
| "by_split": {s: metrics([r for r in rows if r["split"] == s]) for s in sorted({r["split"] for r in rows})}, |
| "by_kind": {k: {"n": sum(r["kind"] == k for r in rows), |
| "accepted": sum(r["kind"] == k and r["accepted"] for r in rows), |
| "rule_rejected": sum(r["kind"] == k and r["by"] == "kural" for r in rows)} |
| for k in ("gerçek hata", "yanlış değer", "kasıtlı doğrulama")}, |
| "duration_s": round(time.time() - t0, 1), "rows": rows} |
| out = Path(args.out_dir) |
| out.mkdir(parents=True, exist_ok=True) |
| name = "hakem_kural.json" if args.rule_only else f"hakem_{res['model'].replace(':', '-').replace('/', '-')}_{args.rule_mode}.json" |
| (out / name).write_text(json.dumps(res, ensure_ascii=False, indent=1), encoding="utf-8") |
| print(json.dumps(res["overall"], ensure_ascii=False), f"\n→ {out / name}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
