File size: 5,148 Bytes
9438a57 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | #!/usr/bin/env python3
"""Hakem ajanın doğruluğu: benchmarks/judge/claims.json üzerindeki etiketli iddialarla ölçülür.
python scripts/judge_accuracy.py --model qwen2.5-coder:14b # kural + hakem modeli
python scripts/judge_accuracy.py --model qwen2.5-coder:14b --rule-mode hint # kural ipucu olarak
python scripts/judge_accuracy.py --model qwen2.5-coder:14b --rule-mode off # yalnızca hakem modeli
python scripts/judge_accuracy.py --rule-only # yalnızca kural aşaması (LLM gerekmez)
"Pozitif" = iddia kabul edildi (code_bug). Gerçek hatalarda kabul doğru (TP), yanlış iddialarda kabul
yanlış pozitiftir (FP). Çıktı: reports/deney/hakem_<model>.json ve konsolda özet.
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "backend"))
from app.agents.judge import Verdict, deliberate_raise_line, judge_bug_report # noqa: E402
from app.sandbox.registry import get_runner # noqa: E402
def metrics(rows: list[dict]) -> dict:
tp = sum(r["label"] == "code_bug" and r["accepted"] for r in rows)
fn = sum(r["label"] == "code_bug" and not r["accepted"] for r in rows)
fp = sum(r["label"] == "test_bug" and r["accepted"] for r in rows)
tn = sum(r["label"] == "test_bug" and not r["accepted"] for r in rows)
from decimal import ROUND_HALF_UP, Decimal
def r1(x): # yarım yukarı yuvarlama (56,25 → 56,3)
return float(Decimal(str(x)).quantize(Decimal("0.1"), rounding=ROUND_HALF_UP)) if x is not None else None
raw = lambda a, b: 100.0 * a / b if b else None # noqa: E731
rec, spe = raw(tp, tp + fn), raw(tn, tn + fp)
return {"n": len(rows), "tp": tp, "fn": fn, "fp": fp, "tn": tn,
"accuracy": r1(raw(tp + tn, len(rows))), "precision": r1(raw(tp, tp + fp)),
"recall": r1(rec), "specificity": r1(spe),
"balanced_accuracy": r1((rec + spe) / 2) if None not in (rec, spe) else None}
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--model")
ap.add_argument("--rule-only", action="store_true")
ap.add_argument("--rule-mode", choices=["veto", "hint", "off"], default="veto")
ap.add_argument("--out-dir", default=str(ROOT / "reports" / "deney"))
args = ap.parse_args()
claims = json.loads((ROOT / "benchmarks" / "judge" / "claims.json").read_text(encoding="utf-8"))
chat = None
if not args.rule_only:
from app.llm.providers import make_llm, is_local
client = make_llm(args.model)
if is_local(args.model) and not client.health()["ok"]:
sys.exit("Ollama'ya ulaşılamadı.")
chat = lambda msgs, **kw: client.chat(msgs, **kw) # noqa: E731
rows = []
t0 = time.time()
for c in claims:
src_file = get_runner(c["language"]).source_filename(c["code"])
if args.rule_only:
ln = deliberate_raise_line(c["code"], src_file, c["failure"])
v = Verdict(False, f"raise/throw satırı {ln}", "kural") if ln else Verdict(True, "kural karar vermedi", "-")
else:
v = judge_bug_report(chat, c["code"], src_file, c["language"], c["test_code"], c["test_name"],
c["failure"], c["claim"], rule_mode=args.rule_mode)
kind = "gerçek hata" if c["label"] == "code_bug" else ("kasıtlı doğrulama" if "-raise" in c["id"] else "yanlış değer")
rows.append({"id": c["id"], "split": c["split"], "label": c["label"], "kind": kind, "accepted": v.accepted,
"by": v.by, "reason": v.reason})
mark = "✓" if v.accepted == (c["label"] == "code_bug") else "✗"
print(f" {mark} {c['id']:<22} {c['label']:<9} kabul={v.accepted!s:<5} ({v.by})")
res = {"model": "kural" if args.rule_only else (args.model or "varsayılan"),
"rule_mode": "yalnız kural" if args.rule_only else args.rule_mode,
"overall": metrics(rows),
"by_stage": {"kural_reddi": sum(r["by"] == "kural" for r in rows)},
"by_split": {s: metrics([r for r in rows if r["split"] == s]) for s in sorted({r["split"] for r in rows})},
"by_kind": {k: {"n": sum(r["kind"] == k for r in rows),
"accepted": sum(r["kind"] == k and r["accepted"] for r in rows),
"rule_rejected": sum(r["kind"] == k and r["by"] == "kural" for r in rows)}
for k in ("gerçek hata", "yanlış değer", "kasıtlı doğrulama")},
"duration_s": round(time.time() - t0, 1), "rows": rows}
out = Path(args.out_dir)
out.mkdir(parents=True, exist_ok=True)
name = "hakem_kural.json" if args.rule_only else f"hakem_{res['model'].replace(':', '-').replace('/', '-')}_{args.rule_mode}.json"
(out / name).write_text(json.dumps(res, ensure_ascii=False, indent=1), encoding="utf-8")
print(json.dumps(res["overall"], ensure_ascii=False), f"\n→ {out / name}")
if __name__ == "__main__":
main()
|