File size: 4,268 Bytes
9438a57 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | #!/usr/bin/env python3
"""Mutasyon skorunun örneklem büyüklüğüne duyarlılığı (referans paketlerle, LLM gerekmez).
Her öğede üretilebilen TÜM mutantlar bir kez değerlendirilir; örnekleme deterministik ve iç içe olduğundan
(N mutantlık örneklem, M > N örnekleminin ön ekidir) N = 10, 20, 30, 50 için skorlar aynı sonuçlardan
hesaplanır. Çıktı: reports/MUTASYON_DUYARLILIK.md (+ .json)
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "backend"))
from app.eval.benchmark import load_manifest # noqa: E402
from app.mutation.mutator import MutationContext, generate_mutants, run_mutation # noqa: E402
from app.sandbox.registry import get_runner # noqa: E402
SIZES = [10, 20, 30, 50]
def score(statuses: list[str]):
valid = [s for s in statuses if s != "invalid"]
return round(100.0 * sum(s in ("killed", "timeout") for s in valid) / len(valid), 1) if valid else None
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--out", default=str(ROOT / "reports" / "MUTASYON_DUYARLILIK.md"))
args = ap.parse_args()
junit4 = get_runner("java").framework() == "junit4"
rows = []
for it in load_manifest():
if not it.get("reference_tests") or it.get("kind") == "ui":
continue
key = "reference_tests_junit4" if junit4 and it.get("reference_tests_junit4") else "reference_tests"
tests = (ROOT / "benchmarks" / it[key]).read_text(encoding="utf-8")
runner = get_runner(it["language"], it.get("kind", "unit"))
base = runner.run(it["code"], tests, it.get("kind", "unit"), None, with_coverage=True)
lang = it["language"]
rep = run_mutation(MutationContext(
it["code"], lang, base,
lambda c: runner.run(c, tests, it.get("kind", "unit"), None, with_coverage=False,
timeout=max(20, int(base.duration_s * 3) + 10))), 100_000,
workers=2 if lang == "java" else 3)
status = {(m.line, m.operator, m.mutated): m.status for m in rep.mutants}
row = {"id": it["id"], "split": it.get("split", "dev"), "language": lang, "all_n": len(rep.mutants),
"all_valid": sum(m.status != "invalid" for m in rep.mutants),
"all": score([m.status for m in rep.mutants])}
for n in SIZES:
sample = generate_mutants(it["code"], lang, n)
row[str(n)] = score([status[(m.line, m.operator, m.mutated)] for m in sample])
rows.append(row)
print(f" {it['id']:<16} tümü={row['all']} ({row['all_n']} mutant) "
+ " ".join(f"N{n}={row[str(n)]}" for n in SIZES), flush=True)
def mean(k):
v = [r[k] for r in rows if r[k] is not None]
return round(sum(v) / len(v), 1) if v else None
def mad(k): # tüm mutantlara göre ortalama mutlak sapma
v = [abs(r[k] - r["all"]) for r in rows if r[k] is not None and r["all"] is not None]
return round(sum(v) / len(v), 1) if v else None
lines = ["# Mutasyon skorunun örneklem büyüklüğüne duyarlılığı", "",
"İnsan yazımı referans paketler; örnekleme operatör aileleri arasında dengeli, sabit tohumlu ve iç içe.", "",
"| Öğe | Bölüm | Dil | Mutant (tümü / geçerli) | " + " | ".join(f"N={n}" for n in SIZES) + " | Tümü |",
"|---|---|---|---|" + "---|" * len(SIZES) + "---|"]
for r in rows:
lines.append(f"| {r['id']} | {r['split']} | {r['language']} | {r['all_n']} / {r['all_valid']} | "
+ " | ".join(f"{r[str(n)]}" for n in SIZES) + f" | {r['all']} |")
lines.append("| **Ortalama** | | | | " + " | ".join(f"**{mean(str(n))}**" for n in SIZES) + f" | **{mean('all')}** |")
lines.append("| Tümüne göre ort. mutlak sapma | | | | " + " | ".join(f"{mad(str(n))}" for n in SIZES) + " | 0 |")
out = Path(args.out)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text("\n".join(lines) + "\n", encoding="utf-8")
out.with_suffix(".json").write_text(json.dumps(rows, indent=1))
print("\n".join(lines[-2:]), f"\n→ {out}")
if __name__ == "__main__":
main()
|