| """Makaledeki tüm sayısal değerleri çalıştırma çıktılarından yeniden hesaplar. |
| |
| Önce: python scripts/run_all.py ve python scripts/hata_enjeksiyonu.py |
| Sonra: python scripts/makale_sayilari.py -> outputs/makale_sayilari.json |
| |
| Kapsam: Tablo 3 (veri kümesi istatistikleri), Tablo 4 (çıkarım), Tablo 5 (EX, Wilson %95 GA, |
| McNemar), Şekil 3a (altın SQL sonuç dağılımı), Şekil 3b (kategori EX), Tablo 6 (hata enjeksiyonu). |
| """ |
| from __future__ import annotations |
|
|
| import collections |
| import glob |
| import json |
| import math |
| import re |
| import sys |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parent.parent |
| sys.path.insert(0, str(ROOT)) |
|
|
| from t2s.evaluate import gold_results, load_benchmark, results_match |
| from t2s.schema import connect |
| from t2s.trnum import parse_date |
|
|
| OUT = ROOT / "outputs" |
|
|
|
|
| def wilson(k: int, n: int, z: float = 1.96): |
| p = k / n |
| d = 1 + z * z / n |
| c = (p + z * z / (2 * n)) / d |
| h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d |
| return round(100 * (c - h), 1), round(100 * (c + h), 1) |
|
|
|
|
| def mcnemar_exact(b: int, c: int) -> float: |
| """İki yönlü kesin McNemar (binom, p=0,5).""" |
| n = b + c |
| k = min(b, c) |
| p = sum(math.comb(n, i) for i in range(0, k + 1)) / 2 ** n |
| return min(1.0, 2 * p) |
|
|
|
|
| def dataset_stats(): |
| tabs = [json.loads(Path(f).read_text(encoding="utf-8")) for f in sorted(glob.glob(str(ROOT / "data/ground_truth/*.json")))] |
| cells = [c for g in tabs for r in g["govde"] for c in r] |
| trnum = sum(1 for c in cells if re.fullmatch(r"-?\(?\d{1,3}(\.\d{3})+(,\d+)?\)?|-?\d+,\d+", c.strip("%"))) |
| body_merged = 0 |
| for g in tabs: |
| if g["bicim"] == "html": |
| html = (ROOT / "data/docs" / g["belge"]).read_text(encoding="utf-8") |
| body_merged += len(re.findall(r"<td[^>]*rowspan", html)) |
| items = load_benchmark() |
| return { |
| "belge": len({g["belge"] for g in tabs}), |
| "tablo": len(tabs), |
| "govde_satiri": sum(len(g["govde"]) for g in tabs), |
| "govde_hucresi": len(cells), |
| "tr_bicimli_sayi": trnum, "tr_bicimli_sayi_yuzde": round(100 * trnum / len(cells), 1), |
| "parantezli_negatif": sum(1 for c in cells if c.startswith("(")), |
| "yuzde_hucresi": sum(1 for c in cells if "%" in c), |
| "tarih_hucresi": sum(1 for c in cells if parse_date(c)), |
| "birlesik_baslik": sum(1 for g in tabs for r in g["baslik_satirlari"] for _, cs, rs in r if cs > 1 or rs > 1), |
| "birlesik_govde": body_merged, |
| "toplam_satirli_tablo": sum(1 for g in tabs if any(re.match(r"(?i)(genel )?toplam", r[0]) for r in g["govde"])), |
| "soru": len(items), |
| "bolum": dict(collections.Counter(i["bolum"] for i in items)), |
| "cevapsiz": sum(1 for i in items if not i["cevaplanabilir"]), |
| "zorluk": dict(collections.Counter(i["zorluk"] for i in items)), |
| "kanitli": sum(1 for i in items if i["kanit"]), |
| "uyarlama": dict(collections.Counter(i["uyarlama"].split(":")[0] for i in items)), |
| "coklu_tablo": sum(1 for i in items if len(i["tablolar"]) > 1), |
| } |
|
|
|
|
| def ablation_detail(): |
| items = [i for i in load_benchmark() if i["altin_sql"]] |
| gold = gold_results(items, OUT / "db/referans.db") |
| out = {} |
| for m in ("tr", "en", "text"): |
| con = connect(OUT / f"db/sistem_{m}.db", readonly=True) |
| c = collections.Counter() |
| for it in items: |
| rows = [tuple(r) for r in con.execute(it["altin_sql"]).fetchall()] |
| g = gold[it["id"]] |
| if results_match(rows, g): |
| c["dogru"] += 1 |
| elif not rows or all(all(v is None for v in r) for r in rows): |
| c["bos_null"] += 1 |
| elif any(isinstance(v, str) and re.search(r"\d[.,]\d", v) for r in rows for v in r): |
| c["metin_sayi"] += 1 |
| elif any(isinstance(v, (int, float)) for r in rows for v in r): |
| c["sayisal_yanlis"] += 1 |
| else: |
| c["diger_yanlis"] += 1 |
| con.close() |
| out[m] = dict(c) |
| return out |
|
|
|
|
| def main(): |
| oz = json.loads((OUT / "ozet.json").read_text(encoding="utf-8")) |
| rec = {s["sistem"]: [json.loads(l) for l in (OUT / f"sonuclar/sonuclar_{s['sistem']}.jsonl").read_text(encoding="utf-8").splitlines()] |
| for s in oz["sistemler"]} |
| ex = {} |
| for name, rs in rec.items(): |
| k = sum(r["dogru"] for r in rs) |
| t = [r for r in rs if r["bolum"] == "test"] |
| kt = sum(r["dogru"] for r in t) |
| ex[name] = {"dogru": k, "n": len(rs), "ga95": wilson(k, len(rs)), "test_dogru": kt, "test_n": len(t), |
| "test_ga95": wilson(kt, len(t))} |
| base = {r["id"]: r["dogru"] for r in rec["kural_tr"]} |
| mc = {} |
| for name in rec: |
| if name == "kural_tr": |
| continue |
| other = {r["id"]: r["dogru"] for r in rec[name]} |
| b = sum(1 for i in base if base[i] and not other[i]) |
| c = sum(1 for i in base if not base[i] and other[i]) |
| mc[name] = {"b": b, "c": c, "p": mcnemar_exact(b, c) if b + c else 1.0} |
| res = {"veri_kumesi": dataset_stats(), "ex": ex, "mcnemar": mc, "ablasyon": ablation_detail(), |
| "ozet": oz, "hata_enjeksiyonu": json.loads((OUT / "hata_enjeksiyonu.json").read_text(encoding="utf-8"))["ozet"]} |
| (OUT / "makale_sayilari.json").write_text(json.dumps(res, ensure_ascii=False, indent=1), encoding="utf-8") |
| (OUT / "sayi_ablasyon_detay.json").write_text(json.dumps(res["ablasyon"], ensure_ascii=False), encoding="utf-8") |
| print(json.dumps({k: res[k] for k in ("veri_kumesi", "ex", "mcnemar", "ablasyon")}, ensure_ascii=False, indent=1)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
