File size: 5,108 Bytes
a72d1cf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 | """Uçtan uca: örnek belgeler -> veritabanları -> test kümesi -> değerlendirme -> rapor.
Kullanım:
python scripts/run_all.py # kural tabanlı sistemler (anahtar gerekmez)
python scripts/run_all.py --claude # + Claude (ANTHROPIC_API_KEY gerekli)
python scripts/run_all.py --openai gpt-4o-mini
python scripts/run_all.py --ollama qwen2.5-coder:7b
python scripts/run_all.py --hizli # yalnızca TR kipi, kural tabanlı
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
import warnings
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
warnings.filterwarnings("ignore")
from t2s.errors import ErrorLog # noqa: E402
from t2s.evaluate import SystemSpec, evaluate_system, gold_on_naive_db, load_benchmark # noqa: E402
from t2s.pipeline import (DOCS_DIR, build_database, build_reference_db, extraction_accuracy, # noqa: E402
number_format_stats)
from t2s.report import write_report # noqa: E402
OUT = ROOT / "outputs"
DB = OUT / "db"
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--claude", nargs="?", const="auto", default=None,
help="Claude model adı (boş/auto: API'deki en yeni model)")
ap.add_argument("--openai", nargs="?", const="auto", default=None, help="OpenAI uyumlu model adı (auto: ilk model)")
ap.add_argument("--ollama", nargs="?", const="auto", default=None, help="Ollama model adı (auto: kurulu ilk model)")
ap.add_argument("--hizli", action="store_true")
args = ap.parse_args()
print("1) Örnek belgeler")
if not any(DOCS_DIR.glob("*.pdf")):
subprocess.run([sys.executable, str(ROOT / "data" / "generate_samples.py")], check=True)
print(" ", ", ".join(sorted(p.name for p in DOCS_DIR.iterdir())))
print("2) Veritabanları (referans + 3 sayı kipi)")
build_reference_db(DB / "referans.db")
modes = ["tr"] if args.hizli else ["tr", "en", "text"]
db_paths = {}
for m in modes:
db_paths[m] = DB / f"sistem_{m}.db"
build_database(db_paths[m], mode=m, verbose=(m == "tr"))
print("3) Tablo çıkarma doğruluğu")
extraction = {}
for meth in ["pdfplumber", "camelot-lattice", "camelot-stream", "auto"]:
extraction[meth] = extraction_accuracy(meth)
acc = sum(r["hucre_dogrulugu"] for r in extraction[meth]) / len(extraction[meth])
print(f" {meth:16} ort. hücre doğruluğu: %{100 * acc:.1f}")
print("4) Test kümesi")
subprocess.run([sys.executable, str(ROOT / "benchmark" / "build_benchmark.py")], check=True)
items = load_benchmark()
print("5) Sistemler")
specs = [SystemSpec("kural_tr", "kural", "tr", True),
SystemSpec("kural_tr_duzeltmesiz", "kural", "tr", False)]
if not args.hizli:
specs += [SystemSpec("kural_en_naif", "kural", "en", True),
SystemSpec("kural_metin_naif", "kural", "text", True)]
llms = []
from t2s.llm import list_models, make_backend
for key, model in (("claude", args.claude), ("openai", args.openai), ("ollama", args.ollama)):
if not model:
continue
if model == "auto": # model senkronizasyonu: sağlayıcının listesinden seç
found = list_models(key)
if not found:
raise SystemExit(f"{key}: kullanılabilir model bulunamadı")
model = found[0].id
print(f" {key}: model otomatik seçildi -> {model}")
llms.append((key, make_backend(key, model)))
for key, be in llms:
specs += [SystemSpec(f"{key}_tr", be, "tr", True), SystemSpec(f"{key}_tr_duzeltmesiz", be, "tr", False)]
if not args.hizli:
specs += [SystemSpec(f"{key}_en_naif", be, "en", True), SystemSpec(f"{key}_metin_naif", be, "text", True)]
log_path = OUT / "hata_gunlugu.jsonl"
if log_path.exists():
log_path.unlink()
elog = ErrorLog(log_path)
summaries = []
for spec in specs:
if spec.db_mode not in db_paths:
continue
s = evaluate_system(spec, items, db_paths, DB / "referans.db", OUT / "sonuclar", elog)
summaries.append(s)
print(f" {spec.name:26} EX=%{s['EX']:5} test=%{s['bolum'].get('test')} "
f"ilk deneme=%{s['EX_ilk_deneme']} cevapsız doğru red=%{s['cevapsiz_dogru_red']}")
print("6) Sayı biçimi ablasyonu (altın SQL, naif veritabanları)")
naive = gold_on_naive_db(items, db_paths, DB / "referans.db")
for r in naive:
print(f" {r['kip']:5} altın SQL EX=%{r['EX']} (bozulan {len(r['bozulan'])}/{r['n']})")
nstats = number_format_stats(db_paths)
ozet = {"sistemler": summaries, "cikarim": extraction, "sayi_ablasyonu": naive,
"sayi_istatistik": nstats, "soru_sayisi": len(items)}
(OUT / "ozet.json").write_text(json.dumps(ozet, ensure_ascii=False, indent=1), encoding="utf-8")
rp = write_report(ozet, OUT / "basarim_raporu.html")
print("7) Rapor ->", rp)
if __name__ == "__main__":
main()
|