| """Uçtan uca: örnek belgeler -> veritabanları -> test kümesi -> değerlendirme -> rapor. |
| |
| Kullanım: |
| python scripts/run_all.py # kural tabanlı sistemler (anahtar gerekmez) |
| python scripts/run_all.py --claude # + Claude (ANTHROPIC_API_KEY gerekli) |
| python scripts/run_all.py --openai gpt-4o-mini |
| python scripts/run_all.py --ollama qwen2.5-coder:7b |
| python scripts/run_all.py --hizli # yalnızca TR kipi, kural tabanlı |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import subprocess |
| import sys |
| import warnings |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parent.parent |
| sys.path.insert(0, str(ROOT)) |
| warnings.filterwarnings("ignore") |
|
|
| from t2s.errors import ErrorLog |
| from t2s.evaluate import SystemSpec, evaluate_system, gold_on_naive_db, load_benchmark |
| from t2s.pipeline import (DOCS_DIR, build_database, build_reference_db, extraction_accuracy, |
| number_format_stats) |
| from t2s.report import write_report |
|
|
| OUT = ROOT / "outputs" |
| DB = OUT / "db" |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--claude", nargs="?", const="auto", default=None, |
| help="Claude model adı (boş/auto: API'deki en yeni model)") |
| ap.add_argument("--openai", nargs="?", const="auto", default=None, help="OpenAI uyumlu model adı (auto: ilk model)") |
| ap.add_argument("--ollama", nargs="?", const="auto", default=None, help="Ollama model adı (auto: kurulu ilk model)") |
| ap.add_argument("--hizli", action="store_true") |
| args = ap.parse_args() |
|
|
| print("1) Örnek belgeler") |
| if not any(DOCS_DIR.glob("*.pdf")): |
| subprocess.run([sys.executable, str(ROOT / "data" / "generate_samples.py")], check=True) |
| print(" ", ", ".join(sorted(p.name for p in DOCS_DIR.iterdir()))) |
|
|
| print("2) Veritabanları (referans + 3 sayı kipi)") |
| build_reference_db(DB / "referans.db") |
| modes = ["tr"] if args.hizli else ["tr", "en", "text"] |
| db_paths = {} |
| for m in modes: |
| db_paths[m] = DB / f"sistem_{m}.db" |
| build_database(db_paths[m], mode=m, verbose=(m == "tr")) |
|
|
| print("3) Tablo çıkarma doğruluğu") |
| extraction = {} |
| for meth in ["pdfplumber", "camelot-lattice", "camelot-stream", "auto"]: |
| extraction[meth] = extraction_accuracy(meth) |
| acc = sum(r["hucre_dogrulugu"] for r in extraction[meth]) / len(extraction[meth]) |
| print(f" {meth:16} ort. hücre doğruluğu: %{100 * acc:.1f}") |
|
|
| print("4) Test kümesi") |
| subprocess.run([sys.executable, str(ROOT / "benchmark" / "build_benchmark.py")], check=True) |
| items = load_benchmark() |
|
|
| print("5) Sistemler") |
| specs = [SystemSpec("kural_tr", "kural", "tr", True), |
| SystemSpec("kural_tr_duzeltmesiz", "kural", "tr", False)] |
| if not args.hizli: |
| specs += [SystemSpec("kural_en_naif", "kural", "en", True), |
| SystemSpec("kural_metin_naif", "kural", "text", True)] |
| llms = [] |
| from t2s.llm import list_models, make_backend |
| for key, model in (("claude", args.claude), ("openai", args.openai), ("ollama", args.ollama)): |
| if not model: |
| continue |
| if model == "auto": |
| found = list_models(key) |
| if not found: |
| raise SystemExit(f"{key}: kullanılabilir model bulunamadı") |
| model = found[0].id |
| print(f" {key}: model otomatik seçildi -> {model}") |
| llms.append((key, make_backend(key, model))) |
| for key, be in llms: |
| specs += [SystemSpec(f"{key}_tr", be, "tr", True), SystemSpec(f"{key}_tr_duzeltmesiz", be, "tr", False)] |
| if not args.hizli: |
| specs += [SystemSpec(f"{key}_en_naif", be, "en", True), SystemSpec(f"{key}_metin_naif", be, "text", True)] |
|
|
| log_path = OUT / "hata_gunlugu.jsonl" |
| if log_path.exists(): |
| log_path.unlink() |
| elog = ErrorLog(log_path) |
| summaries = [] |
| for spec in specs: |
| if spec.db_mode not in db_paths: |
| continue |
| s = evaluate_system(spec, items, db_paths, DB / "referans.db", OUT / "sonuclar", elog) |
| summaries.append(s) |
| print(f" {spec.name:26} EX=%{s['EX']:5} test=%{s['bolum'].get('test')} " |
| f"ilk deneme=%{s['EX_ilk_deneme']} cevapsız doğru red=%{s['cevapsiz_dogru_red']}") |
|
|
| print("6) Sayı biçimi ablasyonu (altın SQL, naif veritabanları)") |
| naive = gold_on_naive_db(items, db_paths, DB / "referans.db") |
| for r in naive: |
| print(f" {r['kip']:5} altın SQL EX=%{r['EX']} (bozulan {len(r['bozulan'])}/{r['n']})") |
| nstats = number_format_stats(db_paths) |
|
|
| ozet = {"sistemler": summaries, "cikarim": extraction, "sayi_ablasyonu": naive, |
| "sayi_istatistik": nstats, "soru_sayisi": len(items)} |
| (OUT / "ozet.json").write_text(json.dumps(ozet, ensure_ascii=False, indent=1), encoding="utf-8") |
| rp = write_report(ozet, OUT / "basarim_raporu.html") |
| print("7) Rapor ->", rp) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
