| |
| """End-to-end evaluation — 4-stage pipeline SOTA karşılaştırması. |
| |
| Metrics: |
| - Detection: mAP50-95 macro, per-source |
| - Classification: macro-F1 tier-stratified (head/mid/tail/rare), top-1/top-5 |
| - Open-set: AUROC on oltr_holdout, FPR@95TPR |
| - Transliteration: CER per language (hit/akk/sum/elx), BLEU, exact match |
| - E2E tablet: gold set accuracy (3253 hitit_local) |
| - Per-period breakdown: OH/MH/NH/OB/NA/NB/Ur-III/ACH |
| |
| Baselines for comparison: |
| - DeepScribe (RetinaNet+ResNet, replicated) |
| - CHURRO (Qwen2.5-VL-3B zero-shot) |
| - CuReD (Kraken transliteration) |
| - PreP-OCR (ByT5 without Viterbi) |
| """ |
| import json, argparse |
| from pathlib import Path |
| from collections import defaultdict, Counter |
| import numpy as np |
|
|
| def compute_map(predictions, ground_truth, iou_threshold=0.5): |
| """COCO-style mAP50-95.""" |
| |
| return {"mAP50": 0.0, "mAP50_95": 0.0} |
|
|
| def compute_macro_f1(preds, targets, tier_map=None): |
| """Tier-stratified macro-F1.""" |
| from sklearn.metrics import f1_score, balanced_accuracy_score |
| |
| overall = { |
| "macro_f1": float(f1_score(targets, preds, average='macro', zero_division=0)), |
| "top1_acc": float((preds == targets).mean()), |
| "balanced_acc": float(balanced_accuracy_score(targets, preds)), |
| } |
| if tier_map is None: |
| return overall |
| |
| per_tier = {} |
| for tier in ['head','mid','tail','rare']: |
| mask = np.array([tier_map.get(t) == tier for t in targets]) |
| if mask.sum() > 0: |
| per_tier[tier] = { |
| "n_samples": int(mask.sum()), |
| "macro_f1": float(f1_score(targets[mask], preds[mask], average='macro', zero_division=0)), |
| "top1_acc": float((preds[mask] == targets[mask]).mean()), |
| } |
| return {**overall, "per_tier": per_tier} |
|
|
| def compute_openset_auroc(in_energies, out_energies): |
| """AUROC for OOD detection (in-dist vs oltr_holdout).""" |
| from sklearn.metrics import roc_auc_score |
| y = np.concatenate([np.zeros(len(in_energies)), np.ones(len(out_energies))]) |
| scores = np.concatenate([in_energies, out_energies]) |
| return float(roc_auc_score(y, scores)) |
|
|
| def compute_cer(hypotheses, references): |
| """Character Error Rate (Levenshtein-based).""" |
| import editdistance |
| total_errors = 0 |
| total_chars = 0 |
| for h, r in zip(hypotheses, references): |
| total_errors += editdistance.eval(h, r) |
| total_chars += len(r) |
| return total_errors / max(1, total_chars) |
|
|
| def evaluate(args): |
| results = { |
| "detection": {}, |
| "classification": {}, |
| "openset": {}, |
| "transliteration": {}, |
| "e2e_tablet": {}, |
| "baselines": {}, |
| } |
| |
| |
| gold_records = [] |
| with open(args.gold_set) as f: |
| for line in f: |
| gold_records.append(json.loads(line)) |
| print(f"Gold set: {len(gold_records)} kayıt") |
| |
| |
| period_breakdown = defaultdict(list) |
| lang_breakdown = defaultdict(list) |
| for r in gold_records: |
| period_breakdown[r.get('period', '?')].append(r) |
| lang_breakdown[r.get('language', '?')].append(r) |
| |
| results['gold_set_breakdown'] = { |
| 'per_period': {p: len(v) for p, v in period_breakdown.items()}, |
| 'per_language': {l: len(v) for l, v in lang_breakdown.items()}, |
| } |
| |
| |
| results['targets_vs_actual'] = { |
| 'detection_mAP50_95': {'target': 0.80, 'deepscribe_baseline': 0.78}, |
| 'classification_macro_f1_head': {'target': 0.92}, |
| 'classification_macro_f1_mid': {'target': 0.75}, |
| 'classification_macro_f1_tail': {'target': 0.50}, |
| 'classification_macro_f1_rare': {'target': 0.25}, |
| 'openset_auroc': {'target': 0.85}, |
| 'cer_hit': {'target': 0.08, 'prep_ocr_baseline': 0.09}, |
| 'cer_akk': {'target': 0.09}, |
| 'cer_sum': {'target': 0.12}, |
| 'cer_elx': {'target': 0.10}, |
| 'churro_baseline_levenshtein': 0.823, |
| 'deepscribe_top5': 0.80, |
| } |
| |
| out = Path(args.output) |
| out.mkdir(parents=True, exist_ok=True) |
| with open(out / "evaluation_report.json", 'w') as f: |
| json.dump(results, f, indent=2, ensure_ascii=False) |
| print(f"Evaluation raporu: {out}/evaluation_report.json") |
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument('--detection-ckpts', nargs='+', help='YOLO per-fold checkpoints') |
| ap.add_argument('--classifier-ckpt') |
| ap.add_argument('--translit-ckpt') |
| ap.add_argument('--gold-set', default='datasets/eval_gold/manifest.jsonl') |
| ap.add_argument('--output', default='hitit_ocr/runs/eval_e2e_v1/') |
| args = ap.parse_args() |
| evaluate(args) |
|
|
| if __name__ == '__main__': |
| main() |
|
|
