File size: 4,809 Bytes
f211247 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 | #!/usr/bin/env python3
"""End-to-end evaluation — 4-stage pipeline SOTA karşılaştırması.
Metrics:
- Detection: mAP50-95 macro, per-source
- Classification: macro-F1 tier-stratified (head/mid/tail/rare), top-1/top-5
- Open-set: AUROC on oltr_holdout, FPR@95TPR
- Transliteration: CER per language (hit/akk/sum/elx), BLEU, exact match
- E2E tablet: gold set accuracy (3253 hitit_local)
- Per-period breakdown: OH/MH/NH/OB/NA/NB/Ur-III/ACH
Baselines for comparison:
- DeepScribe (RetinaNet+ResNet, replicated)
- CHURRO (Qwen2.5-VL-3B zero-shot)
- CuReD (Kraken transliteration)
- PreP-OCR (ByT5 without Viterbi)
"""
import json, argparse
from pathlib import Path
from collections import defaultdict, Counter
import numpy as np
def compute_map(predictions, ground_truth, iou_threshold=0.5):
"""COCO-style mAP50-95."""
# Placeholder — gerçek impl için pycocotools kullan
return {"mAP50": 0.0, "mAP50_95": 0.0}
def compute_macro_f1(preds, targets, tier_map=None):
"""Tier-stratified macro-F1."""
from sklearn.metrics import f1_score, balanced_accuracy_score
overall = {
"macro_f1": float(f1_score(targets, preds, average='macro', zero_division=0)),
"top1_acc": float((preds == targets).mean()),
"balanced_acc": float(balanced_accuracy_score(targets, preds)),
}
if tier_map is None:
return overall
per_tier = {}
for tier in ['head','mid','tail','rare']:
mask = np.array([tier_map.get(t) == tier for t in targets])
if mask.sum() > 0:
per_tier[tier] = {
"n_samples": int(mask.sum()),
"macro_f1": float(f1_score(targets[mask], preds[mask], average='macro', zero_division=0)),
"top1_acc": float((preds[mask] == targets[mask]).mean()),
}
return {**overall, "per_tier": per_tier}
def compute_openset_auroc(in_energies, out_energies):
"""AUROC for OOD detection (in-dist vs oltr_holdout)."""
from sklearn.metrics import roc_auc_score
y = np.concatenate([np.zeros(len(in_energies)), np.ones(len(out_energies))])
scores = np.concatenate([in_energies, out_energies])
return float(roc_auc_score(y, scores))
def compute_cer(hypotheses, references):
"""Character Error Rate (Levenshtein-based)."""
import editdistance
total_errors = 0
total_chars = 0
for h, r in zip(hypotheses, references):
total_errors += editdistance.eval(h, r)
total_chars += len(r)
return total_errors / max(1, total_chars)
def evaluate(args):
results = {
"detection": {},
"classification": {},
"openset": {},
"transliteration": {},
"e2e_tablet": {},
"baselines": {},
}
# Gold set yükle
gold_records = []
with open(args.gold_set) as f:
for line in f:
gold_records.append(json.loads(line))
print(f"Gold set: {len(gold_records)} kayıt")
# Per-period breakdown için
period_breakdown = defaultdict(list)
lang_breakdown = defaultdict(list)
for r in gold_records:
period_breakdown[r.get('period', '?')].append(r)
lang_breakdown[r.get('language', '?')].append(r)
results['gold_set_breakdown'] = {
'per_period': {p: len(v) for p, v in period_breakdown.items()},
'per_language': {l: len(v) for l, v in lang_breakdown.items()},
}
# Hedefler tablosu (PIPELINE.md'den)
results['targets_vs_actual'] = {
'detection_mAP50_95': {'target': 0.80, 'deepscribe_baseline': 0.78},
'classification_macro_f1_head': {'target': 0.92},
'classification_macro_f1_mid': {'target': 0.75},
'classification_macro_f1_tail': {'target': 0.50},
'classification_macro_f1_rare': {'target': 0.25},
'openset_auroc': {'target': 0.85},
'cer_hit': {'target': 0.08, 'prep_ocr_baseline': 0.09},
'cer_akk': {'target': 0.09},
'cer_sum': {'target': 0.12},
'cer_elx': {'target': 0.10},
'churro_baseline_levenshtein': 0.823, # karşılaştırma
'deepscribe_top5': 0.80,
}
out = Path(args.output)
out.mkdir(parents=True, exist_ok=True)
with open(out / "evaluation_report.json", 'w') as f:
json.dump(results, f, indent=2, ensure_ascii=False)
print(f"Evaluation raporu: {out}/evaluation_report.json")
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--detection-ckpts', nargs='+', help='YOLO per-fold checkpoints')
ap.add_argument('--classifier-ckpt')
ap.add_argument('--translit-ckpt')
ap.add_argument('--gold-set', default='datasets/eval_gold/manifest.jsonl')
ap.add_argument('--output', default='hitit_ocr/runs/eval_e2e_v1/')
args = ap.parse_args()
evaluate(args)
if __name__ == '__main__':
main()
|