File size: 3,483 Bytes
f211247 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | #!/usr/bin/env python3
"""Seq2seq training data builder.
hitit_local mark.txt'den tablet-level image-transliteration pair oluştur.
Her tablet için: image + sign sequence (reading order ile)
Output: datasets/processed/seq2seq_pairs.jsonl
"""
import json, os
from pathlib import Path
from collections import defaultdict
ROOT = Path("/arf/scratch/stakan/hitit-proje")
SOURCES = ROOT / "datasets" / "sources"
def extract_sign_sequence_from_mark(mark_path):
"""mark.txt JSON'dan tablet sign sequence'ı çıkar (reading order'da)."""
try:
data = json.load(open(mark_path))
except Exception:
return None
spots = data.get('spots', [])
if not spots:
return None
# Her spot: title = "<order>_<label>" (örn "1_AN", "2_me")
# y, x coordinates ile satır + sütun sıralaması
signs = []
for spot in spots:
title = spot.get('title', '')
# Parse order + label
if '_' in title:
order_str, label = title.split('_', 1)
try:
order = int(order_str)
except:
order = 0
else:
order = 0
label = title
signs.append({
'order': order,
'label': label.strip(),
'x': spot.get('x', 0),
'y': spot.get('y', 0),
'w': spot.get('width', 0),
'h': spot.get('height', 0),
})
# Reading order: önce satır (y), sonra x
# Ama 'order' zaten verilmiş
signs.sort(key=lambda s: s['order'])
return signs
def build_pairs():
"""hitit_local klasöründeki her tablet için pair oluştur."""
pairs = []
hitit_root = SOURCES / "hitit_local"
for folder in sorted(hitit_root.iterdir()):
if not folder.is_dir(): continue
mark_path = folder / "mark.txt"
if not mark_path.exists(): continue
# Image: klasördeki ilk image
images = [f for f in folder.iterdir()
if f.suffix.lower() in {'.jpg', '.jpeg', '.png'}]
if not images: continue
image_path = images[0]
signs = extract_sign_sequence_from_mark(mark_path)
if not signs: continue
# Sequence string: "AN me-na-aḫ-ḫa-an-da ki-iš-..."
# Simple join with space; advanced: cuneiform Unicode output
sequence_plain = " ".join(s['label'] for s in signs)
sequence_abz = " ".join(s['label'].upper() for s in signs)
pairs.append({
'tablet_id': folder.name,
'image_path': str(image_path),
'n_signs': len(signs),
'signs': signs,
'sequence_plain': sequence_plain,
'sequence_abz': sequence_abz,
})
# Output
out = ROOT / "datasets/processed/seq2seq_pairs.jsonl"
with open(out, 'w') as f:
for p in pairs:
f.write(json.dumps(p, ensure_ascii=False) + '\n')
# Summary
n = len(pairs)
n_signs_total = sum(p['n_signs'] for p in pairs)
print(f"Pairs: {n} tablet")
print(f"Toplam sign: {n_signs_total:,}")
print(f"Ortalama: {n_signs_total/max(1,n):.0f} sign/tablet")
print(f"Output: {out}")
# Uzunluk dağılımı
lens = sorted([p['n_signs'] for p in pairs])
if lens:
print(f"Min: {lens[0]}, median: {lens[len(lens)//2]}, max: {lens[-1]}")
return pairs
if __name__ == '__main__':
build_pairs()
|