| |
| """Seq2seq training data builder. |
| |
| hitit_local mark.txt'den tablet-level image-transliteration pair oluştur. |
| Her tablet için: image + sign sequence (reading order ile) |
| |
| Output: datasets/processed/seq2seq_pairs.jsonl |
| """ |
| import json, os |
| from pathlib import Path |
| from collections import defaultdict |
|
|
| ROOT = Path("/arf/scratch/stakan/hitit-proje") |
| SOURCES = ROOT / "datasets" / "sources" |
|
|
| def extract_sign_sequence_from_mark(mark_path): |
| """mark.txt JSON'dan tablet sign sequence'ı çıkar (reading order'da).""" |
| try: |
| data = json.load(open(mark_path)) |
| except Exception: |
| return None |
| |
| spots = data.get('spots', []) |
| if not spots: |
| return None |
| |
| |
| |
| signs = [] |
| for spot in spots: |
| title = spot.get('title', '') |
| |
| if '_' in title: |
| order_str, label = title.split('_', 1) |
| try: |
| order = int(order_str) |
| except: |
| order = 0 |
| else: |
| order = 0 |
| label = title |
| |
| signs.append({ |
| 'order': order, |
| 'label': label.strip(), |
| 'x': spot.get('x', 0), |
| 'y': spot.get('y', 0), |
| 'w': spot.get('width', 0), |
| 'h': spot.get('height', 0), |
| }) |
| |
| |
| |
| signs.sort(key=lambda s: s['order']) |
| |
| return signs |
|
|
| def build_pairs(): |
| """hitit_local klasöründeki her tablet için pair oluştur.""" |
| pairs = [] |
| hitit_root = SOURCES / "hitit_local" |
| |
| for folder in sorted(hitit_root.iterdir()): |
| if not folder.is_dir(): continue |
| |
| mark_path = folder / "mark.txt" |
| if not mark_path.exists(): continue |
| |
| |
| images = [f for f in folder.iterdir() |
| if f.suffix.lower() in {'.jpg', '.jpeg', '.png'}] |
| if not images: continue |
| image_path = images[0] |
| |
| signs = extract_sign_sequence_from_mark(mark_path) |
| if not signs: continue |
| |
| |
| |
| sequence_plain = " ".join(s['label'] for s in signs) |
| sequence_abz = " ".join(s['label'].upper() for s in signs) |
| |
| pairs.append({ |
| 'tablet_id': folder.name, |
| 'image_path': str(image_path), |
| 'n_signs': len(signs), |
| 'signs': signs, |
| 'sequence_plain': sequence_plain, |
| 'sequence_abz': sequence_abz, |
| }) |
| |
| |
| out = ROOT / "datasets/processed/seq2seq_pairs.jsonl" |
| with open(out, 'w') as f: |
| for p in pairs: |
| f.write(json.dumps(p, ensure_ascii=False) + '\n') |
| |
| |
| n = len(pairs) |
| n_signs_total = sum(p['n_signs'] for p in pairs) |
| print(f"Pairs: {n} tablet") |
| print(f"Toplam sign: {n_signs_total:,}") |
| print(f"Ortalama: {n_signs_total/max(1,n):.0f} sign/tablet") |
| print(f"Output: {out}") |
| |
| |
| lens = sorted([p['n_signs'] for p in pairs]) |
| if lens: |
| print(f"Min: {lens[0]}, median: {lens[len(lens)//2]}, max: {lens[-1]}") |
| |
| return pairs |
|
|
| if __name__ == '__main__': |
| build_pairs() |
|
|
