File size: 2,966 Bytes
f211247 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | #!/usr/bin/env python3
"""Image → Transliteration seq2seq model.
Reference: SumTablets Simmons 2024, HATFormer 2024.
Architecture: ViT encoder + ByT5 decoder + cross-attention.
"""
import torch
import torch.nn as nn
from pathlib import Path
class VisionByT5(nn.Module):
"""ViT encoder + ByT5 decoder for cuneiform transliteration."""
def __init__(self, encoder_name="facebook/dinov2-large",
decoder_name="google/byt5-small",
image_size=384):
super().__init__()
# Encoder (lazy import — transformers library)
from transformers import AutoModel, T5ForConditionalGeneration, AutoTokenizer
self.encoder = AutoModel.from_pretrained(encoder_name)
self.decoder = T5ForConditionalGeneration.from_pretrained(decoder_name)
self.tokenizer = AutoTokenizer.from_pretrained(decoder_name)
# Projection: encoder hidden → decoder hidden
enc_dim = self.encoder.config.hidden_size
dec_dim = self.decoder.config.d_model
self.enc_to_dec_proj = nn.Linear(enc_dim, dec_dim)
# LoRA adapt encoder
self._apply_lora(self.encoder, r=32, alpha=64)
def _apply_lora(self, module, r=32, alpha=64):
try:
from peft import LoraConfig, get_peft_model
config = LoraConfig(
r=r, lora_alpha=alpha,
target_modules=["query", "key", "value", "dense"],
lora_dropout=0.1, bias="none"
)
module = get_peft_model(module, config)
except ImportError:
print("peft not installed; LoRA skipped", flush=True)
def forward(self, pixel_values, labels=None):
"""
pixel_values: (B, 3, H, W)
labels: (B, T) — target byte sequence
"""
# Encode image
enc_out = self.encoder(pixel_values=pixel_values)
enc_hidden = enc_out.last_hidden_state # (B, N_patches, enc_dim)
# Project to decoder dim
enc_hidden = self.enc_to_dec_proj(enc_hidden)
# Decode
outputs = self.decoder(
encoder_outputs=(enc_hidden,),
labels=labels,
)
return outputs # loss + logits
@torch.no_grad()
def generate(self, pixel_values, max_length=512, num_beams=5):
enc_out = self.encoder(pixel_values=pixel_values)
enc_hidden = self.enc_to_dec_proj(enc_out.last_hidden_state)
return self.decoder.generate(
encoder_outputs=(enc_hidden,),
max_length=max_length,
num_beams=num_beams,
length_penalty=0.6,
no_repeat_ngram_size=3,
early_stopping=True,
)
if __name__ == '__main__':
print("Vision-ByT5 seq2seq model architecture template hazır.")
print("Training data: tablet image + transliteration pair")
print("Expected chrF (SumTablets baseline): 0.9755")
|