podman4's picture
download
raw
5.66 kB
"""Step 3 OCR dataloader: (crop image, OCR text) -> BaberuOCRModel inputs.
Produces ``pixel_values`` (DINOv2-preprocessed crop) + char-tokenized
``input_ids``/``labels`` for BaberuOCRModel.forward. The model prepends the
vision prefix and computes CE loss on the text positions only (it slices out
the vision prefix and shifts internally), so ``labels == input_ids`` with pad
positions set to -100.
"""
from __future__ import annotations
from functools import partial
import torch
from torch.utils.data import DataLoader, IterableDataset, get_worker_info
from transformers import AutoImageProcessor
from ocr_pairs import iter_ocr_pairs, load_text_index
from tokenization_baberu import BaberuTokenizer
class BaberuOCRDataset(IterableDataset):
def __init__(
self,
crops_dir: str,
text_index: dict,
tokenizer: BaberuTokenizer,
image_processor,
max_text_len: int = 64,
langs: set[str] | None = None,
shuffle_buffer: int = 8192,
seed: int = 42,
):
self.crops_dir = crops_dir
self.text_index = text_index
self.tokenizer = tokenizer
self.image_processor = image_processor
self.max_text_len = max_text_len
self.langs = langs
self.shuffle_buffer = shuffle_buffer
self.seed = seed
self._epoch = 0
def __iter__(self):
import random
wi = get_worker_info()
wid, nw = (0, 1) if wi is None else (wi.id, wi.num_workers)
shard_filter = (lambda si: si % nw == wid) if nw > 1 else None
# Per-epoch shard-order shuffle, SHARED across workers (no worker term)
# so `si % nw == wid` stays a disjoint partition. The reservoir buffer
# draw is worker-varied (it does not affect the partition). Together
# they interleave the language-clustered shards so Step 3 sees a mixed
# stream instead of an unintended language curriculum.
shard_order_seed = self.seed + self._epoch
rng = random.Random(self.seed + self._epoch * 7919 + wid + 1)
self._epoch += 1
t2i = self.tokenizer._token_to_id
unk = self.tokenizer.unk_token_id
bos, eos = self.tokenizer.bos_token_id, self.tokenizer.eos_token_id
def _emit(item):
_cid, img, text, _lang = item
pv = self.image_processor(img, return_tensors="pt")["pixel_values"][0]
ids = [bos] + [t2i.get(c, unk) for c in text[: self.max_text_len]] + [eos]
return {"pixel_values": pv, "input_ids": ids}
stream = iter_ocr_pairs(
self.crops_dir, self.text_index, langs=self.langs,
shard_filter=shard_filter, shard_order_seed=shard_order_seed,
)
n = self.shuffle_buffer
if n and n > 1:
buf = []
for item in stream:
buf.append(item)
if len(buf) >= n:
j = rng.randrange(len(buf))
buf[j], buf[-1] = buf[-1], buf[j]
yield _emit(buf.pop())
rng.shuffle(buf)
for item in buf:
yield _emit(item)
else:
for item in stream:
yield _emit(item)
def _collate(batch: list[dict], pad_id: int) -> dict[str, torch.Tensor]:
maxlen = max(len(b["input_ids"]) for b in batch)
n = len(batch)
input_ids = torch.full((n, maxlen), pad_id, dtype=torch.long)
labels = torch.full((n, maxlen), -100, dtype=torch.long) # ignore pad in loss
attn = torch.zeros((n, maxlen), dtype=torch.long)
for i, b in enumerate(batch):
ids = torch.tensor(b["input_ids"], dtype=torch.long)
m = len(ids)
input_ids[i, :m] = ids
labels[i, :m] = ids # model shifts internally
attn[i, :m] = 1
pixel_values = torch.stack([b["pixel_values"] for b in batch])
return {"pixel_values": pixel_values, "input_ids": input_ids,
"labels": labels, "attention_mask": attn}
def build_ocr_image_processor(vision_model: str = "facebook/dinov2-base"):
"""Image processor for OCR crops — shared by training and eval so they match.
CRITICAL: ``do_center_crop=False``. The default DINOv2 processor resizes the
shortest edge to 256 then center-crops 224x224, which throws away the top and
bottom of tall vertical 縦書き crops (77% of ja crops have h/w>1.5; the model
only ever saw the middle ~20-50% of the text — see the aspect-vs-CER blow-up).
Resize the whole crop to 224x224 instead (this is exactly what manga-ocr does),
so every character is visible.
"""
return AutoImageProcessor.from_pretrained(
vision_model, do_center_crop=False, size={"height": 224, "width": 224}, crop_size=None
)
def build_ocr_dataloader(
crops_dir: str,
index_path: str,
tokenizer: BaberuTokenizer,
vision_model: str = "facebook/dinov2-base",
batch_size: int = 64,
num_workers: int = 4,
max_text_len: int = 64,
langs: set[str] | None = None,
shuffle_buffer: int = 8192,
seed: int = 42,
) -> DataLoader:
image_processor = build_ocr_image_processor(vision_model)
text_index = load_text_index(index_path)
ds = BaberuOCRDataset(
crops_dir, text_index, tokenizer, image_processor, max_text_len, langs,
shuffle_buffer=shuffle_buffer, seed=seed,
)
return DataLoader(
ds,
batch_size=batch_size,
num_workers=num_workers,
collate_fn=partial(_collate, pad_id=tokenizer.pad_token_id),
pin_memory=True,
prefetch_factor=4 if num_workers > 0 else None,
persistent_workers=num_workers > 0,
)

Xet Storage Details

Size:
5.66 kB
·
Xet hash:
50e942538fb0054c2b561890dd4bdfb66274ea2c4b69de43c7d1700763801458

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.