Buckets:
| """Step 3 OCR dataloader: (crop image, OCR text) -> BaberuOCRModel inputs. | |
| Produces ``pixel_values`` (DINOv2-preprocessed crop) + char-tokenized | |
| ``input_ids``/``labels`` for BaberuOCRModel.forward. The model prepends the | |
| vision prefix and computes CE loss on the text positions only (it slices out | |
| the vision prefix and shifts internally), so ``labels == input_ids`` with pad | |
| positions set to -100. | |
| """ | |
| from __future__ import annotations | |
| from functools import partial | |
| import torch | |
| from torch.utils.data import DataLoader, IterableDataset, get_worker_info | |
| from transformers import AutoImageProcessor | |
| from ocr_pairs import iter_ocr_pairs, load_text_index | |
| from tokenization_baberu import BaberuTokenizer | |
| class BaberuOCRDataset(IterableDataset): | |
| def __init__( | |
| self, | |
| crops_dir: str, | |
| text_index: dict, | |
| tokenizer: BaberuTokenizer, | |
| image_processor, | |
| max_text_len: int = 64, | |
| langs: set[str] | None = None, | |
| shuffle_buffer: int = 8192, | |
| seed: int = 42, | |
| ): | |
| self.crops_dir = crops_dir | |
| self.text_index = text_index | |
| self.tokenizer = tokenizer | |
| self.image_processor = image_processor | |
| self.max_text_len = max_text_len | |
| self.langs = langs | |
| self.shuffle_buffer = shuffle_buffer | |
| self.seed = seed | |
| self._epoch = 0 | |
| def __iter__(self): | |
| import random | |
| wi = get_worker_info() | |
| wid, nw = (0, 1) if wi is None else (wi.id, wi.num_workers) | |
| shard_filter = (lambda si: si % nw == wid) if nw > 1 else None | |
| # Per-epoch shard-order shuffle, SHARED across workers (no worker term) | |
| # so `si % nw == wid` stays a disjoint partition. The reservoir buffer | |
| # draw is worker-varied (it does not affect the partition). Together | |
| # they interleave the language-clustered shards so Step 3 sees a mixed | |
| # stream instead of an unintended language curriculum. | |
| shard_order_seed = self.seed + self._epoch | |
| rng = random.Random(self.seed + self._epoch * 7919 + wid + 1) | |
| self._epoch += 1 | |
| t2i = self.tokenizer._token_to_id | |
| unk = self.tokenizer.unk_token_id | |
| bos, eos = self.tokenizer.bos_token_id, self.tokenizer.eos_token_id | |
| def _emit(item): | |
| _cid, img, text, _lang = item | |
| pv = self.image_processor(img, return_tensors="pt")["pixel_values"][0] | |
| ids = [bos] + [t2i.get(c, unk) for c in text[: self.max_text_len]] + [eos] | |
| return {"pixel_values": pv, "input_ids": ids} | |
| stream = iter_ocr_pairs( | |
| self.crops_dir, self.text_index, langs=self.langs, | |
| shard_filter=shard_filter, shard_order_seed=shard_order_seed, | |
| ) | |
| n = self.shuffle_buffer | |
| if n and n > 1: | |
| buf = [] | |
| for item in stream: | |
| buf.append(item) | |
| if len(buf) >= n: | |
| j = rng.randrange(len(buf)) | |
| buf[j], buf[-1] = buf[-1], buf[j] | |
| yield _emit(buf.pop()) | |
| rng.shuffle(buf) | |
| for item in buf: | |
| yield _emit(item) | |
| else: | |
| for item in stream: | |
| yield _emit(item) | |
| def _collate(batch: list[dict], pad_id: int) -> dict[str, torch.Tensor]: | |
| maxlen = max(len(b["input_ids"]) for b in batch) | |
| n = len(batch) | |
| input_ids = torch.full((n, maxlen), pad_id, dtype=torch.long) | |
| labels = torch.full((n, maxlen), -100, dtype=torch.long) # ignore pad in loss | |
| attn = torch.zeros((n, maxlen), dtype=torch.long) | |
| for i, b in enumerate(batch): | |
| ids = torch.tensor(b["input_ids"], dtype=torch.long) | |
| m = len(ids) | |
| input_ids[i, :m] = ids | |
| labels[i, :m] = ids # model shifts internally | |
| attn[i, :m] = 1 | |
| pixel_values = torch.stack([b["pixel_values"] for b in batch]) | |
| return {"pixel_values": pixel_values, "input_ids": input_ids, | |
| "labels": labels, "attention_mask": attn} | |
| def build_ocr_image_processor(vision_model: str = "facebook/dinov2-base"): | |
| """Image processor for OCR crops — shared by training and eval so they match. | |
| CRITICAL: ``do_center_crop=False``. The default DINOv2 processor resizes the | |
| shortest edge to 256 then center-crops 224x224, which throws away the top and | |
| bottom of tall vertical 縦書き crops (77% of ja crops have h/w>1.5; the model | |
| only ever saw the middle ~20-50% of the text — see the aspect-vs-CER blow-up). | |
| Resize the whole crop to 224x224 instead (this is exactly what manga-ocr does), | |
| so every character is visible. | |
| """ | |
| return AutoImageProcessor.from_pretrained( | |
| vision_model, do_center_crop=False, size={"height": 224, "width": 224}, crop_size=None | |
| ) | |
| def build_ocr_dataloader( | |
| crops_dir: str, | |
| index_path: str, | |
| tokenizer: BaberuTokenizer, | |
| vision_model: str = "facebook/dinov2-base", | |
| batch_size: int = 64, | |
| num_workers: int = 4, | |
| max_text_len: int = 64, | |
| langs: set[str] | None = None, | |
| shuffle_buffer: int = 8192, | |
| seed: int = 42, | |
| ) -> DataLoader: | |
| image_processor = build_ocr_image_processor(vision_model) | |
| text_index = load_text_index(index_path) | |
| ds = BaberuOCRDataset( | |
| crops_dir, text_index, tokenizer, image_processor, max_text_len, langs, | |
| shuffle_buffer=shuffle_buffer, seed=seed, | |
| ) | |
| return DataLoader( | |
| ds, | |
| batch_size=batch_size, | |
| num_workers=num_workers, | |
| collate_fn=partial(_collate, pad_id=tokenizer.pad_token_id), | |
| pin_memory=True, | |
| prefetch_factor=4 if num_workers > 0 else None, | |
| persistent_workers=num_workers > 0, | |
| ) | |
Xet Storage Details
- Size:
- 5.66 kB
- Xet hash:
- 50e942538fb0054c2b561890dd4bdfb66274ea2c4b69de43c7d1700763801458
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.