"""Streaming data pipeline, filtering, caption mixing and batching. Nothing in this module downloads the dataset to disk. Hugging Face's streaming Parquet reader is used, and each 16x16 PNG is decoded in memory and mapped to the native 4,097-colour palette. """ from __future__ import annotations import hashlib import io import random import re from dataclasses import dataclass, field from typing import Dict, Iterable, Iterator, List, Optional, Sequence, Tuple import numpy as np from .arabic import CuratedArabicStore, make_arabic_store from .config import DATASET_ID, IMAGE_SIZE, PIXELS_PER_IMAGE, TEXT_MAX_LENGTH, DataConfig from .palette import Palette from .text import BPETokenizer from .utils import prefetch_iter _ALNUM = re.compile(r"[A-Za-z0-9\u0600-\u06FF]") def as_pil_image(image): """Normalise the many ways HF can hand back an image to a PIL RGBA image.""" if image is None: return None if hasattr(image, "convert"): return image.convert("RGBA") if isinstance(image, dict): try: from PIL import Image if image.get("bytes"): return Image.open(io.BytesIO(image["bytes"])).convert("RGBA") if image.get("path"): return Image.open(image["path"]).convert("RGBA") except Exception: return None return None # --------------------------------------------------------------------------- # Stats / filtering # --------------------------------------------------------------------------- @dataclass class DatasetStats: scanned: int = 0 accepted: int = 0 empty: int = 0 too_few_opaque: int = 0 too_few_colors: int = 0 bad_caption: int = 0 duplicate: int = 0 wrong_size: int = 0 palette_missing: int = 0 held_out: int = 0 def report(self) -> str: return ( f"scanned={self.scanned:,} accepted={self.accepted:,} | " f"empty={self.empty:,} few_opaque={self.too_few_opaque:,} " f"few_colors={self.too_few_colors:,} bad_caption={self.bad_caption:,} " f"duplicate={self.duplicate:,} wrong_size={self.wrong_size:,} " f"palette_missing={self.palette_missing:,} held_out={self.held_out:,}" ) _LOW_INFO = {"object", "item", "thing", "stuff", "unknown", "n/a", "none", "sprite", "?"} def is_low_information(caption: str) -> bool: c = caption.strip().lower() if not c or c in _LOW_INFO: return True if len(c) < 2: return True if not _ALNUM.search(c): return True return False def subject_bucket(subject: str) -> int: h = hashlib.blake2b(subject.strip().lower().encode("utf-8"), digest_size=4).digest() return int.from_bytes(h, "big") % 100 def keep_subject_for_split(subject: str, holdout_pct: float, target: str) -> bool: if holdout_pct <= 0: return True bucket = subject_bucket(subject) threshold = int(round(holdout_pct * 100)) if target == "train": return bucket >= threshold return bucket < threshold # --------------------------------------------------------------------------- # Streaming source # --------------------------------------------------------------------------- def stream_dataset( split: str, palette: Palette, data_config: DataConfig, source_split: Optional[str] = None, stats: Optional[DatasetStats] = None, max_examples: Optional[int] = None, ) -> Iterator[dict]: """Yield filtered examples with 256 native-palette tokens. Args: split: "train" or "validation"; controls subject-holdout filtering. source_split: the actual HF split to read. Defaults to ``split``. For a subject-disjoint validation set, pass source_split="train" and split="validation". """ from datasets import load_dataset stats = stats or DatasetStats() source_split = source_split or split ds = load_dataset(data_config.dataset_id, split=source_split, streaming=True) sharded = False if data_config.world_size > 1: try: ds = ds.shard(num_shards=data_config.world_size, index=data_config.rank) sharded = True except Exception: # Older datasets versions: fall back to row_id filtering below. pass if source_split == "train" and data_config.shuffle_buffer > 0: ds = ds.shuffle(seed=data_config.seed, buffer_size=data_config.shuffle_buffer) ds = prefetch_iter(ds, buffer_size=data_config.prefetch) seen_hashes: set[int] = set() accepted = 0 for example in ds: stats.scanned += 1 image = as_pil_image(example.get("image")) if image is None: stats.wrong_size += 1 continue if image.size != (IMAGE_SIZE, IMAGE_SIZE): stats.wrong_size += 1 continue row_id = int(example.get("row_id", -1)) if data_config.world_size > 1 and not sharded and row_id >= 0: if row_id % data_config.world_size != data_config.rank: continue caption = str(example.get("caption", "")).strip() subject = str(example.get("subject", "")).strip() or caption if not keep_subject_for_split(subject, data_config.subject_holdout, split): stats.held_out += 1 continue if not caption or len(caption) < data_config.min_caption_chars or len(caption) > data_config.max_caption_chars: stats.bad_caption += 1 continue if is_low_information(caption) or is_low_information(subject): stats.bad_caption += 1 continue tokens, missing = palette.from_pil(image) if missing: stats.palette_missing += missing opaque = int((tokens != 0).sum()) if opaque == 0: stats.empty += 1 continue if opaque < data_config.min_opaque_pixels: stats.too_few_opaque += 1 continue unique = int(len(np.unique(tokens))) - (1 if (tokens == 0).any() else 0) if unique < data_config.min_unique_colors: stats.too_few_colors += 1 continue if data_config.dedupe: h = hashlib.blake2b( tokens.tobytes() + b"|" + caption.encode("utf-8"), digest_size=8 ).digest() key = int.from_bytes(h, "big") if key in seen_hashes: stats.duplicate += 1 continue if len(seen_hashes) < data_config.max_dedupe_entries: seen_hashes.add(key) stats.accepted += 1 accepted += 1 yield { "tokens": tokens.astype(np.uint16, copy=False), "caption": caption, "subject": subject, "row_id": int(example.get("row_id", -1)), "opaque": opaque, "unique_colors": unique, } if max_examples is not None and accepted >= max_examples: break # --------------------------------------------------------------------------- # Caption mixing / batching # --------------------------------------------------------------------------- class CaptionMixer: def __init__( self, arabic_store=None, p_arabic: float = 0.20, p_egyptian: float = 0.20, p_mixed: float = 0.05, p_subject: float = 0.40, seed: int = 42, ): self.arabic_store = arabic_store self.p_arabic = p_arabic self.p_egyptian = p_egyptian self.p_mixed = p_mixed self.p_subject = p_subject self.rng = random.Random(seed) def _english(self, example: dict, rng: random.Random) -> str: caption = example["caption"] subject = example["subject"] if subject and rng.random() < self.p_subject: return subject return caption def build(self, example: dict, rng: random.Random, force_language: Optional[str] = None) -> Tuple[str, str]: english = self._english(example, rng) if force_language == "en": return english, "en" arabic = None if self.arabic_store is not None: arabic = self.arabic_store.get( example["caption"], example["subject"], dialect="msa", salt=str(rng.random()) ) if force_language == "ar": return (arabic, "ar") if arabic else (english, "en") if arabic and rng.random() < self.p_arabic: lang = "ar" if rng.random() < self.p_egyptian: eg = self.arabic_store.get( example["caption"], example["subject"], dialect="egyptian", salt=str(rng.random()), ) if eg: arabic = eg lang = "ar-eg" text = arabic else: text = english lang = "en" if self.arabic_store is not None and rng.random() < self.p_mixed: text = self.arabic_store.mixed(english, rng=rng) lang = "mixed" return text, lang def pair(self, example: dict) -> Optional[Tuple[str, str]]: if self.arabic_store is None: return None return self.arabic_store.get_pair(example["caption"], example["subject"]) def build_batch( examples: Sequence[dict], tokenizer: BPETokenizer, mixer: CaptionMixer, rng: random.Random, force_language: Optional[str] = None, ) -> dict: texts: List[str] = [] langs: List[str] = [] en_texts: List[str] = [] ar_texts: List[str] = [] has_ar: List[bool] = [] for example in examples: text, lang = mixer.build(example, rng, force_language=force_language) texts.append(text) langs.append(lang) pair = mixer.pair(example) if pair is None: en_texts.append(example["caption"]) ar_texts.append("") has_ar.append(False) else: en, ar = pair en_texts.append(en) ar_texts.append(ar) has_ar.append(True) gen_ids, gen_mask = tokenizer.encode_batch(texts, max_length=tokenizer.max_length) en_ids, en_mask = tokenizer.encode_batch(en_texts, max_length=tokenizer.max_length) # Tokenizing empty strings gives a single [UNK]; the mask is unused for # examples without a pair, so this is harmless. ar_ids, ar_mask = tokenizer.encode_batch(ar_texts, max_length=tokenizer.max_length) tokens = np.stack([ np.asarray(e["tokens"], dtype=np.int64).reshape(-1) for e in examples ]) return { "tokens": tokens, "text_ids": gen_ids, "text_mask": gen_mask, "en_ids": en_ids, "en_mask": en_mask, "ar_ids": ar_ids, "ar_mask": ar_mask, "has_ar": np.asarray(has_ar, dtype=np.bool_), "langs": langs, "texts": texts, } def to_torch_batch(batch: dict, device) -> dict: import torch out = {} for key in ["tokens", "text_ids", "text_mask", "en_ids", "en_mask", "ar_ids", "ar_mask", "has_ar"]: value = batch[key] if key in ("text_mask", "en_mask", "ar_mask", "has_ar"): out[key] = torch.as_tensor(value, dtype=torch.bool, device=device) else: out[key] = torch.as_tensor(value, dtype=torch.long, device=device) out["langs"] = batch["langs"] out["texts"] = batch["texts"] return out # --------------------------------------------------------------------------- # Batch sources # --------------------------------------------------------------------------- class ListBatchSource: """In-memory batches for feasibility experiments and validation.""" def __init__( self, examples: Sequence[dict], tokenizer: BPETokenizer, mixer: CaptionMixer, batch_size: int, seed: int = 42, infinite: bool = True, force_language: Optional[str] = None, rank: int = 0, world_size: int = 1, ): self.examples = list(examples) self.rank = rank self.world_size = world_size self.tokenizer = tokenizer self.mixer = mixer self.batch_size = batch_size self.seed = seed self.infinite = infinite self.force_language = force_language self.rng = random.Random(seed) def _one_epoch(self) -> Iterator[dict]: order = [ i for i in range(len(self.examples)) if i % max(1, self.world_size) == self.rank ] self.rng.shuffle(order) for start in range(0, len(order), self.batch_size): idx = order[start : start + self.batch_size] batch = [self.examples[i] for i in idx] if self.force_language is not None: # Build with a forced language where possible. kept = [] for ex in batch: text, lang = self.mixer.build(ex, self.rng, force_language=self.force_language) if lang == self.force_language: kept.append(ex) if not kept: continue batch = kept yield build_batch( batch, self.tokenizer, self.mixer, self.rng, force_language=self.force_language, ) def __iter__(self) -> Iterator[dict]: if not self.examples: raise ValueError("ListBatchSource received no examples.") while True: yielded = False for batch in self._one_epoch(): yielded = True yield batch if not self.infinite: break if not yielded: break def steps_per_epoch(self) -> int: return max(1, len(self.examples) // self.batch_size) class StreamingBatchSource: """Infinite batches from the Hugging Face streaming dataset.""" def __init__( self, palette: Palette, tokenizer: BPETokenizer, mixer: CaptionMixer, data_config: DataConfig, batch_size: int, split: str = "train", source_split: Optional[str] = None, seed: int = 42, max_examples_per_epoch: Optional[int] = None, force_language: Optional[str] = None, ): self.palette = palette self.tokenizer = tokenizer self.mixer = mixer self.data_config = data_config self.batch_size = batch_size self.split = split self.source_split = source_split self.seed = seed self.max_examples_per_epoch = max_examples_per_epoch self.force_language = force_language self.rng = random.Random(seed) self.epoch = 0 def _one_epoch(self) -> Iterator[dict]: stats = DatasetStats() buffer: List[dict] = [] for example in stream_dataset( self.split, self.palette, self.data_config, source_split=self.source_split, stats=stats, max_examples=self.max_examples_per_epoch, ): buffer.append(example) if len(buffer) >= self.batch_size: if self.force_language is not None: kept = [] for ex in buffer: _, lang = self.mixer.build(ex, self.rng, force_language=self.force_language) if lang == self.force_language: kept.append(ex) buffer = kept if buffer: yield build_batch( buffer, self.tokenizer, self.mixer, self.rng, force_language=self.force_language, ) buffer = [] if buffer: yield build_batch( buffer, self.tokenizer, self.mixer, self.rng, force_language=self.force_language, ) print(f"[data] epoch {self.epoch} stats: {stats.report()}") self.epoch += 1 def __iter__(self) -> Iterator[dict]: while True: yielded = False for batch in self._one_epoch(): yielded = True yield batch if not yielded: raise RuntimeError("The streaming dataset produced no accepted examples.") # --------------------------------------------------------------------------- # Convenience collectors # --------------------------------------------------------------------------- def collect_captions( n: int = 5_000, split: str = "train", data_config: Optional[DataConfig] = None, include_subjects: bool = True, ) -> List[str]: """Collect unique captions without decoding images (for tokenizer training).""" from datasets import load_dataset data_config = data_config or DataConfig() ds = load_dataset(data_config.dataset_id, split=split, streaming=True) try: ds = ds.select_columns(["caption", "subject"] if include_subjects else ["caption"]) except Exception: pass ds = prefetch_iter(ds, buffer_size=data_config.prefetch) texts: List[str] = [] seen = set() for ex in ds: for key in (["caption", "subject"] if include_subjects else ["caption"]): text = str(ex.get(key, "")).strip() if not text or text in seen or is_low_information(text): continue seen.add(text) texts.append(text) if len(texts) >= n: return texts return texts def collect_examples( palette: Palette, n: int, data_config: Optional[DataConfig] = None, split: str = "train", source_split: Optional[str] = None, arabic_store=None, require_arabic: bool = False, seed: int = 42, progress_every: Optional[int] = None, ) -> List[dict]: """Collect a small in-memory subset for feasibility runs / validation.""" data_config = data_config or DataConfig(seed=seed) examples: List[dict] = [] progress_every = progress_every or (250 if require_arabic else 2_000) for ex in stream_dataset( split, palette, data_config, source_split=source_split, max_examples=None, ): if require_arabic: if arabic_store is None: raise ValueError("require_arabic=True needs an Arabic store.") ar = arabic_store.get(ex["caption"], ex["subject"]) if ar is None: continue examples.append(ex) if len(examples) % progress_every == 0: print(f" collected {len(examples):,}/{n:,}...") if len(examples) >= n: break return examples def make_mixer( arabic_mode: str = "curated", arabic_cache: Optional[str] = None, p_arabic: float = 0.20, p_egyptian: float = 0.20, p_mixed: float = 0.05, p_subject: float = 0.40, seed: int = 42, ) -> CaptionMixer: store = make_arabic_store(arabic_mode, arabic_cache) return CaptionMixer( arabic_store=store, p_arabic=p_arabic, p_egyptian=p_egyptian, p_mixed=p_mixed, p_subject=p_subject, seed=seed, )