belal212's picture
final compact model + model card
ee385a9 verified
Raw History Blame Contribute Delete
19.2 kB
"""Streaming data pipeline, filtering, caption mixing and batching.
Nothing in this module downloads the dataset to disk. Hugging Face's streaming
Parquet reader is used, and each 16x16 PNG is decoded in memory and mapped to
the native 4,097-colour palette.
"""
from __future__ import annotations
import hashlib
import io
import random
import re
from dataclasses import dataclass, field
from typing import Dict, Iterable, Iterator, List, Optional, Sequence, Tuple
import numpy as np
from .arabic import CuratedArabicStore, make_arabic_store
from .config import DATASET_ID, IMAGE_SIZE, PIXELS_PER_IMAGE, TEXT_MAX_LENGTH, DataConfig
from .palette import Palette
from .text import BPETokenizer
from .utils import prefetch_iter
_ALNUM = re.compile(r"[A-Za-z0-9\u0600-\u06FF]")
def as_pil_image(image):
"""Normalise the many ways HF can hand back an image to a PIL RGBA image."""
if image is None:
return None
if hasattr(image, "convert"):
return image.convert("RGBA")
if isinstance(image, dict):
try:
from PIL import Image
if image.get("bytes"):
return Image.open(io.BytesIO(image["bytes"])).convert("RGBA")
if image.get("path"):
return Image.open(image["path"]).convert("RGBA")
except Exception:
return None
return None
# ---------------------------------------------------------------------------
# Stats / filtering
# ---------------------------------------------------------------------------
@dataclass
class DatasetStats:
scanned: int = 0
accepted: int = 0
empty: int = 0
too_few_opaque: int = 0
too_few_colors: int = 0
bad_caption: int = 0
duplicate: int = 0
wrong_size: int = 0
palette_missing: int = 0
held_out: int = 0
def report(self) -> str:
return (
f"scanned={self.scanned:,} accepted={self.accepted:,} | "
f"empty={self.empty:,} few_opaque={self.too_few_opaque:,} "
f"few_colors={self.too_few_colors:,} bad_caption={self.bad_caption:,} "
f"duplicate={self.duplicate:,} wrong_size={self.wrong_size:,} "
f"palette_missing={self.palette_missing:,} held_out={self.held_out:,}"
)
_LOW_INFO = {"object", "item", "thing", "stuff", "unknown", "n/a", "none", "sprite", "?"}
def is_low_information(caption: str) -> bool:
c = caption.strip().lower()
if not c or c in _LOW_INFO:
return True
if len(c) < 2:
return True
if not _ALNUM.search(c):
return True
return False
def subject_bucket(subject: str) -> int:
h = hashlib.blake2b(subject.strip().lower().encode("utf-8"), digest_size=4).digest()
return int.from_bytes(h, "big") % 100
def keep_subject_for_split(subject: str, holdout_pct: float, target: str) -> bool:
if holdout_pct <= 0:
return True
bucket = subject_bucket(subject)
threshold = int(round(holdout_pct * 100))
if target == "train":
return bucket >= threshold
return bucket < threshold
# ---------------------------------------------------------------------------
# Streaming source
# ---------------------------------------------------------------------------
def stream_dataset(
split: str,
palette: Palette,
data_config: DataConfig,
source_split: Optional[str] = None,
stats: Optional[DatasetStats] = None,
max_examples: Optional[int] = None,
) -> Iterator[dict]:
"""Yield filtered examples with 256 native-palette tokens.
Args:
split: "train" or "validation"; controls subject-holdout filtering.
source_split: the actual HF split to read. Defaults to ``split``.
For a subject-disjoint validation set, pass source_split="train"
and split="validation".
"""
from datasets import load_dataset
stats = stats or DatasetStats()
source_split = source_split or split
ds = load_dataset(data_config.dataset_id, split=source_split, streaming=True)
sharded = False
if data_config.world_size > 1:
try:
ds = ds.shard(num_shards=data_config.world_size, index=data_config.rank)
sharded = True
except Exception:
# Older datasets versions: fall back to row_id filtering below.
pass
if source_split == "train" and data_config.shuffle_buffer > 0:
ds = ds.shuffle(seed=data_config.seed, buffer_size=data_config.shuffle_buffer)
ds = prefetch_iter(ds, buffer_size=data_config.prefetch)
seen_hashes: set[int] = set()
accepted = 0
for example in ds:
stats.scanned += 1
image = as_pil_image(example.get("image"))
if image is None:
stats.wrong_size += 1
continue
if image.size != (IMAGE_SIZE, IMAGE_SIZE):
stats.wrong_size += 1
continue
row_id = int(example.get("row_id", -1))
if data_config.world_size > 1 and not sharded and row_id >= 0:
if row_id % data_config.world_size != data_config.rank:
continue
caption = str(example.get("caption", "")).strip()
subject = str(example.get("subject", "")).strip() or caption
if not keep_subject_for_split(subject, data_config.subject_holdout, split):
stats.held_out += 1
continue
if not caption or len(caption) < data_config.min_caption_chars or len(caption) > data_config.max_caption_chars:
stats.bad_caption += 1
continue
if is_low_information(caption) or is_low_information(subject):
stats.bad_caption += 1
continue
tokens, missing = palette.from_pil(image)
if missing:
stats.palette_missing += missing
opaque = int((tokens != 0).sum())
if opaque == 0:
stats.empty += 1
continue
if opaque < data_config.min_opaque_pixels:
stats.too_few_opaque += 1
continue
unique = int(len(np.unique(tokens))) - (1 if (tokens == 0).any() else 0)
if unique < data_config.min_unique_colors:
stats.too_few_colors += 1
continue
if data_config.dedupe:
h = hashlib.blake2b(
tokens.tobytes() + b"|" + caption.encode("utf-8"), digest_size=8
).digest()
key = int.from_bytes(h, "big")
if key in seen_hashes:
stats.duplicate += 1
continue
if len(seen_hashes) < data_config.max_dedupe_entries:
seen_hashes.add(key)
stats.accepted += 1
accepted += 1
yield {
"tokens": tokens.astype(np.uint16, copy=False),
"caption": caption,
"subject": subject,
"row_id": int(example.get("row_id", -1)),
"opaque": opaque,
"unique_colors": unique,
}
if max_examples is not None and accepted >= max_examples:
break
# ---------------------------------------------------------------------------
# Caption mixing / batching
# ---------------------------------------------------------------------------
class CaptionMixer:
def __init__(
self,
arabic_store=None,
p_arabic: float = 0.20,
p_egyptian: float = 0.20,
p_mixed: float = 0.05,
p_subject: float = 0.40,
seed: int = 42,
):
self.arabic_store = arabic_store
self.p_arabic = p_arabic
self.p_egyptian = p_egyptian
self.p_mixed = p_mixed
self.p_subject = p_subject
self.rng = random.Random(seed)
def _english(self, example: dict, rng: random.Random) -> str:
caption = example["caption"]
subject = example["subject"]
if subject and rng.random() < self.p_subject:
return subject
return caption
def build(self, example: dict, rng: random.Random, force_language: Optional[str] = None) -> Tuple[str, str]:
english = self._english(example, rng)
if force_language == "en":
return english, "en"
arabic = None
if self.arabic_store is not None:
arabic = self.arabic_store.get(
example["caption"], example["subject"], dialect="msa", salt=str(rng.random())
)
if force_language == "ar":
return (arabic, "ar") if arabic else (english, "en")
if arabic and rng.random() < self.p_arabic:
lang = "ar"
if rng.random() < self.p_egyptian:
eg = self.arabic_store.get(
example["caption"], example["subject"], dialect="egyptian",
salt=str(rng.random()),
)
if eg:
arabic = eg
lang = "ar-eg"
text = arabic
else:
text = english
lang = "en"
if self.arabic_store is not None and rng.random() < self.p_mixed:
text = self.arabic_store.mixed(english, rng=rng)
lang = "mixed"
return text, lang
def pair(self, example: dict) -> Optional[Tuple[str, str]]:
if self.arabic_store is None:
return None
return self.arabic_store.get_pair(example["caption"], example["subject"])
def build_batch(
examples: Sequence[dict],
tokenizer: BPETokenizer,
mixer: CaptionMixer,
rng: random.Random,
force_language: Optional[str] = None,
) -> dict:
texts: List[str] = []
langs: List[str] = []
en_texts: List[str] = []
ar_texts: List[str] = []
has_ar: List[bool] = []
for example in examples:
text, lang = mixer.build(example, rng, force_language=force_language)
texts.append(text)
langs.append(lang)
pair = mixer.pair(example)
if pair is None:
en_texts.append(example["caption"])
ar_texts.append("")
has_ar.append(False)
else:
en, ar = pair
en_texts.append(en)
ar_texts.append(ar)
has_ar.append(True)
gen_ids, gen_mask = tokenizer.encode_batch(texts, max_length=tokenizer.max_length)
en_ids, en_mask = tokenizer.encode_batch(en_texts, max_length=tokenizer.max_length)
# Tokenizing empty strings gives a single [UNK]; the mask is unused for
# examples without a pair, so this is harmless.
ar_ids, ar_mask = tokenizer.encode_batch(ar_texts, max_length=tokenizer.max_length)
tokens = np.stack([
np.asarray(e["tokens"], dtype=np.int64).reshape(-1) for e in examples
])
return {
"tokens": tokens,
"text_ids": gen_ids,
"text_mask": gen_mask,
"en_ids": en_ids,
"en_mask": en_mask,
"ar_ids": ar_ids,
"ar_mask": ar_mask,
"has_ar": np.asarray(has_ar, dtype=np.bool_),
"langs": langs,
"texts": texts,
}
def to_torch_batch(batch: dict, device) -> dict:
import torch
out = {}
for key in ["tokens", "text_ids", "text_mask", "en_ids", "en_mask", "ar_ids", "ar_mask", "has_ar"]:
value = batch[key]
if key in ("text_mask", "en_mask", "ar_mask", "has_ar"):
out[key] = torch.as_tensor(value, dtype=torch.bool, device=device)
else:
out[key] = torch.as_tensor(value, dtype=torch.long, device=device)
out["langs"] = batch["langs"]
out["texts"] = batch["texts"]
return out
# ---------------------------------------------------------------------------
# Batch sources
# ---------------------------------------------------------------------------
class ListBatchSource:
"""In-memory batches for feasibility experiments and validation."""
def __init__(
self,
examples: Sequence[dict],
tokenizer: BPETokenizer,
mixer: CaptionMixer,
batch_size: int,
seed: int = 42,
infinite: bool = True,
force_language: Optional[str] = None,
rank: int = 0,
world_size: int = 1,
):
self.examples = list(examples)
self.rank = rank
self.world_size = world_size
self.tokenizer = tokenizer
self.mixer = mixer
self.batch_size = batch_size
self.seed = seed
self.infinite = infinite
self.force_language = force_language
self.rng = random.Random(seed)
def _one_epoch(self) -> Iterator[dict]:
order = [
i for i in range(len(self.examples))
if i % max(1, self.world_size) == self.rank
]
self.rng.shuffle(order)
for start in range(0, len(order), self.batch_size):
idx = order[start : start + self.batch_size]
batch = [self.examples[i] for i in idx]
if self.force_language is not None:
# Build with a forced language where possible.
kept = []
for ex in batch:
text, lang = self.mixer.build(ex, self.rng, force_language=self.force_language)
if lang == self.force_language:
kept.append(ex)
if not kept:
continue
batch = kept
yield build_batch(
batch, self.tokenizer, self.mixer, self.rng,
force_language=self.force_language,
)
def __iter__(self) -> Iterator[dict]:
if not self.examples:
raise ValueError("ListBatchSource received no examples.")
while True:
yielded = False
for batch in self._one_epoch():
yielded = True
yield batch
if not self.infinite:
break
if not yielded:
break
def steps_per_epoch(self) -> int:
return max(1, len(self.examples) // self.batch_size)
class StreamingBatchSource:
"""Infinite batches from the Hugging Face streaming dataset."""
def __init__(
self,
palette: Palette,
tokenizer: BPETokenizer,
mixer: CaptionMixer,
data_config: DataConfig,
batch_size: int,
split: str = "train",
source_split: Optional[str] = None,
seed: int = 42,
max_examples_per_epoch: Optional[int] = None,
force_language: Optional[str] = None,
):
self.palette = palette
self.tokenizer = tokenizer
self.mixer = mixer
self.data_config = data_config
self.batch_size = batch_size
self.split = split
self.source_split = source_split
self.seed = seed
self.max_examples_per_epoch = max_examples_per_epoch
self.force_language = force_language
self.rng = random.Random(seed)
self.epoch = 0
def _one_epoch(self) -> Iterator[dict]:
stats = DatasetStats()
buffer: List[dict] = []
for example in stream_dataset(
self.split, self.palette, self.data_config,
source_split=self.source_split, stats=stats,
max_examples=self.max_examples_per_epoch,
):
buffer.append(example)
if len(buffer) >= self.batch_size:
if self.force_language is not None:
kept = []
for ex in buffer:
_, lang = self.mixer.build(ex, self.rng, force_language=self.force_language)
if lang == self.force_language:
kept.append(ex)
buffer = kept
if buffer:
yield build_batch(
buffer, self.tokenizer, self.mixer, self.rng,
force_language=self.force_language,
)
buffer = []
if buffer:
yield build_batch(
buffer, self.tokenizer, self.mixer, self.rng,
force_language=self.force_language,
)
print(f"[data] epoch {self.epoch} stats: {stats.report()}")
self.epoch += 1
def __iter__(self) -> Iterator[dict]:
while True:
yielded = False
for batch in self._one_epoch():
yielded = True
yield batch
if not yielded:
raise RuntimeError("The streaming dataset produced no accepted examples.")
# ---------------------------------------------------------------------------
# Convenience collectors
# ---------------------------------------------------------------------------
def collect_captions(
n: int = 5_000,
split: str = "train",
data_config: Optional[DataConfig] = None,
include_subjects: bool = True,
) -> List[str]:
"""Collect unique captions without decoding images (for tokenizer training)."""
from datasets import load_dataset
data_config = data_config or DataConfig()
ds = load_dataset(data_config.dataset_id, split=split, streaming=True)
try:
ds = ds.select_columns(["caption", "subject"] if include_subjects else ["caption"])
except Exception:
pass
ds = prefetch_iter(ds, buffer_size=data_config.prefetch)
texts: List[str] = []
seen = set()
for ex in ds:
for key in (["caption", "subject"] if include_subjects else ["caption"]):
text = str(ex.get(key, "")).strip()
if not text or text in seen or is_low_information(text):
continue
seen.add(text)
texts.append(text)
if len(texts) >= n:
return texts
return texts
def collect_examples(
palette: Palette,
n: int,
data_config: Optional[DataConfig] = None,
split: str = "train",
source_split: Optional[str] = None,
arabic_store=None,
require_arabic: bool = False,
seed: int = 42,
progress_every: Optional[int] = None,
) -> List[dict]:
"""Collect a small in-memory subset for feasibility runs / validation."""
data_config = data_config or DataConfig(seed=seed)
examples: List[dict] = []
progress_every = progress_every or (250 if require_arabic else 2_000)
for ex in stream_dataset(
split, palette, data_config, source_split=source_split,
max_examples=None,
):
if require_arabic:
if arabic_store is None:
raise ValueError("require_arabic=True needs an Arabic store.")
ar = arabic_store.get(ex["caption"], ex["subject"])
if ar is None:
continue
examples.append(ex)
if len(examples) % progress_every == 0:
print(f" collected {len(examples):,}/{n:,}...")
if len(examples) >= n:
break
return examples
def make_mixer(
arabic_mode: str = "curated",
arabic_cache: Optional[str] = None,
p_arabic: float = 0.20,
p_egyptian: float = 0.20,
p_mixed: float = 0.05,
p_subject: float = 0.40,
seed: int = 42,
) -> CaptionMixer:
store = make_arabic_store(arabic_mode, arabic_cache)
return CaptionMixer(
arabic_store=store,
p_arabic=p_arabic,
p_egyptian=p_egyptian,
p_mixed=p_mixed,
p_subject=p_subject,
seed=seed,
)