Download pixelgen/data.py from nomeda-lab/pixel-art-tiny-adaptive: direct link, hf CLI and curl.
- Browser
- Download file 19.2 kB
-
https://huggingface.co/nomeda-lab/pixel-art-tiny-adaptive/resolve/main/pixelgen/data.py
- Command line
-
hf download hf://nomeda-lab/pixel-art-tiny-adaptive/pixelgen/data.py
-
curl -L -o data.py https://huggingface.co/nomeda-lab/pixel-art-tiny-adaptive/resolve/main/pixelgen/data.py
19.2 kB
| """Streaming data pipeline, filtering, caption mixing and batching. | |
| Nothing in this module downloads the dataset to disk. Hugging Face's streaming | |
| Parquet reader is used, and each 16x16 PNG is decoded in memory and mapped to | |
| the native 4,097-colour palette. | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import io | |
| import random | |
| import re | |
| from dataclasses import dataclass, field | |
| from typing import Dict, Iterable, Iterator, List, Optional, Sequence, Tuple | |
| import numpy as np | |
| from .arabic import CuratedArabicStore, make_arabic_store | |
| from .config import DATASET_ID, IMAGE_SIZE, PIXELS_PER_IMAGE, TEXT_MAX_LENGTH, DataConfig | |
| from .palette import Palette | |
| from .text import BPETokenizer | |
| from .utils import prefetch_iter | |
| _ALNUM = re.compile(r"[A-Za-z0-9\u0600-\u06FF]") | |
| def as_pil_image(image): | |
| """Normalise the many ways HF can hand back an image to a PIL RGBA image.""" | |
| if image is None: | |
| return None | |
| if hasattr(image, "convert"): | |
| return image.convert("RGBA") | |
| if isinstance(image, dict): | |
| try: | |
| from PIL import Image | |
| if image.get("bytes"): | |
| return Image.open(io.BytesIO(image["bytes"])).convert("RGBA") | |
| if image.get("path"): | |
| return Image.open(image["path"]).convert("RGBA") | |
| except Exception: | |
| return None | |
| return None | |
| # --------------------------------------------------------------------------- | |
| # Stats / filtering | |
| # --------------------------------------------------------------------------- | |
| class DatasetStats: | |
| scanned: int = 0 | |
| accepted: int = 0 | |
| empty: int = 0 | |
| too_few_opaque: int = 0 | |
| too_few_colors: int = 0 | |
| bad_caption: int = 0 | |
| duplicate: int = 0 | |
| wrong_size: int = 0 | |
| palette_missing: int = 0 | |
| held_out: int = 0 | |
| def report(self) -> str: | |
| return ( | |
| f"scanned={self.scanned:,} accepted={self.accepted:,} | " | |
| f"empty={self.empty:,} few_opaque={self.too_few_opaque:,} " | |
| f"few_colors={self.too_few_colors:,} bad_caption={self.bad_caption:,} " | |
| f"duplicate={self.duplicate:,} wrong_size={self.wrong_size:,} " | |
| f"palette_missing={self.palette_missing:,} held_out={self.held_out:,}" | |
| ) | |
| _LOW_INFO = {"object", "item", "thing", "stuff", "unknown", "n/a", "none", "sprite", "?"} | |
| def is_low_information(caption: str) -> bool: | |
| c = caption.strip().lower() | |
| if not c or c in _LOW_INFO: | |
| return True | |
| if len(c) < 2: | |
| return True | |
| if not _ALNUM.search(c): | |
| return True | |
| return False | |
| def subject_bucket(subject: str) -> int: | |
| h = hashlib.blake2b(subject.strip().lower().encode("utf-8"), digest_size=4).digest() | |
| return int.from_bytes(h, "big") % 100 | |
| def keep_subject_for_split(subject: str, holdout_pct: float, target: str) -> bool: | |
| if holdout_pct <= 0: | |
| return True | |
| bucket = subject_bucket(subject) | |
| threshold = int(round(holdout_pct * 100)) | |
| if target == "train": | |
| return bucket >= threshold | |
| return bucket < threshold | |
| # --------------------------------------------------------------------------- | |
| # Streaming source | |
| # --------------------------------------------------------------------------- | |
| def stream_dataset( | |
| split: str, | |
| palette: Palette, | |
| data_config: DataConfig, | |
| source_split: Optional[str] = None, | |
| stats: Optional[DatasetStats] = None, | |
| max_examples: Optional[int] = None, | |
| ) -> Iterator[dict]: | |
| """Yield filtered examples with 256 native-palette tokens. | |
| Args: | |
| split: "train" or "validation"; controls subject-holdout filtering. | |
| source_split: the actual HF split to read. Defaults to ``split``. | |
| For a subject-disjoint validation set, pass source_split="train" | |
| and split="validation". | |
| """ | |
| from datasets import load_dataset | |
| stats = stats or DatasetStats() | |
| source_split = source_split or split | |
| ds = load_dataset(data_config.dataset_id, split=source_split, streaming=True) | |
| sharded = False | |
| if data_config.world_size > 1: | |
| try: | |
| ds = ds.shard(num_shards=data_config.world_size, index=data_config.rank) | |
| sharded = True | |
| except Exception: | |
| # Older datasets versions: fall back to row_id filtering below. | |
| pass | |
| if source_split == "train" and data_config.shuffle_buffer > 0: | |
| ds = ds.shuffle(seed=data_config.seed, buffer_size=data_config.shuffle_buffer) | |
| ds = prefetch_iter(ds, buffer_size=data_config.prefetch) | |
| seen_hashes: set[int] = set() | |
| accepted = 0 | |
| for example in ds: | |
| stats.scanned += 1 | |
| image = as_pil_image(example.get("image")) | |
| if image is None: | |
| stats.wrong_size += 1 | |
| continue | |
| if image.size != (IMAGE_SIZE, IMAGE_SIZE): | |
| stats.wrong_size += 1 | |
| continue | |
| row_id = int(example.get("row_id", -1)) | |
| if data_config.world_size > 1 and not sharded and row_id >= 0: | |
| if row_id % data_config.world_size != data_config.rank: | |
| continue | |
| caption = str(example.get("caption", "")).strip() | |
| subject = str(example.get("subject", "")).strip() or caption | |
| if not keep_subject_for_split(subject, data_config.subject_holdout, split): | |
| stats.held_out += 1 | |
| continue | |
| if not caption or len(caption) < data_config.min_caption_chars or len(caption) > data_config.max_caption_chars: | |
| stats.bad_caption += 1 | |
| continue | |
| if is_low_information(caption) or is_low_information(subject): | |
| stats.bad_caption += 1 | |
| continue | |
| tokens, missing = palette.from_pil(image) | |
| if missing: | |
| stats.palette_missing += missing | |
| opaque = int((tokens != 0).sum()) | |
| if opaque == 0: | |
| stats.empty += 1 | |
| continue | |
| if opaque < data_config.min_opaque_pixels: | |
| stats.too_few_opaque += 1 | |
| continue | |
| unique = int(len(np.unique(tokens))) - (1 if (tokens == 0).any() else 0) | |
| if unique < data_config.min_unique_colors: | |
| stats.too_few_colors += 1 | |
| continue | |
| if data_config.dedupe: | |
| h = hashlib.blake2b( | |
| tokens.tobytes() + b"|" + caption.encode("utf-8"), digest_size=8 | |
| ).digest() | |
| key = int.from_bytes(h, "big") | |
| if key in seen_hashes: | |
| stats.duplicate += 1 | |
| continue | |
| if len(seen_hashes) < data_config.max_dedupe_entries: | |
| seen_hashes.add(key) | |
| stats.accepted += 1 | |
| accepted += 1 | |
| yield { | |
| "tokens": tokens.astype(np.uint16, copy=False), | |
| "caption": caption, | |
| "subject": subject, | |
| "row_id": int(example.get("row_id", -1)), | |
| "opaque": opaque, | |
| "unique_colors": unique, | |
| } | |
| if max_examples is not None and accepted >= max_examples: | |
| break | |
| # --------------------------------------------------------------------------- | |
| # Caption mixing / batching | |
| # --------------------------------------------------------------------------- | |
| class CaptionMixer: | |
| def __init__( | |
| self, | |
| arabic_store=None, | |
| p_arabic: float = 0.20, | |
| p_egyptian: float = 0.20, | |
| p_mixed: float = 0.05, | |
| p_subject: float = 0.40, | |
| seed: int = 42, | |
| ): | |
| self.arabic_store = arabic_store | |
| self.p_arabic = p_arabic | |
| self.p_egyptian = p_egyptian | |
| self.p_mixed = p_mixed | |
| self.p_subject = p_subject | |
| self.rng = random.Random(seed) | |
| def _english(self, example: dict, rng: random.Random) -> str: | |
| caption = example["caption"] | |
| subject = example["subject"] | |
| if subject and rng.random() < self.p_subject: | |
| return subject | |
| return caption | |
| def build(self, example: dict, rng: random.Random, force_language: Optional[str] = None) -> Tuple[str, str]: | |
| english = self._english(example, rng) | |
| if force_language == "en": | |
| return english, "en" | |
| arabic = None | |
| if self.arabic_store is not None: | |
| arabic = self.arabic_store.get( | |
| example["caption"], example["subject"], dialect="msa", salt=str(rng.random()) | |
| ) | |
| if force_language == "ar": | |
| return (arabic, "ar") if arabic else (english, "en") | |
| if arabic and rng.random() < self.p_arabic: | |
| lang = "ar" | |
| if rng.random() < self.p_egyptian: | |
| eg = self.arabic_store.get( | |
| example["caption"], example["subject"], dialect="egyptian", | |
| salt=str(rng.random()), | |
| ) | |
| if eg: | |
| arabic = eg | |
| lang = "ar-eg" | |
| text = arabic | |
| else: | |
| text = english | |
| lang = "en" | |
| if self.arabic_store is not None and rng.random() < self.p_mixed: | |
| text = self.arabic_store.mixed(english, rng=rng) | |
| lang = "mixed" | |
| return text, lang | |
| def pair(self, example: dict) -> Optional[Tuple[str, str]]: | |
| if self.arabic_store is None: | |
| return None | |
| return self.arabic_store.get_pair(example["caption"], example["subject"]) | |
| def build_batch( | |
| examples: Sequence[dict], | |
| tokenizer: BPETokenizer, | |
| mixer: CaptionMixer, | |
| rng: random.Random, | |
| force_language: Optional[str] = None, | |
| ) -> dict: | |
| texts: List[str] = [] | |
| langs: List[str] = [] | |
| en_texts: List[str] = [] | |
| ar_texts: List[str] = [] | |
| has_ar: List[bool] = [] | |
| for example in examples: | |
| text, lang = mixer.build(example, rng, force_language=force_language) | |
| texts.append(text) | |
| langs.append(lang) | |
| pair = mixer.pair(example) | |
| if pair is None: | |
| en_texts.append(example["caption"]) | |
| ar_texts.append("") | |
| has_ar.append(False) | |
| else: | |
| en, ar = pair | |
| en_texts.append(en) | |
| ar_texts.append(ar) | |
| has_ar.append(True) | |
| gen_ids, gen_mask = tokenizer.encode_batch(texts, max_length=tokenizer.max_length) | |
| en_ids, en_mask = tokenizer.encode_batch(en_texts, max_length=tokenizer.max_length) | |
| # Tokenizing empty strings gives a single [UNK]; the mask is unused for | |
| # examples without a pair, so this is harmless. | |
| ar_ids, ar_mask = tokenizer.encode_batch(ar_texts, max_length=tokenizer.max_length) | |
| tokens = np.stack([ | |
| np.asarray(e["tokens"], dtype=np.int64).reshape(-1) for e in examples | |
| ]) | |
| return { | |
| "tokens": tokens, | |
| "text_ids": gen_ids, | |
| "text_mask": gen_mask, | |
| "en_ids": en_ids, | |
| "en_mask": en_mask, | |
| "ar_ids": ar_ids, | |
| "ar_mask": ar_mask, | |
| "has_ar": np.asarray(has_ar, dtype=np.bool_), | |
| "langs": langs, | |
| "texts": texts, | |
| } | |
| def to_torch_batch(batch: dict, device) -> dict: | |
| import torch | |
| out = {} | |
| for key in ["tokens", "text_ids", "text_mask", "en_ids", "en_mask", "ar_ids", "ar_mask", "has_ar"]: | |
| value = batch[key] | |
| if key in ("text_mask", "en_mask", "ar_mask", "has_ar"): | |
| out[key] = torch.as_tensor(value, dtype=torch.bool, device=device) | |
| else: | |
| out[key] = torch.as_tensor(value, dtype=torch.long, device=device) | |
| out["langs"] = batch["langs"] | |
| out["texts"] = batch["texts"] | |
| return out | |
| # --------------------------------------------------------------------------- | |
| # Batch sources | |
| # --------------------------------------------------------------------------- | |
| class ListBatchSource: | |
| """In-memory batches for feasibility experiments and validation.""" | |
| def __init__( | |
| self, | |
| examples: Sequence[dict], | |
| tokenizer: BPETokenizer, | |
| mixer: CaptionMixer, | |
| batch_size: int, | |
| seed: int = 42, | |
| infinite: bool = True, | |
| force_language: Optional[str] = None, | |
| rank: int = 0, | |
| world_size: int = 1, | |
| ): | |
| self.examples = list(examples) | |
| self.rank = rank | |
| self.world_size = world_size | |
| self.tokenizer = tokenizer | |
| self.mixer = mixer | |
| self.batch_size = batch_size | |
| self.seed = seed | |
| self.infinite = infinite | |
| self.force_language = force_language | |
| self.rng = random.Random(seed) | |
| def _one_epoch(self) -> Iterator[dict]: | |
| order = [ | |
| i for i in range(len(self.examples)) | |
| if i % max(1, self.world_size) == self.rank | |
| ] | |
| self.rng.shuffle(order) | |
| for start in range(0, len(order), self.batch_size): | |
| idx = order[start : start + self.batch_size] | |
| batch = [self.examples[i] for i in idx] | |
| if self.force_language is not None: | |
| # Build with a forced language where possible. | |
| kept = [] | |
| for ex in batch: | |
| text, lang = self.mixer.build(ex, self.rng, force_language=self.force_language) | |
| if lang == self.force_language: | |
| kept.append(ex) | |
| if not kept: | |
| continue | |
| batch = kept | |
| yield build_batch( | |
| batch, self.tokenizer, self.mixer, self.rng, | |
| force_language=self.force_language, | |
| ) | |
| def __iter__(self) -> Iterator[dict]: | |
| if not self.examples: | |
| raise ValueError("ListBatchSource received no examples.") | |
| while True: | |
| yielded = False | |
| for batch in self._one_epoch(): | |
| yielded = True | |
| yield batch | |
| if not self.infinite: | |
| break | |
| if not yielded: | |
| break | |
| def steps_per_epoch(self) -> int: | |
| return max(1, len(self.examples) // self.batch_size) | |
| class StreamingBatchSource: | |
| """Infinite batches from the Hugging Face streaming dataset.""" | |
| def __init__( | |
| self, | |
| palette: Palette, | |
| tokenizer: BPETokenizer, | |
| mixer: CaptionMixer, | |
| data_config: DataConfig, | |
| batch_size: int, | |
| split: str = "train", | |
| source_split: Optional[str] = None, | |
| seed: int = 42, | |
| max_examples_per_epoch: Optional[int] = None, | |
| force_language: Optional[str] = None, | |
| ): | |
| self.palette = palette | |
| self.tokenizer = tokenizer | |
| self.mixer = mixer | |
| self.data_config = data_config | |
| self.batch_size = batch_size | |
| self.split = split | |
| self.source_split = source_split | |
| self.seed = seed | |
| self.max_examples_per_epoch = max_examples_per_epoch | |
| self.force_language = force_language | |
| self.rng = random.Random(seed) | |
| self.epoch = 0 | |
| def _one_epoch(self) -> Iterator[dict]: | |
| stats = DatasetStats() | |
| buffer: List[dict] = [] | |
| for example in stream_dataset( | |
| self.split, self.palette, self.data_config, | |
| source_split=self.source_split, stats=stats, | |
| max_examples=self.max_examples_per_epoch, | |
| ): | |
| buffer.append(example) | |
| if len(buffer) >= self.batch_size: | |
| if self.force_language is not None: | |
| kept = [] | |
| for ex in buffer: | |
| _, lang = self.mixer.build(ex, self.rng, force_language=self.force_language) | |
| if lang == self.force_language: | |
| kept.append(ex) | |
| buffer = kept | |
| if buffer: | |
| yield build_batch( | |
| buffer, self.tokenizer, self.mixer, self.rng, | |
| force_language=self.force_language, | |
| ) | |
| buffer = [] | |
| if buffer: | |
| yield build_batch( | |
| buffer, self.tokenizer, self.mixer, self.rng, | |
| force_language=self.force_language, | |
| ) | |
| print(f"[data] epoch {self.epoch} stats: {stats.report()}") | |
| self.epoch += 1 | |
| def __iter__(self) -> Iterator[dict]: | |
| while True: | |
| yielded = False | |
| for batch in self._one_epoch(): | |
| yielded = True | |
| yield batch | |
| if not yielded: | |
| raise RuntimeError("The streaming dataset produced no accepted examples.") | |
| # --------------------------------------------------------------------------- | |
| # Convenience collectors | |
| # --------------------------------------------------------------------------- | |
| def collect_captions( | |
| n: int = 5_000, | |
| split: str = "train", | |
| data_config: Optional[DataConfig] = None, | |
| include_subjects: bool = True, | |
| ) -> List[str]: | |
| """Collect unique captions without decoding images (for tokenizer training).""" | |
| from datasets import load_dataset | |
| data_config = data_config or DataConfig() | |
| ds = load_dataset(data_config.dataset_id, split=split, streaming=True) | |
| try: | |
| ds = ds.select_columns(["caption", "subject"] if include_subjects else ["caption"]) | |
| except Exception: | |
| pass | |
| ds = prefetch_iter(ds, buffer_size=data_config.prefetch) | |
| texts: List[str] = [] | |
| seen = set() | |
| for ex in ds: | |
| for key in (["caption", "subject"] if include_subjects else ["caption"]): | |
| text = str(ex.get(key, "")).strip() | |
| if not text or text in seen or is_low_information(text): | |
| continue | |
| seen.add(text) | |
| texts.append(text) | |
| if len(texts) >= n: | |
| return texts | |
| return texts | |
| def collect_examples( | |
| palette: Palette, | |
| n: int, | |
| data_config: Optional[DataConfig] = None, | |
| split: str = "train", | |
| source_split: Optional[str] = None, | |
| arabic_store=None, | |
| require_arabic: bool = False, | |
| seed: int = 42, | |
| progress_every: Optional[int] = None, | |
| ) -> List[dict]: | |
| """Collect a small in-memory subset for feasibility runs / validation.""" | |
| data_config = data_config or DataConfig(seed=seed) | |
| examples: List[dict] = [] | |
| progress_every = progress_every or (250 if require_arabic else 2_000) | |
| for ex in stream_dataset( | |
| split, palette, data_config, source_split=source_split, | |
| max_examples=None, | |
| ): | |
| if require_arabic: | |
| if arabic_store is None: | |
| raise ValueError("require_arabic=True needs an Arabic store.") | |
| ar = arabic_store.get(ex["caption"], ex["subject"]) | |
| if ar is None: | |
| continue | |
| examples.append(ex) | |
| if len(examples) % progress_every == 0: | |
| print(f" collected {len(examples):,}/{n:,}...") | |
| if len(examples) >= n: | |
| break | |
| return examples | |
| def make_mixer( | |
| arabic_mode: str = "curated", | |
| arabic_cache: Optional[str] = None, | |
| p_arabic: float = 0.20, | |
| p_egyptian: float = 0.20, | |
| p_mixed: float = 0.05, | |
| p_subject: float = 0.40, | |
| seed: int = 42, | |
| ) -> CaptionMixer: | |
| store = make_arabic_store(arabic_mode, arabic_cache) | |
| return CaptionMixer( | |
| arabic_store=store, | |
| p_arabic=p_arabic, | |
| p_egyptian=p_egyptian, | |
| p_mixed=p_mixed, | |
| p_subject=p_subject, | |
| seed=seed, | |
| ) | |