Text-to-Image
English
numpy
machine-learning
deep-learning
generative-ai
text-2-image
image-generation
open-weights
model-weights
ai-art
pixel-art
game-development
gamedev
game-assets
asset-generator
sprite-generator
offline
tiny-model
numpy-runtime
int8-quantization
self-supervised
procedural-data
gpt
english-prompts
awesome-ai
| """Tokenizer utilities for PXG-Tiny (text <-> ids), thin wrapper over config.""" | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) | |
| from pxg_tiny import config as C # noqa: E402,E501 | |
| encode_caption = C.encode_caption | |
| decode_caption = C.decode_caption | |
| CAP_LEN = C.CAP_LEN | |
| def sanitize(text: str) -> str: | |
| """Lowercase + fold unsupported glyphs (mirrors byte-fallback philosophy).""" | |
| return "".join(ch if C.decode_char(C.encode_char(ch)) == ch | |
| else "?" for ch in text.lower()) | |