"""Prompt tokenizer for the Wan umT5 encoder. Cleaning matches Wan's ``HuggingfaceTokenizer`` with ``clean='whitespace'`` (ftfy, HTML unescape, collapsed whitespace). Token ids come from the ``tokenizer.json`` shipped in ``Wan-AI/Wan2.1-T2V-1.3B`` (``google/umt5-xxl``). Sequences are truncated or padded to 512 tokens. The tokenizer's template appends ````. """ from __future__ import annotations import html from pathlib import Path import mlx.core as mx def basic_clean(text: str) -> str: import ftfy text = ftfy.fix_text(text) text = html.unescape(html.unescape(text)) return text.strip() def whitespace_clean(text: str) -> str: import regex as re text = re.sub(r"\s+", " ", text) return text.strip() def clean_prompt(text: str) -> str: return whitespace_clean(basic_clean(text)) class UMT5Tokenizer: def __init__(self, tokenizer_json: str | Path, seq_len: int = 512): from tokenizers import Tokenizer self.tokenizer = Tokenizer.from_file(str(tokenizer_json)) self.seq_len = seq_len pad = self.tokenizer.token_to_id("") self.pad_id = 0 if pad is None else pad def encode(self, prompts): if isinstance(prompts, str): prompts = [prompts] ids = [] mask = [] for prompt in prompts: encoded = self.tokenizer.encode(clean_prompt(prompt)) row = list(encoded.ids) if len(row) > self.seq_len: row = row[: self.seq_len] valid = len(row) if len(row) < self.seq_len: row = row + [self.pad_id] * (self.seq_len - len(row)) ids.append(row) mask.append([1] * valid + [0] * (self.seq_len - valid)) return mx.array(ids, dtype=mx.int32), mx.array(mask, dtype=mx.int32)