Text-to-Video
MLX
apple-silicon
pratikgx's picture
Add unofficial MLX port of Apple FastFrames-S (code only; official weights loaded at runtime)
e87ef2c verified
Raw History Blame Contribute Delete
1.83 kB
"""Prompt tokenizer for the Wan umT5 encoder.
Cleaning matches Wan's ``HuggingfaceTokenizer`` with ``clean='whitespace'``
(ftfy, HTML unescape, collapsed whitespace). Token ids come from the
``tokenizer.json`` shipped in ``Wan-AI/Wan2.1-T2V-1.3B`` (``google/umt5-xxl``).
Sequences are truncated or padded to 512 tokens. The tokenizer's template
appends ``</s>``.
"""
from __future__ import annotations
import html
from pathlib import Path
import mlx.core as mx
def basic_clean(text: str) -> str:
import ftfy
text = ftfy.fix_text(text)
text = html.unescape(html.unescape(text))
return text.strip()
def whitespace_clean(text: str) -> str:
import regex as re
text = re.sub(r"\s+", " ", text)
return text.strip()
def clean_prompt(text: str) -> str:
return whitespace_clean(basic_clean(text))
class UMT5Tokenizer:
def __init__(self, tokenizer_json: str | Path, seq_len: int = 512):
from tokenizers import Tokenizer
self.tokenizer = Tokenizer.from_file(str(tokenizer_json))
self.seq_len = seq_len
pad = self.tokenizer.token_to_id("<pad>")
self.pad_id = 0 if pad is None else pad
def encode(self, prompts):
if isinstance(prompts, str):
prompts = [prompts]
ids = []
mask = []
for prompt in prompts:
encoded = self.tokenizer.encode(clean_prompt(prompt))
row = list(encoded.ids)
if len(row) > self.seq_len:
row = row[: self.seq_len]
valid = len(row)
if len(row) < self.seq_len:
row = row + [self.pad_id] * (self.seq_len - len(row))
ids.append(row)
mask.append([1] * valid + [0] * (self.seq_len - valid))
return mx.array(ids, dtype=mx.int32), mx.array(mask, dtype=mx.int32)