Text-to-Video
MLX
apple-silicon
File size: 1,827 Bytes
e87ef2c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
"""Prompt tokenizer for the Wan umT5 encoder.

Cleaning matches Wan's ``HuggingfaceTokenizer`` with ``clean='whitespace'``
(ftfy, HTML unescape, collapsed whitespace). Token ids come from the
``tokenizer.json`` shipped in ``Wan-AI/Wan2.1-T2V-1.3B`` (``google/umt5-xxl``).
Sequences are truncated or padded to 512 tokens. The tokenizer's template
appends ``</s>``.
"""

from __future__ import annotations

import html
from pathlib import Path

import mlx.core as mx


def basic_clean(text: str) -> str:
    import ftfy

    text = ftfy.fix_text(text)
    text = html.unescape(html.unescape(text))
    return text.strip()


def whitespace_clean(text: str) -> str:
    import regex as re

    text = re.sub(r"\s+", " ", text)
    return text.strip()


def clean_prompt(text: str) -> str:
    return whitespace_clean(basic_clean(text))


class UMT5Tokenizer:
    def __init__(self, tokenizer_json: str | Path, seq_len: int = 512):
        from tokenizers import Tokenizer

        self.tokenizer = Tokenizer.from_file(str(tokenizer_json))
        self.seq_len = seq_len
        pad = self.tokenizer.token_to_id("<pad>")
        self.pad_id = 0 if pad is None else pad

    def encode(self, prompts):
        if isinstance(prompts, str):
            prompts = [prompts]
        ids = []
        mask = []
        for prompt in prompts:
            encoded = self.tokenizer.encode(clean_prompt(prompt))
            row = list(encoded.ids)
            if len(row) > self.seq_len:
                row = row[: self.seq_len]
            valid = len(row)
            if len(row) < self.seq_len:
                row = row + [self.pad_id] * (self.seq_len - len(row))
            ids.append(row)
            mask.append([1] * valid + [0] * (self.seq_len - valid))
        return mx.array(ids, dtype=mx.int32), mx.array(mask, dtype=mx.int32)