Spaces:
Sleeping
Sleeping
Download blockdiffusion/text/simple_tokenizer.py from wkea/blockdiffusion-api: direct link, hf CLI and curl.
- Browser
- Download file 4.35 kB
-
https://huggingface.co/spaces/wkea/blockdiffusion-api/resolve/main/blockdiffusion/text/simple_tokenizer.py
- Command line
-
hf download hf://spaces/wkea/blockdiffusion-api/blockdiffusion/text/simple_tokenizer.py
-
curl -L -o simple_tokenizer.py https://huggingface.co/spaces/wkea/blockdiffusion-api/resolve/main/blockdiffusion/text/simple_tokenizer.py
4.35 kB
| from __future__ import annotations | |
| import re | |
| from dataclasses import dataclass | |
| from typing import Dict, Iterable, List, Optional, Sequence | |
| def _is_cjk(ch: str) -> bool: | |
| """ | |
| 粗略判断 CJK 字符(覆盖常见中文/日文/韩文范围)。 | |
| """ | |
| code = ord(ch) | |
| return ( | |
| 0x4E00 <= code <= 0x9FFF # CJK Unified Ideographs | |
| or 0x3400 <= code <= 0x4DBF # CJK Unified Ideographs Extension A | |
| or 0x3040 <= code <= 0x30FF # Japanese Hiragana/Katakana | |
| or 0xAC00 <= code <= 0xD7AF # Hangul Syllables | |
| ) | |
| def tokenize(text: str) -> List[str]: | |
| """ | |
| 轻量 tokenizer: | |
| - 英文/数字按“词”切分 | |
| - 中文(CJK)按“单字”切分(更适合无空格中文提示词) | |
| """ | |
| text = (text or "").strip().lower() | |
| if not text: | |
| return [] | |
| tokens: List[str] = [] | |
| buf: List[str] = [] | |
| def flush_buf() -> None: | |
| if buf: | |
| tokens.append("".join(buf)) | |
| buf.clear() | |
| for ch in text: | |
| if _is_cjk(ch): | |
| flush_buf() | |
| tokens.append(ch) | |
| continue | |
| # 英文/数字累计成 word,其他当作分隔符 | |
| if ch.isalnum() or ch in {"_", "-"}: | |
| buf.append(ch) | |
| else: | |
| flush_buf() | |
| flush_buf() | |
| return tokens | |
| class VocabConfig: | |
| max_vocab: int = 8000 | |
| min_freq: int = 1 | |
| class SimpleVocab: | |
| """ | |
| 极简词表: | |
| - id=0: <pad> | |
| - id=1: <unk> | |
| """ | |
| PAD = "<pad>" | |
| UNK = "<unk>" | |
| def __init__(self, token_to_id: Dict[str, int]): | |
| if self.PAD not in token_to_id or self.UNK not in token_to_id: | |
| raise ValueError("token_to_id 必须包含 <pad>/<unk>") | |
| self.token_to_id = token_to_id | |
| self.id_to_token = {i: t for t, i in token_to_id.items()} | |
| def pad_id(self) -> int: | |
| return int(self.token_to_id[self.PAD]) | |
| def unk_id(self) -> int: | |
| return int(self.token_to_id[self.UNK]) | |
| def size(self) -> int: | |
| return len(self.token_to_id) | |
| def build(cls, texts: Iterable[str], cfg: VocabConfig, extra_tokens: Optional[Sequence[str]] = None) -> "SimpleVocab": | |
| """ | |
| 构建词表。 | |
| - texts:训练语料 | |
| - extra_tokens:可选“强制加入”的 token 文本(会先 tokenize 再加入) | |
| 说明: | |
| - 续训时 tokenizer/vocab 必须与 checkpoint 一致,否则 embedding 尺寸会不匹配; | |
| 因此 extra_tokens 仅适用于“新训练”阶段。 | |
| """ | |
| freq: Dict[str, int] = {} | |
| for t in texts: | |
| for tok in tokenize(t): | |
| freq[tok] = freq.get(tok, 0) + 1 | |
| # 按频率排序,截断到 max_vocab(保留 PAD/UNK) | |
| items = sorted(freq.items(), key=lambda x: (-x[1], x[0])) | |
| items = [it for it in items if it[1] >= cfg.min_freq] | |
| token_to_id = {cls.PAD: 0, cls.UNK: 1} | |
| # 先插入强制 token(优先级高于按频率截断) | |
| if extra_tokens: | |
| forced: List[str] = [] | |
| for t in extra_tokens: | |
| forced.extend(tokenize(t)) | |
| for tok in forced: | |
| if tok in token_to_id: | |
| continue | |
| if len(token_to_id) >= int(cfg.max_vocab): | |
| break | |
| token_to_id[tok] = len(token_to_id) | |
| # 再按频率补齐到 max_vocab | |
| for tok, _ in items: | |
| if tok in token_to_id: | |
| continue | |
| if len(token_to_id) >= int(cfg.max_vocab): | |
| break | |
| token_to_id[tok] = len(token_to_id) | |
| return cls(token_to_id) | |
| def encode(self, text: str, max_len: int) -> List[int]: | |
| toks = tokenize(text) | |
| ids = [self.token_to_id.get(t, self.unk_id) for t in toks[:max_len]] | |
| if len(ids) < max_len: | |
| ids.extend([self.pad_id] * (max_len - len(ids))) | |
| return ids | |
| def to_dict(self) -> Dict: | |
| return {"token_to_id": self.token_to_id} | |
| def from_dict(cls, d: Dict) -> "SimpleVocab": | |
| return cls(token_to_id=dict(d["token_to_id"])) | |