blockdiffusion-api / blockdiffusion /text /simple_tokenizer.py
wkea's picture
Deploy Space
c388140 verified
Raw History Blame Contribute Delete
4.35 kB
from __future__ import annotations
import re
from dataclasses import dataclass
from typing import Dict, Iterable, List, Optional, Sequence
def _is_cjk(ch: str) -> bool:
"""
粗略判断 CJK 字符(覆盖常见中文/日文/韩文范围)。
"""
code = ord(ch)
return (
0x4E00 <= code <= 0x9FFF # CJK Unified Ideographs
or 0x3400 <= code <= 0x4DBF # CJK Unified Ideographs Extension A
or 0x3040 <= code <= 0x30FF # Japanese Hiragana/Katakana
or 0xAC00 <= code <= 0xD7AF # Hangul Syllables
)
def tokenize(text: str) -> List[str]:
"""
轻量 tokenizer:
- 英文/数字按“词”切分
- 中文(CJK)按“单字”切分(更适合无空格中文提示词)
"""
text = (text or "").strip().lower()
if not text:
return []
tokens: List[str] = []
buf: List[str] = []
def flush_buf() -> None:
if buf:
tokens.append("".join(buf))
buf.clear()
for ch in text:
if _is_cjk(ch):
flush_buf()
tokens.append(ch)
continue
# 英文/数字累计成 word,其他当作分隔符
if ch.isalnum() or ch in {"_", "-"}:
buf.append(ch)
else:
flush_buf()
flush_buf()
return tokens
@dataclass
class VocabConfig:
max_vocab: int = 8000
min_freq: int = 1
class SimpleVocab:
"""
极简词表:
- id=0: <pad>
- id=1: <unk>
"""
PAD = "<pad>"
UNK = "<unk>"
def __init__(self, token_to_id: Dict[str, int]):
if self.PAD not in token_to_id or self.UNK not in token_to_id:
raise ValueError("token_to_id 必须包含 <pad>/<unk>")
self.token_to_id = token_to_id
self.id_to_token = {i: t for t, i in token_to_id.items()}
@property
def pad_id(self) -> int:
return int(self.token_to_id[self.PAD])
@property
def unk_id(self) -> int:
return int(self.token_to_id[self.UNK])
@property
def size(self) -> int:
return len(self.token_to_id)
@classmethod
def build(cls, texts: Iterable[str], cfg: VocabConfig, extra_tokens: Optional[Sequence[str]] = None) -> "SimpleVocab":
"""
构建词表。
- texts:训练语料
- extra_tokens:可选“强制加入”的 token 文本(会先 tokenize 再加入)
说明:
- 续训时 tokenizer/vocab 必须与 checkpoint 一致,否则 embedding 尺寸会不匹配;
因此 extra_tokens 仅适用于“新训练”阶段。
"""
freq: Dict[str, int] = {}
for t in texts:
for tok in tokenize(t):
freq[tok] = freq.get(tok, 0) + 1
# 按频率排序,截断到 max_vocab(保留 PAD/UNK)
items = sorted(freq.items(), key=lambda x: (-x[1], x[0]))
items = [it for it in items if it[1] >= cfg.min_freq]
token_to_id = {cls.PAD: 0, cls.UNK: 1}
# 先插入强制 token(优先级高于按频率截断)
if extra_tokens:
forced: List[str] = []
for t in extra_tokens:
forced.extend(tokenize(t))
for tok in forced:
if tok in token_to_id:
continue
if len(token_to_id) >= int(cfg.max_vocab):
break
token_to_id[tok] = len(token_to_id)
# 再按频率补齐到 max_vocab
for tok, _ in items:
if tok in token_to_id:
continue
if len(token_to_id) >= int(cfg.max_vocab):
break
token_to_id[tok] = len(token_to_id)
return cls(token_to_id)
def encode(self, text: str, max_len: int) -> List[int]:
toks = tokenize(text)
ids = [self.token_to_id.get(t, self.unk_id) for t in toks[:max_len]]
if len(ids) < max_len:
ids.extend([self.pad_id] * (max_len - len(ids)))
return ids
def to_dict(self) -> Dict:
return {"token_to_id": self.token_to_id}
@classmethod
def from_dict(cls, d: Dict) -> "SimpleVocab":
return cls(token_to_id=dict(d["token_to_id"]))