from __future__ import annotations import re from dataclasses import dataclass from typing import Dict, Iterable, List, Optional, Sequence def _is_cjk(ch: str) -> bool: """ 粗略判断 CJK 字符(覆盖常见中文/日文/韩文范围)。 """ code = ord(ch) return ( 0x4E00 <= code <= 0x9FFF # CJK Unified Ideographs or 0x3400 <= code <= 0x4DBF # CJK Unified Ideographs Extension A or 0x3040 <= code <= 0x30FF # Japanese Hiragana/Katakana or 0xAC00 <= code <= 0xD7AF # Hangul Syllables ) def tokenize(text: str) -> List[str]: """ 轻量 tokenizer: - 英文/数字按“词”切分 - 中文(CJK)按“单字”切分(更适合无空格中文提示词) """ text = (text or "").strip().lower() if not text: return [] tokens: List[str] = [] buf: List[str] = [] def flush_buf() -> None: if buf: tokens.append("".join(buf)) buf.clear() for ch in text: if _is_cjk(ch): flush_buf() tokens.append(ch) continue # 英文/数字累计成 word,其他当作分隔符 if ch.isalnum() or ch in {"_", "-"}: buf.append(ch) else: flush_buf() flush_buf() return tokens @dataclass class VocabConfig: max_vocab: int = 8000 min_freq: int = 1 class SimpleVocab: """ 极简词表: - id=0: - id=1: """ PAD = "" UNK = "" def __init__(self, token_to_id: Dict[str, int]): if self.PAD not in token_to_id or self.UNK not in token_to_id: raise ValueError("token_to_id 必须包含 /") self.token_to_id = token_to_id self.id_to_token = {i: t for t, i in token_to_id.items()} @property def pad_id(self) -> int: return int(self.token_to_id[self.PAD]) @property def unk_id(self) -> int: return int(self.token_to_id[self.UNK]) @property def size(self) -> int: return len(self.token_to_id) @classmethod def build(cls, texts: Iterable[str], cfg: VocabConfig, extra_tokens: Optional[Sequence[str]] = None) -> "SimpleVocab": """ 构建词表。 - texts:训练语料 - extra_tokens:可选“强制加入”的 token 文本(会先 tokenize 再加入) 说明: - 续训时 tokenizer/vocab 必须与 checkpoint 一致,否则 embedding 尺寸会不匹配; 因此 extra_tokens 仅适用于“新训练”阶段。 """ freq: Dict[str, int] = {} for t in texts: for tok in tokenize(t): freq[tok] = freq.get(tok, 0) + 1 # 按频率排序,截断到 max_vocab(保留 PAD/UNK) items = sorted(freq.items(), key=lambda x: (-x[1], x[0])) items = [it for it in items if it[1] >= cfg.min_freq] token_to_id = {cls.PAD: 0, cls.UNK: 1} # 先插入强制 token(优先级高于按频率截断) if extra_tokens: forced: List[str] = [] for t in extra_tokens: forced.extend(tokenize(t)) for tok in forced: if tok in token_to_id: continue if len(token_to_id) >= int(cfg.max_vocab): break token_to_id[tok] = len(token_to_id) # 再按频率补齐到 max_vocab for tok, _ in items: if tok in token_to_id: continue if len(token_to_id) >= int(cfg.max_vocab): break token_to_id[tok] = len(token_to_id) return cls(token_to_id) def encode(self, text: str, max_len: int) -> List[int]: toks = tokenize(text) ids = [self.token_to_id.get(t, self.unk_id) for t in toks[:max_len]] if len(ids) < max_len: ids.extend([self.pad_id] * (max_len - len(ids))) return ids def to_dict(self) -> Dict: return {"token_to_id": self.token_to_id} @classmethod def from_dict(cls, d: Dict) -> "SimpleVocab": return cls(token_to_id=dict(d["token_to_id"]))