Spaces:
Sleeping
Sleeping
File size: 4,346 Bytes
c388140 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 | from __future__ import annotations
import re
from dataclasses import dataclass
from typing import Dict, Iterable, List, Optional, Sequence
def _is_cjk(ch: str) -> bool:
"""
粗略判断 CJK 字符(覆盖常见中文/日文/韩文范围)。
"""
code = ord(ch)
return (
0x4E00 <= code <= 0x9FFF # CJK Unified Ideographs
or 0x3400 <= code <= 0x4DBF # CJK Unified Ideographs Extension A
or 0x3040 <= code <= 0x30FF # Japanese Hiragana/Katakana
or 0xAC00 <= code <= 0xD7AF # Hangul Syllables
)
def tokenize(text: str) -> List[str]:
"""
轻量 tokenizer:
- 英文/数字按“词”切分
- 中文(CJK)按“单字”切分(更适合无空格中文提示词)
"""
text = (text or "").strip().lower()
if not text:
return []
tokens: List[str] = []
buf: List[str] = []
def flush_buf() -> None:
if buf:
tokens.append("".join(buf))
buf.clear()
for ch in text:
if _is_cjk(ch):
flush_buf()
tokens.append(ch)
continue
# 英文/数字累计成 word,其他当作分隔符
if ch.isalnum() or ch in {"_", "-"}:
buf.append(ch)
else:
flush_buf()
flush_buf()
return tokens
@dataclass
class VocabConfig:
max_vocab: int = 8000
min_freq: int = 1
class SimpleVocab:
"""
极简词表:
- id=0: <pad>
- id=1: <unk>
"""
PAD = "<pad>"
UNK = "<unk>"
def __init__(self, token_to_id: Dict[str, int]):
if self.PAD not in token_to_id or self.UNK not in token_to_id:
raise ValueError("token_to_id 必须包含 <pad>/<unk>")
self.token_to_id = token_to_id
self.id_to_token = {i: t for t, i in token_to_id.items()}
@property
def pad_id(self) -> int:
return int(self.token_to_id[self.PAD])
@property
def unk_id(self) -> int:
return int(self.token_to_id[self.UNK])
@property
def size(self) -> int:
return len(self.token_to_id)
@classmethod
def build(cls, texts: Iterable[str], cfg: VocabConfig, extra_tokens: Optional[Sequence[str]] = None) -> "SimpleVocab":
"""
构建词表。
- texts:训练语料
- extra_tokens:可选“强制加入”的 token 文本(会先 tokenize 再加入)
说明:
- 续训时 tokenizer/vocab 必须与 checkpoint 一致,否则 embedding 尺寸会不匹配;
因此 extra_tokens 仅适用于“新训练”阶段。
"""
freq: Dict[str, int] = {}
for t in texts:
for tok in tokenize(t):
freq[tok] = freq.get(tok, 0) + 1
# 按频率排序,截断到 max_vocab(保留 PAD/UNK)
items = sorted(freq.items(), key=lambda x: (-x[1], x[0]))
items = [it for it in items if it[1] >= cfg.min_freq]
token_to_id = {cls.PAD: 0, cls.UNK: 1}
# 先插入强制 token(优先级高于按频率截断)
if extra_tokens:
forced: List[str] = []
for t in extra_tokens:
forced.extend(tokenize(t))
for tok in forced:
if tok in token_to_id:
continue
if len(token_to_id) >= int(cfg.max_vocab):
break
token_to_id[tok] = len(token_to_id)
# 再按频率补齐到 max_vocab
for tok, _ in items:
if tok in token_to_id:
continue
if len(token_to_id) >= int(cfg.max_vocab):
break
token_to_id[tok] = len(token_to_id)
return cls(token_to_id)
def encode(self, text: str, max_len: int) -> List[int]:
toks = tokenize(text)
ids = [self.token_to_id.get(t, self.unk_id) for t in toks[:max_len]]
if len(ids) < max_len:
ids.extend([self.pad_id] * (max_len - len(ids)))
return ids
def to_dict(self) -> Dict:
return {"token_to_id": self.token_to_id}
@classmethod
def from_dict(cls, d: Dict) -> "SimpleVocab":
return cls(token_to_id=dict(d["token_to_id"]))
|