feat: add j72 tokenizer model, reversible adapter and model card
Browse files- README.md +65 -0
- tokenizer.model +3 -0
- tokenizer.sha256 +1 -0
- tokenizer_adapters.py +40 -0
README.md
CHANGED
|
@@ -1,3 +1,68 @@
|
|
| 1 |
---
|
| 2 |
license: cc-by-nc-4.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
license: cc-by-nc-4.0
|
| 3 |
+
language:
|
| 4 |
+
- ja
|
| 5 |
+
library_name: sentencepiece
|
| 6 |
+
tags:
|
| 7 |
+
- tokenizer
|
| 8 |
+
- sentencepiece
|
| 9 |
+
- unigram
|
| 10 |
+
- japanese
|
| 11 |
---
|
| 12 |
+
|
| 13 |
+
# j72-tokenizer
|
| 14 |
+
|
| 15 |
+
日本語向けの SentencePiece Unigram トークナイザ(語彙 72,000)。J72 系列モデルの学習・推論で使用しているものと同一のファイルです。
|
| 16 |
+
|
| 17 |
+
改行や `▁` などを私用領域文字 `U+E000` でエスケープする薄いラッパー(`ReversibleSentencePieceAdapter`)と組み合わせて使うことで、任意の文字列を `decode(encode(text)) == text` で復元できます。
|
| 18 |
+
|
| 19 |
+
## ファイル
|
| 20 |
+
|
| 21 |
+
| ファイル | 内容 |
|
| 22 |
+
|---|---|
|
| 23 |
+
| `tokenizer.model` | SentencePiece モデル本体 |
|
| 24 |
+
| `tokenizer.sha256` | `tokenizer.model` の SHA-256 |
|
| 25 |
+
| `tokenizer_adapters.py` | 可逆エスケープラッパー(`escape-e000-v2`) |
|
| 26 |
+
|
| 27 |
+
`tokenizer.model` の SHA-256:
|
| 28 |
+
|
| 29 |
+
```
|
| 30 |
+
73f58a921e338883470bc99c4cc364a3d5ef955a4c28b8ef62f0af508208f03f
|
| 31 |
+
```
|
| 32 |
+
|
| 33 |
+
## 仕様
|
| 34 |
+
|
| 35 |
+
| 項目 | 値 |
|
| 36 |
+
|---|---|
|
| 37 |
+
| 方式 | SentencePiece Unigram |
|
| 38 |
+
| 語彙数 | 72,000 |
|
| 39 |
+
| 特殊トークン | `<unk>`=0, `<s>`=1, `</s>`=2(pad なし) |
|
| 40 |
+
| byte fallback | あり(`<0x00>`〜`<0xFF>` の 256 個) |
|
| 41 |
+
| 正規化 | なし(identity) |
|
| 42 |
+
| dummy prefix | なし |
|
| 43 |
+
|
| 44 |
+
## 使い方
|
| 45 |
+
|
| 46 |
+
```python
|
| 47 |
+
from huggingface_hub import snapshot_download
|
| 48 |
+
import sys
|
| 49 |
+
|
| 50 |
+
path = snapshot_download("j-llm/j72-tokenizer")
|
| 51 |
+
sys.path.insert(0, path)
|
| 52 |
+
from tokenizer_adapters import ReversibleSentencePieceAdapter
|
| 53 |
+
|
| 54 |
+
tok = ReversibleSentencePieceAdapter(f"{path}/tokenizer.model")
|
| 55 |
+
text = "吾輩は猫である。\n名前はまだ無い。"
|
| 56 |
+
ids = tok.encode(text)
|
| 57 |
+
assert tok.decode(ids) == text
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
## 注意
|
| 61 |
+
|
| 62 |
+
- `tokenizer.model` を SentencePiece で直接読むだけでは、`\r\n`・`U+2028`・`▁` などを含む文字列が元に戻りません。モデルの学習時と同じ ID 列を得るには、必ず `ReversibleSentencePieceAdapter` を通してください。
|
| 63 |
+
- `encode` は BOS/EOS を付与しません。必要な場合は呼び出し側で ID 1 / 2 を追加してください。
|
| 64 |
+
- `transformers` の `AutoTokenizer` 用ファイル(`tokenizer.json` など)は同梱していません。
|
| 65 |
+
|
| 66 |
+
## ライセンス
|
| 67 |
+
|
| 68 |
+
[CC BY-NC 4.0](https://creativecommons.org/licenses/by-nc/4.0/)(表示・非営利)
|
tokenizer.model
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:73f58a921e338883470bc99c4cc364a3d5ef955a4c28b8ef62f0af508208f03f
|
| 3 |
+
size 1521458
|
tokenizer.sha256
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
73f58a921e338883470bc99c4cc364a3d5ef955a4c28b8ef62f0af508208f03f tokenizer.model
|
tokenizer_adapters.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""SentencePiece adapters for j72-tokenizer (reversible escape wrapper, escape-e000-v2)."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
class SentencePieceAdapter:
|
| 5 |
+
kind="native_sentencepiece"
|
| 6 |
+
def __init__(self, model):
|
| 7 |
+
import sentencepiece as spm
|
| 8 |
+
self.processor=spm.SentencePieceProcessor(model_file=str(model)); self.model=str(model)
|
| 9 |
+
def encode(self,text): return self.processor.encode(text,out_type=int)
|
| 10 |
+
def decode(self,ids): return self.processor.decode(ids)
|
| 11 |
+
def pieces(self,ids): return [self.processor.id_to_piece(i) for i in ids]
|
| 12 |
+
def is_byte(self,token):
|
| 13 |
+
try:
|
| 14 |
+
return bool(self.processor.is_byte(token))
|
| 15 |
+
except (TypeError, ValueError):
|
| 16 |
+
if isinstance(token, int):
|
| 17 |
+
token = self.processor.id_to_piece(token)
|
| 18 |
+
return bool(self.processor.is_byte(token))
|
| 19 |
+
def is_unk(self,token): return token==self.processor.unk_id() or token=="<unk>"
|
| 20 |
+
def unk_id(self): return self.processor.unk_id()
|
| 21 |
+
def vocabulary(self): return [{"piece":self.processor.id_to_piece(i),"score":self.processor.get_score(i)} for i in range(self.processor.get_piece_size())]
|
| 22 |
+
|
| 23 |
+
class ReversibleSentencePieceAdapter(SentencePieceAdapter):
|
| 24 |
+
"""Escape wrapper kept separate from native SP results."""
|
| 25 |
+
kind="reversible_sentencepiece_wrapper"
|
| 26 |
+
ESC="\ue000"
|
| 27 |
+
def _escape(self,text):
|
| 28 |
+
return (text.replace(self.ESC,self.ESC+self.ESC).replace("\r\n",self.ESC+"R").replace("\r",self.ESC+"r").replace("\n",self.ESC+"n").replace("\u2028",self.ESC+"L").replace("\u2029",self.ESC+"P").replace("▁",self.ESC+"s").replace("\x00",self.ESC+"0"))
|
| 29 |
+
def _unescape(self,text):
|
| 30 |
+
out=[]; i=0
|
| 31 |
+
while i<len(text):
|
| 32 |
+
if text[i]!=self.ESC: out.append(text[i]); i+=1; continue
|
| 33 |
+
if i+1>=len(text): raise ValueError("dangling sentinel")
|
| 34 |
+
c=text[i+1]; out.append(self.ESC if c==self.ESC else {"R":"\r\n","r":"\r","n":"\n","L":"\u2028","P":"\u2029","s":"▁","0":"\x00"}.get(c,self.ESC+c)); i+=2
|
| 35 |
+
return "".join(out)
|
| 36 |
+
@classmethod
|
| 37 |
+
def escape_text(cls, text):
|
| 38 |
+
return cls._escape(object.__new__(cls), text)
|
| 39 |
+
def encode(self,text): return self.processor.encode(self._escape(text),out_type=int)
|
| 40 |
+
def decode(self,ids): return self._unescape(self.processor.decode(ids))
|