DoL-1.2-Base / Tokenizer /generic_bpe /byte_fallback.py
GABIRI's picture
Add DoL 1.2 model card and inference runtime
431ffa3 verified
Raw History Blame Contribute Delete
1.53 kB
# -*- coding: utf-8 -*-
"""Byte-fallback helpers shared by tokenizer tracks."""
from __future__ import annotations
from Tokenizer.unified.encoded import EncodedToken
def byte_token(byte: int) -> str:
return f"<0x{byte:02X}>"
def is_byte_token(token: str) -> bool:
return token.startswith("<0x") and token.endswith(">") and len(token) == 6
def decode_byte_token(token: str) -> int:
return int(token[3:5], 16)
def encode_byte_fallback(
text: str,
vocab: dict[str, int],
unk_id: int,
base_start: int = 0,
track: str = "misc",
) -> list[EncodedToken]:
tokens: list[EncodedToken] = []
for rel_pos, ch in enumerate(text):
direct = vocab.get(ch)
start = base_start + rel_pos
end = start + 1
if direct is not None:
tokens.append(EncodedToken(direct, ch, track, start, end))
continue
# ``surrogatepass`` keeps encoding from raising on lone surrogates
# (U+D800–U+DFFF), which appear in scraped/raw web text. A strict
# ``encode`` would raise UnicodeEncodeError and abort the whole shard;
# decode mirrors this with ``errors="replace"``.
for byte in ch.encode("utf-8", "surrogatepass"):
tok = byte_token(byte)
tokens.append(EncodedToken(vocab.get(tok, unk_id), tok, track, start, end))
return tokens
def decode_bytes(byte_tokens: list[str]) -> str:
return bytes(decode_byte_token(token) for token in byte_tokens).decode(
"utf-8", errors="replace"
)