Download ml/tok.py from paralaif/tiny-scene-three: direct link, hf CLI and curl.
- Browser
- Download file 707 Bytes
-
https://huggingface.co/paralaif/tiny-scene-three/resolve/main/ml/tok.py
- Command line
-
hf download hf://paralaif/tiny-scene-three/ml/tok.py
-
curl -L -o tok.py https://huggingface.co/paralaif/tiny-scene-three/resolve/main/ml/tok.py
707 Bytes
| """Tokenizer: prompt is split into characters, the answer into scene-language tokens.""" | |
| import re | |
| from lexicon import CHARS, SPECIAL, scene_tokens | |
| from qa_topics import TOPICS | |
| MAX_PROMPT = 160 | |
| CTX = 208 | |
| VOCAB = SPECIAL + list(CHARS) + scene_tokens(list(TOPICS)) | |
| ID = {t: i for i, t in enumerate(VOCAB)} | |
| PAD, BOS, SEP, EOS = 0, 1, 2, 3 | |
| _OK = set(CHARS) | |
| def normalize(text): | |
| text = "".join(c for c in text.lower().replace("’", "'") if c in _OK) | |
| return re.sub(r"\s+", " ", text).strip()[:MAX_PROMPT] | |
| def encode_prompt(text): | |
| return [BOS] + [ID[c] for c in normalize(text)] + [SEP] | |
| def encode_pair(text, tokens): | |
| p = encode_prompt(text) | |
| return p, [ID[t] for t in tokens] + [EOS] | |