tiny-scene-three / ml /tok.py
paralaif's picture
Tiny Scene: text to animated three.js scenes with a 4.85M Transformer
dd53693 verified
Raw History Blame Contribute Delete
707 Bytes
"""Tokenizer: prompt is split into characters, the answer into scene-language tokens."""
import re
from lexicon import CHARS, SPECIAL, scene_tokens
from qa_topics import TOPICS
MAX_PROMPT = 160
CTX = 208
VOCAB = SPECIAL + list(CHARS) + scene_tokens(list(TOPICS))
ID = {t: i for i, t in enumerate(VOCAB)}
PAD, BOS, SEP, EOS = 0, 1, 2, 3
_OK = set(CHARS)
def normalize(text):
text = "".join(c for c in text.lower().replace("’", "'") if c in _OK)
return re.sub(r"\s+", " ", text).strip()[:MAX_PROMPT]
def encode_prompt(text):
return [BOS] + [ID[c] for c in normalize(text)] + [SEP]
def encode_pair(text, tokens):
p = encode_prompt(text)
return p, [ID[t] for t in tokens] + [EOS]