"""Tokenizer: prompt is split into characters, the answer into scene-language tokens.""" import re from lexicon import CHARS, SPECIAL, scene_tokens from qa_topics import TOPICS MAX_PROMPT = 160 CTX = 208 VOCAB = SPECIAL + list(CHARS) + scene_tokens(list(TOPICS)) ID = {t: i for i, t in enumerate(VOCAB)} PAD, BOS, SEP, EOS = 0, 1, 2, 3 _OK = set(CHARS) def normalize(text): text = "".join(c for c in text.lower().replace("’", "'") if c in _OK) return re.sub(r"\s+", " ", text).strip()[:MAX_PROMPT] def encode_prompt(text): return [BOS] + [ID[c] for c in normalize(text)] + [SEP] def encode_pair(text, tokens): p = encode_prompt(text) return p, [ID[t] for t in tokens] + [EOS]