File size: 1,818 Bytes
09255bb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | #!/usr/bin/env python3
"""Decode tern_tc token ids to text with the published 8K tokenizer.
Pure-python GPT-2 byte-level BPE decoder over results/data8k/tokenizer.json
(no dependencies beyond the standard library) -- the same file the C engine
and the harness use.
Usage:
python3 scripts/decode_ids.py 1 3 204 276 405 659 85 1516
python3 scripts/decode_ids.py --specials 1 3 204 276 405 659 85 1516
"""
import json
import os
import sys
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
def _byte_decoder():
cs = list(range(ord("!"), ord("~") + 1)) + list(range(ord("\xa1"), ord("\xac") + 1)) \
+ list(range(ord("\xae"), ord("\xff") + 1))
n = 0
table = {}
for b in range(2 ** 8):
if b in cs:
table[chr(b)] = b
else:
table[chr(256 + n)] = b
n += 1
return table
BYTE_DECODER = _byte_decoder()
def decode(ids, tokenizer, skip_specials=True):
vocab = tokenizer["model"]["vocab"]
id2tok = {v: k for k, v in vocab.items()}
special = {a["id"]: a["content"] for a in tokenizer.get("added_tokens", [])}
parts = []
for i in ids:
if i in special:
if not skip_specials:
parts.append(special[i])
continue
parts.append(id2tok[i])
joined = "".join(parts)
return bytes(BYTE_DECODER[c] for c in joined).decode("utf-8", errors="replace")
def main():
args = sys.argv[1:]
skip = True
if args and args[0] == "--specials":
skip = False
args = args[1:]
if not args:
print(__doc__)
sys.exit(1)
ids = [int(a) for a in args]
tok = json.load(open(os.path.join(ROOT, "results/data8k/tokenizer.json")))
print(decode(ids, tok, skip_specials=skip))
if __name__ == "__main__":
main()
|