probes/tok_shape.py

import json

d = json.load(open("tmp_tok.json", encoding="utf-8"))
v = (d.get("model") or {}).get("vocab") or {}
lt, gt = chr(60), chr(62)
unk = lt + "unk" + gt
endo = lt + "endoftext" + gt
print("model type:", (d.get("model") or {}).get("type"), "| vocab size:", len(v))
print("added ids:", [t.get("id") for t in d.get("added_tokens", [])])
print("added:", [t.get("content") for t in d.get("added_tokens", [])])
print("unk in vocab:", unk in v, "| endoftext in vocab:", endo in v)
print("id of endoftext:", v.get(endo))
print("contiguous 0..N-1:", sorted(int(i) for i in v.values()) == list(range(len(v))))