mirror probes/ so the kept code is self-contained

probes/tok_shape.py

@@ -0,0 +1,13 @@
+import json
+
+d = json.load(open("tmp_tok.json", encoding="utf-8"))
+v = (d.get("model") or {}).get("vocab") or {}
+lt, gt = chr(60), chr(62)
+unk = lt + "unk" + gt
+endo = lt + "endoftext" + gt
+print("model type:", (d.get("model") or {}).get("type"), "| vocab size:", len(v))
+print("added ids:", [t.get("id") for t in d.get("added_tokens", [])])
+print("added:", [t.get("content") for t in d.get("added_tokens", [])])
+print("unk in vocab:", unk in v, "| endoftext in vocab:", endo in v)
+print("id of endoftext:", v.get(endo))
+print("contiguous 0..N-1:", sorted(int(i) for i in v.values()) == list(range(len(v))))

probes/tok_unk_check.py

@@ -0,0 +1,20 @@
+import json
+
+vocab = json.load(open("tmp_tok.json", encoding="utf-8"))["model"]["vocab"]
+
+
+def content(d, key):
+    v = d.get(key)
+    return v.get("content") if isinstance(v, dict) else v
+
+
+for f in ("code/config/smolm2_tokenizer_config.json", "code/config/smolm2_special_tokens_map.json"):
+    d = json.load(open(f, encoding="utf-8"))
+    u = content(d, "unk_token")
+    e = content(d, "eos_token")
+    b = content(d, "bos_token")
+    print(f)
+    print("   unk:", repr(u), "in_vocab:", (u in vocab) if u else None,
+          "-> would null it:", bool(u) and u not in vocab)
+    print("   eos:", repr(e), "in_vocab:", (e in vocab) if e else None, "id:", vocab.get(e) if e else None)
+    print("   bos:", repr(b), "in_vocab:", (b in vocab) if b else None, "id:", vocab.get(b) if b else None)