publish_model.py: ship tokenizer_config.json + special_tokens_map.json (Gate 5 could not load a bare tokenizer.json)

build/publish_model.py

@@ -266,6 +266,38 @@ def main():
         # Gate 5 could never run (found by running it, 2026-09-21T13:40Z).
         shutil.copy(os.path.join(a.src, "tokenizer.json"), src)     # ship the tokenizer that cut the mix
         have.add("tokenizer.json")
+    # CARD_FILES already promised tokenizer_config.json and special_tokens_map.json, but nothing ever put
+    # them in the export, so the repo shipped a bare tokenizer.json. Without them AutoTokenizer falls back
+    # to a literal unknown-token name that this BPE vocab does not contain, so decoding raised
+    # "Unk token not found in the vocabulary" and Gate 5 could not load the tokenizer (2026-09-21T13:48Z).
+    # They are fetched from the tokenizer's own hub repo, so the published tokenizer is byte-complete.
+    TOK_SOURCE = "HuggingFaceTB/SmolLM2-135M"
+    tk = man.get("tokenizer")                       # manifest may hold a string or a mapping
+    if isinstance(tk, dict):
+        TOK_SOURCE = tk.get("hub") or tk.get("repo") or TOK_SOURCE
+    elif isinstance(tk, str) and "/" in tk:
+        TOK_SOURCE = tk
+    print("tokenizer source for config files:", TOK_SOURCE, flush=True)
+    for extra in ("tokenizer_config.json", "special_tokens_map.json"):
+        if extra in have:
+            continue
+        blob = urllib.request.urlopen(f"https://huggingface.co/{TOK_SOURCE}/resolve/main/{extra}",
+                                       timeout=120).read()
+        dec = json.loads(blob.decode("utf-8"))
+        u = dec.get("unk_token")
+        u = (u.get("content") if isinstance(u, dict) else u)
+        if u and u not in json.load(open(os.path.join(src, "tokenizer.json"), encoding="utf-8")
+                                    )["model"]["vocab"]:
+            dec["unk_token"] = None        # never declare a token the vocab lacks
+            blob = json.dumps(dec, indent=2, ensure_ascii=False).encode("utf-8")
+        with open(os.path.join(src, extra), "wb") as fh:
+            fh.write(blob)
+        have.add(extra)
+        print("tokenizer config", extra, len(blob), "bytes sha256",
+              sha(os.path.join(src, extra))[:16], flush=True)
+    for required in ("tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"):
+        if required not in have:
+            raise SystemExit(f"tokenizer is incomplete: {required} is missing from the export")
     cfg = json.load(open(os.path.join(src, "config.json")))
     run = {}
     if os.path.exists(os.path.join(src, "run_summary.json")):