publish: ship tokenizer.json with the mix, sha-checked against the manifest

build/publish_mix.py

@@ -4,6 +4,7 @@ import argparse
 import hashlib
 import json
 import os
+import shutil
 import struct
 import sys
 import urllib.request
@@ -152,7 +153,20 @@ def main():
     for grp in ("shards", "val_shards"):
         for rec in man.get(grp, []):
             to_upload.append((rec["file"], os.path.join(a.root, rec["file"])))
-    for extra in ("manifest.json", "README.md", "ATTRIBUTION.md", "audit.json",
+    # Ship the exact tokenizer bytes the mix was cut with, verified against the manifest's recorded
+    # sha256. A dataset of token ids is only reproducible alongside the tokenizer that produced them,
+    # and "download it from SmolLM2-135M and hope the file never changed" is not reproducibility (§3.12).
+    tp = os.path.join(a.root, "tokenizer.json")
+    if not os.path.exists(tp):
+        from huggingface_hub import hf_hub_download
+        shutil.copyfile(hf_hub_download("HuggingFaceTB/SmolLM2-135M", "tokenizer.json"), tp)
+    got = hashlib.sha256(open(tp, "rb").read()).hexdigest()
+    want_sha = man["tokenizer"]["sha256"]
+    if got != want_sha:
+        print(f"FATAL: tokenizer.json sha {got[:16]} != manifest {want_sha[:16]} -- the ids in this mix "
+              "were not produced by the tokenizer being published with them", flush=True)
+        sys.exit(8)
+    for extra in ("manifest.json", "README.md", "ATTRIBUTION.md", "audit.json", "tokenizer.json",
                   "build_state.json"):
         p = os.path.join(a.root, extra)
         if os.path.exists(p):