Upload build/publish_mix.py with huggingface_hub

build/publish_mix.py

@@ -154,6 +154,18 @@ def main():
     if int(aud.get("mix_tokens_audited") or 0) != int(man.get("total_tokens") or -1):
         problems.append(f"audit covered {aud.get('mix_tokens_audited')} tokens but the manifest holds "
                         f"{man.get('total_tokens')}")
+    # The token total alone cannot tell two mixes of the same size apart, and a filtered rebuild can
+    # produce exactly that (review finding). Recompute the digest over the shard list the same way the
+    # audit did and require equality, so a stale audit.json cannot certify a mix it never scanned.
+    want_digest = hashlib.sha256(json.dumps(
+        [[s["file"], s.get("tokens"), s.get("docs")] for s in man["shards"]] +
+        [["VAL:" + s["file"], s.get("tokens"), s.get("docs")] for s in man.get("val_shards", [])],
+        sort_keys=True).encode()).hexdigest()
+    if aud.get("mix_bytes_sha256") != want_digest:
+        problems.append(f"audit.json is bound to a different mix: its digest is "
+                        f"{str(aud.get('mix_bytes_sha256'))[:16]}, this manifest is {want_digest[:16]}")
+    if int(aud.get("val_documents_audited") or 0) <= 0:
+        problems.append("the audit never scanned the held-out validation shards")
     if problems:
         print("FATAL: contamination gate refuses publication:", flush=True)
         for p in problems: