Upload build/publish_mix.py with huggingface_hub

build/publish_mix.py

@@ -171,6 +171,17 @@ def main():
         p = os.path.join(a.root, extra)
         if os.path.exists(p):
             to_upload.append((extra, p))
+    # The contamination masks are part of the mix's definition: without the exact exclusion set, replaying
+    # build_mix over the same staged sources produces a different -- less clean -- mix (§3.3, §3.12). They
+    # are ~1 byte per staged document, so shipping them costs ~1 MB.
+    fdir = os.path.join(a.root, "filter")
+    if man.get("contamination_filter") and os.path.isdir(fdir):
+        for name in sorted(os.listdir(fdir)):
+            to_upload.append((f"filter/{name}", os.path.join(fdir, name)))
+    elif man.get("contamination_filter"):
+        print("FATAL: manifest records a contamination filter but mixroot/filter/ is absent -- "
+              "this mix would be published unreproducible", flush=True)
+        sys.exit(9)
     print(f"uploading {len(to_upload)} files to {a.repo}", flush=True)
     if a.dry_run:
         for np, lp in to_upload: