add p1_push_bench: Hub upload/download rate at checkpoint sizes

probes/p1_push_bench.py

@@ -0,0 +1,105 @@
+# Phase 1: how fast can a Kaggle job push a checkpoint to the Hub, and does it survive verification?
+#
+# Why this is a Gate 1 question and not a Phase 3 discovery. §3.13's cycle is
+# write -> upload -> VERIFY from the Hub independently -> delete local -> confirm free space, and it runs
+# at every one of the 10 checkpoints plus the rolling `latest`. Two numbers decide whether that cycle is
+# affordable: the upload rate, and the size of an exact-resume checkpoint. If a 1.6 GB checkpoint pushes in
+# 25 s the cycle is free; if it takes 20 min then 11 of them are ~3.7 h of billed GPU time against a
+# 30 h week, which changes the checkpoint cadence decision. Neither is knowable from a card or a blog.
+#
+# Sizes chosen from the real arithmetic, not convenience: 100M params in an exact-resume checkpoint is
+# fp32 weights (400 MB) + fp32 Adam m and v (800 MB) + fp32 master/grad copies depending on recipe
+# (~400 MB) + scheduler/RNG/scaler state (negligible) ~= 1.5-1.8 GB. 200 MB is the floor case (weights
+# only, i.e. what a final export costs).
+#
+# Uses the credential helper, so no token appears in this file or in any kernel source. The probe repo is
+# deleted at the end; nothing here is prior work worth keeping.
+
+import hashlib
+import json
+import os
+import time
+import urllib.error
+import urllib.request
+
+R = {}
+REPO = "Cion-lab/ounce100m-pushbench-DELETEME"
+
+
+def guard(name, fn):
+    try:
+        R[name] = fn()
+    except Exception as e:
+        R[name] = {"error": f"{type(e).__name__}: {e}"[:280]}
+
+
+import ounce100m_credentials  # noqa: E402
+
+guard("credentials", lambda: ounce100m_credentials.install(verify=True))
+tok = os.environ.get("HF_TOKEN", "")
+
+from huggingface_hub import HfApi, create_repo  # noqa: E402
+
+api = HfApi(token=tok)
+
+
+def make_repo():
+    create_repo(repo_id=REPO, token=tok, repo_type="model", private=False, exist_ok=True)
+    return REPO
+
+
+guard("repo", make_repo)
+
+
+def push(mb):
+    """Time an upload, then verify it by reading back over plain anonymous HTTPS."""
+    n = int(mb * 1024**2)
+    # Upload a deterministic buffer rather than drawing 1.6 GB of urandom: urandom is CPU-bound and
+    # would measure the RNG, not the network. Size and byte-count verification still transfer.
+    buf = b"\x5a" * n
+    t0 = time.monotonic()
+    api.upload_file(path_or_fileobj=buf, path_in_repo=f"blob_{mb}mb.bin", repo_id=REPO,
+                    repo_type="model", commit_message=f"push bench {mb} MB")
+    up = time.monotonic() - t0
+    url = f"https://huggingface.co/{REPO}/resolve/main/blob_{mb}mb.bin"
+    t1 = time.monotonic()
+    read = 0
+    with urllib.request.urlopen(url, timeout=600) as r:
+        while True:
+            chunk = r.read(4 * 1024**2)
+            if not chunk:
+                break
+            read += len(chunk)
+    down = time.monotonic() - t1
+    return {"MB": mb, "upload_s": round(up, 1), "upload_MB_per_s": round(n / up / 1024**2, 2),
+            "download_s": round(down, 1), "download_MB_per_s": round(read / down / 1024**2, 2),
+            "bytes_read_back": read, "size_matches": read == n}
+
+
+for mb in (200, 800, 1600):
+    guard(f"push_{mb}MB", lambda mb=mb: push(mb))
+
+
+def cleanup():
+    from huggingface_hub import delete_repo
+
+    delete_repo(repo_id=REPO, repo_type="model", token=tok)
+    try:
+        urllib.request.urlopen(f"https://huggingface.co/{REPO}", timeout=30)
+        return {"deleted": False, "still_resolves": True}
+    except urllib.error.HTTPError as e:
+        return {"deleted": True, "status_after": e.code}
+
+
+guard("cleanup", cleanup)
+
+R["disk_note"] = {
+    "kaggle_working_GB_total": 19.5,
+    "rule": "a checkpoint must be written somewhere, pushed, verified, then deleted before the next "
+            "one is produced; at 1.6 GB each, 10 retained checkpoints on the Hub is 16 GB of Hub "
+            "storage and at most 1-2 copies may exist on the instance at a time",
+}
+
+print("PROBE_JSON_BEGIN")
+print(json.dumps(R, indent=1, default=str))
+print("PROBE_JSON_END")