push artifacts/scripts/hf_push.py

artifacts/scripts/hf_push.py

@@ -2,19 +2,23 @@
 """Push Ingot (SN97) build-pod artifacts to HF repo eyad-silx/progress.
 
 Modes: --small (docs/logs/scripts), --big (GGUFs + QAT states), default = both.
-Only uploads new/changed files (size+mtime_ns fingerprint), records sha256 in
-state for the MANIFEST. State: /root/.hf_push_state.json. Log: /root/hf_push.log.
+Only uploads new/changed files (size+mtime_ns fingerprint). Files rejected by the
+free-plan private-storage cap are retried on the overflow repo (silx-ai/Progress,
+org quota) and, if that also fails, recorded as pending with sha256 in the
+MANIFEST (bytes stay on the pod + local rescue cache).
+State: /root/.hf_push_state.json. Log: /root/hf_push.log.
 """
-import argparse, glob, hashlib, json, os, sys
+import argparse, fcntl, glob, hashlib, json, os, sys
 from datetime import datetime, timezone
 from pathlib import Path
 from huggingface_hub import HfApi
 
 REPO = "eyad-silx/progress"
-# Free-plan private-storage cap (~20 GiB) on eyad-silx fills up with 3 GGUFs.
+# Free-plan private-storage caps (~20 GiB user eyad-silx, org silx-ai also full).
 # Files that hit the cap overflow to this private org repo (token has repo.write):
 OVERFLOW_REPO = "silx-ai/Progress"
 OVERFLOW_PREFIX = "ingot-backup/"
+
 TOKEN = Path("/root/.hf_token").read_text().strip()
 STATE_P = Path("/root/.hf_push_state.json")
 LOG_P = Path("/root/hf_push.log")
@@ -30,10 +34,14 @@ Pushed by `hf_push.py` — checks hourly and uploads any new/changed artifact.
 - `states/` — QAT state checkpoints (`t0..tN.safetensors`)
 - `artifacts/` — `HANDOFF.md`, build/train/eval scripts, run logs, sample draws,
   measurement results (`measure*.jsonl`, `pick*.jsonl`, `drawn.jsonl`)
-- `MANIFEST.md` — size + sha256 + upload time of every artifact (regenerated each push)
+- `MANIFEST.md` — size + sha256 + upload time of every artifact (regenerated each push),
+  including a "Pending" section for files blocked by the free-plan private-storage cap
+  (their bytes are preserved on the build pod and in a local rescue cache)
 - `PROGRESS.md` — push history
 
 The GGUFs are ~6.8 GB each (PQ2_0 quantized 27B linear-attention models).
+Note: chal-identity's GGUF is byte-identical to king-ix's (sha256 in MANIFEST) — the
+identity refit test round-trips the king exactly.
 """
 
 def now():
@@ -97,7 +105,15 @@ def main():
     else:
         tracks = small_tracks() + big_tracks()
 
+    lockf = open("/root/.hf_push.lock", "w")
+    try:
+        fcntl.flock(lockf, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except BlockingIOError:
+        log("skipped", reason="another push run in progress")
+        sys.exit(0)
+
     state = load_state()
+    pending = state.get("_pending", {})
     api = HfApi(token=TOKEN)
     results = []
     for local, repo in tracks:
@@ -106,9 +122,10 @@ def main():
         st = os.stat(local)
         fp = f"{st.st_size}:{st.st_mtime_ns}"
         prev = state.get(repo)
-        if prev and prev["fp"] == fp:
+        if prev and prev.get("fp") == fp:
             results.append((repo, "unchanged"))
             continue
+        d = None
         try:
             d = sha256(local)
             dest = REPO
@@ -116,8 +133,7 @@ def main():
                 api.upload_file(path_or_fileobj=local, path_in_repo=repo, repo_id=REPO,
                                 repo_type="model", commit_message=f"push {repo}")
             except Exception as e1:
-                msg = str(e1)
-                if "storage limit" not in msg.lower():
+                if "storage limit" not in str(e1).lower():
                     raise
                 api.upload_file(path_or_fileobj=local, path_in_repo=OVERFLOW_PREFIX + repo,
                                 repo_id=OVERFLOW_REPO, repo_type="model",
@@ -125,24 +141,41 @@ def main():
                 dest = OVERFLOW_REPO
                 log("overflowed", local=local, repo=repo, to=OVERFLOW_REPO, sha256=d, size=st.st_size)
             state[repo] = {"fp": fp, "sha256": d, "size": st.st_size, "uploaded_at": now(), "dest": dest}
+            pending.pop(repo, None)
+            state["_pending"] = pending
             save_state(state)
             if dest == REPO:
                 log("uploaded", local=local, repo=repo, sha256=d, size=st.st_size)
             results.append((repo, "uploaded"))
         except Exception as e:
-            log("error", local=local, repo=repo, err=repr(e)[:500])
+            entry = {"fp": fp, "size": st.st_size, "blocked_at": now()}
+            if d:
+                entry["sha256"] = d
+            pending[repo] = entry
+            state["_pending"] = pending
+            save_state(state)
+            log("error", local=local, repo=repo, err=repr(e)[:400])
             results.append((repo, "FAILED"))
 
     # regenerate docs (manifest / progress / readme) from state
     try:
         manifest = ["# Manifest — eyad-silx/progress (+ overflow repo silx-ai/Progress)", "",
-                    "Files with `stored in` = silx-ai/Progress live at `ingot-backup/<repo path>` in that repo",
-                    "(eyad-silx free-plan private-storage cap ~20 GiB; overflow keeps everything private).", "",
+                    "Files with `stored in` = silx-ai/Progress live at `ingot-backup/<repo path>` in that repo.",
+                    "Free-plan private-storage caps (~20 GiB user eyad-silx; org silx-ai also at cap) block",
+                    "the rest — see the Pending section; those bytes live on the build pod and in a local",
+                    "rescue cache, and push automatically once a cap is lifted (plan upgrade / space freed).", "",
                     "| repo path | size (bytes) | sha256 | uploaded (UTC) | stored in |",
                     "|---|---|---|---|---|"]
-        for repo in sorted(state):
+        for repo in sorted(k for k in state if not k.startswith("_")):
             v = state[repo]
             manifest.append(f"| `{repo}` | {v['size']:,} | `{v['sha256']}` | {v['uploaded_at']} | {v.get('dest', REPO)} |")
+        if pending:
+            manifest += ["", "## Pending — blocked by private-storage quota", "",
+                         "| repo path | size (bytes) | sha256 | blocked (UTC) |",
+                         "|---|---|---|---|"]
+            for repo in sorted(pending):
+                v = pending[repo]
+                manifest.append(f"| `{repo}` | {v['size']:,} | `{v.get('sha256', '(hash pending)')}` | {v['blocked_at']} |")
         Path("/tmp/MANIFEST.md").write_text("\n".join(manifest) + "\n")
         api.upload_file(path_or_fileobj="/tmp/MANIFEST.md", path_in_repo="MANIFEST.md",
                         repo_id=REPO, repo_type="model", commit_message="update MANIFEST.md")
@@ -163,7 +196,8 @@ def main():
     n_up = sum(1 for r in results if r[1] == "uploaded")
     n_un = sum(1 for r in results if r[1] == "unchanged")
     n_fail = sum(1 for r in results if r[1] == "FAILED")
-    log("run_done", uploaded=n_up, unchanged=n_un, failed=n_fail)
+    log("run_done", uploaded=n_up, unchanged=n_un, failed=n_fail,
+        pending=len(pending))
     sys.exit(1 if n_fail else 0)
 
 if __name__ == "__main__":