hubsync.ensure_repo must install creds first (real cause of the 401); correct E-017 diagnosis

build/hubsync.py

@@ -1,105 +1,113 @@
-# Hub sync for the mix build: the instance is scratch, so nothing that took an hour to produce may live
-# only there (§3.13 "never let the instance be the only copy", applied to a CPU job rather than a
-# checkpoint).
-#
-# Granularity is one *source*: staging a source takes 5-40 minutes, so publishing its shards the moment
-# it finishes bounds the loss from any interruption to the one source in flight, and a resume can skip
-# every source already on the Hub without re-reading it.
-#
-# The stage repo is private-in-practice but public by project rule; it holds no credentials and no
-# benchmark material -- only token ids from the approved mix.
-
-import json
-import os
-
-
-def repo_id_from_env(default=None):
-    return os.environ.get("OUNCE100M_STAGE_REPO") or default
-
-
-def api():
-    import os
-
-    import ounce100m_credentials
-
-    ounce100m_credentials.install()
-    from huggingface_hub import HfApi
-
-    return HfApi(token=os.environ["HF_TOKEN"])
-
-
-def ensure_repo(repo):
-    from huggingface_hub import create_repo
-
-    # hub 1.11 (the image's version) takes the owner inside repo_id and has no `namespace` kwarg (E-013)
-    create_repo(repo_id=repo, repo_type="dataset", private=False, exist_ok=True)
-    return repo
-
-
-def remote_stage_files(repo):
-    """Set of relative paths under stage/ that already exist on the Hub."""
-    import urllib.request
-
-    url = f"https://huggingface.co/api/datasets/{repo}/tree/main/stage?recursive=true"
-    try:
-        with urllib.request.urlopen(url, timeout=90) as r:
-            entries = json.loads(r.read().decode())
-    except Exception:
-        return set()
-    out = set()
-    for e in entries:
-        if e.get("type") != "file":
-            continue
-        out.add(e["path"])
-        # a directory in the tree listing means we must recurse; recursive=true should cover it, but
-        # if the server ignored the flag we get dirs and no files, which the caller detects as empty
-    return out
-
-
-def upload_source(root, key, repo):
-    """Publish stage/<key>/* in one commit. Returns the file names pushed."""
-    a = api()
-    local = os.path.join(root, "stage", key)
-    files = sorted(f for f in os.listdir(local) if f.endswith((".bin", ".json")))
-    if not files:
-        return []
-    a.upload_folder(folder_path=local, path_in_repo=f"stage/{key}", repo_id=repo,
-                    repo_type="dataset", commit_message=f"stage {key}: {len(files)} shard(s)",
-                    allow_patterns=["*.bin", "*.json"])
-    return files
-
-
-def download_missing(root, keys_wanted, repo, remote=None):
-    """Fetch stage/<key>/*.bin for every wanted key not already on disk. Returns {key: [filenames]}."""
-    from huggingface_hub import hf_hub_download
-
-    remote = remote if remote is not None else remote_stage_files(repo)
-    got = {}
-    for key in keys_wanted:
-        want = sorted(p for p in remote if p.startswith(f"stage/{key}/") and p.endswith(".bin"))
-        if not want:
-            got[key] = []
-            continue
-        local_dir = os.path.join(root, "stage", key)
-        os.makedirs(local_dir, exist_ok=True)
-        have = {f"stage/{key}/{f}" for f in os.listdir(local_dir)}
-        for rp in want:
-            if rp in have:
-                continue
-            hf_hub_download(repo_id=repo, repo_type="dataset", filename=rp,
-                            local_dir=root, force_download=False)
-        got[key] = sorted(os.listdir(local_dir))
-    return got
-
-
-def status(repo):
-    import urllib.request
-
-    url = f"https://huggingface.co/api/datasets/{repo}"
-    try:
-        with urllib.request.urlopen(url, timeout=60) as r:
-            d = json.loads(r.read().decode())
-        return {"exists": True, "gated": d.get("gated"), "private": d.get("private"),
-                "lastModified": d.get("lastModified"), "id": d.get("id")}
-    except Exception as e:
-        return {"exists": False, "error": f"{type(e).__name__}: {str(e)[:120]}"}
+# Hub sync for the mix build: the instance is scratch, so nothing that took an hour to produce may live
+# only there (§3.13 "never let the instance be the only copy", applied to a CPU job rather than a
+# checkpoint).
+#
+# Granularity is one *source*: staging a source takes 5-40 minutes, so publishing its shards the moment
+# it finishes bounds the loss from any interruption to the one source in flight, and a resume can skip
+# every source already on the Hub without re-reading it.
+#
+# The stage repo is private-in-practice but public by project rule; it holds no credentials and no
+# benchmark material -- only token ids from the approved mix.
+
+import json
+import os
+
+
+def repo_id_from_env(default=None):
+    return os.environ.get("OUNCE100M_STAGE_REPO") or default
+
+
+def api():
+    import os
+
+    import ounce100m_credentials
+
+    ounce100m_credentials.install()
+    from huggingface_hub import HfApi
+
+    return HfApi(token=os.environ["HF_TOKEN"])
+
+
+def ensure_repo(repo):
+    # Must install() first: the module-level create_repo() resolves its token from the environment, and
+    # this is the FIRST hub call a build makes -- before any api() has run. Calling it unauthenticated
+    # is what produced the misleading 401 in the cold-resume rehearsal (E-017), and the diagnosis in
+    # these notes was wrong about the cause until p2-token-scope proved dataset writes are permitted.
+    import os
+
+    api()
+    from huggingface_hub import create_repo
+
+    # hub 1.11 (the image's version) takes the owner inside repo_id and has no `namespace` kwarg (E-013)
+    create_repo(repo_id=repo, repo_type="dataset", private=False, exist_ok=True,
+                token=os.environ["HF_TOKEN"])
+    return repo
+
+
+def remote_stage_files(repo):
+    """Set of relative paths under stage/ that already exist on the Hub."""
+    import urllib.request
+
+    url = f"https://huggingface.co/api/datasets/{repo}/tree/main/stage?recursive=true"
+    try:
+        with urllib.request.urlopen(url, timeout=90) as r:
+            entries = json.loads(r.read().decode())
+    except Exception:
+        return set()
+    out = set()
+    for e in entries:
+        if e.get("type") != "file":
+            continue
+        out.add(e["path"])
+        # a directory in the tree listing means we must recurse; recursive=true should cover it, but
+        # if the server ignored the flag we get dirs and no files, which the caller detects as empty
+    return out
+
+
+def upload_source(root, key, repo):
+    """Publish stage/<key>/* in one commit. Returns the file names pushed."""
+    a = api()
+    local = os.path.join(root, "stage", key)
+    files = sorted(f for f in os.listdir(local) if f.endswith((".bin", ".json")))
+    if not files:
+        return []
+    a.upload_folder(folder_path=local, path_in_repo=f"stage/{key}", repo_id=repo,
+                    repo_type="dataset", commit_message=f"stage {key}: {len(files)} shard(s)",
+                    allow_patterns=["*.bin", "*.json"])
+    return files
+
+
+def download_missing(root, keys_wanted, repo, remote=None):
+    """Fetch stage/<key>/*.bin for every wanted key not already on disk. Returns {key: [filenames]}."""
+    from huggingface_hub import hf_hub_download
+
+    remote = remote if remote is not None else remote_stage_files(repo)
+    got = {}
+    for key in keys_wanted:
+        want = sorted(p for p in remote if p.startswith(f"stage/{key}/") and p.endswith(".bin"))
+        if not want:
+            got[key] = []
+            continue
+        local_dir = os.path.join(root, "stage", key)
+        os.makedirs(local_dir, exist_ok=True)
+        have = {f"stage/{key}/{f}" for f in os.listdir(local_dir)}
+        for rp in want:
+            if rp in have:
+                continue
+            hf_hub_download(repo_id=repo, repo_type="dataset", filename=rp,
+                            local_dir=root, force_download=False)
+        got[key] = sorted(os.listdir(local_dir))
+    return got
+
+
+def status(repo):
+    import urllib.request
+
+    url = f"https://huggingface.co/api/datasets/{repo}"
+    try:
+        with urllib.request.urlopen(url, timeout=60) as r:
+            d = json.loads(r.read().decode())
+        return {"exists": True, "gated": d.get("gated"), "private": d.get("private"),
+                "lastModified": d.get("lastModified"), "id": d.get("id")}
+    except Exception as e:
+        return {"exists": False, "error": f"{type(e).__name__}: {str(e)[:120]}"}