add credential helper + Hub write/bulk-upload roundtrip probe

ounce100m_credentials.py

@@ -0,0 +1,88 @@
+"""Runtime credentials for ounce100m jobs, fetched from the account's own private Kaggle dataset.
+
+Why this exists. §2 forbids the HF token reaching a public artifact -- job logs, model cards, or any
+code mirrored to the Hub -- and requires it to come "from the environment or a secret store". Internet-
+enabled Kaggle sessions arrive already authenticated against the Kaggle API (verified: CLI 2.0.2
+preinstalled, `auth_method: ACCESS_TOKEN`), and `dodosoomro/ounce100m-secret-store` is a private dataset
+that is anonymously unreachable and absent from public search. So a job can retrieve its own credential
+at run time: no token in kernel source, no token in the public code repo, no token in any log.
+
+Note that `datasetDataSources` mounting was tried and did NOT populate /kaggle/input (see
+memory/ERRORS.md), so this module downloads instead of mounting. Download also has the advantage of
+working in a freshly created kernel with no metadata to keep in sync.
+
+Rules for callers:
+  * never print, log, or write the token to /kaggle/working -- working-dir files become kernel outputs
+  * call `install()` once at process start, before any huggingface_hub import
+  * the staging directory is under /tmp, which is not published as an artifact
+"""
+
+import json
+import os
+import subprocess
+import zipfile
+
+DS_SLUG = "ounce100m-secret-store"
+DS_ID = f"dodosoomro/{DS_SLUG}"
+STAGE = "/tmp/.ounce100m"  # deliberately not under /kaggle/working
+CREDS = "credentials.json"
+
+
+class CredentialError(RuntimeError):
+    pass
+
+
+def _download():
+    os.makedirs(STAGE, exist_ok=True)
+    os.chmod(STAGE, 0o700)
+    r = subprocess.run(
+        ["kaggle", "datasets", "download", "-d", DS_ID, "-p", STAGE, "--unzip"],
+        capture_output=True, text=True, timeout=300)
+    if r.returncode != 0:
+        # Scrubbed: an error string from the CLI could otherwise echo the token into a retained log.
+        raise CredentialError(f"kaggle datasets download rc={r.returncode}")
+    path = os.path.join(STAGE, CREDS)
+    if not os.path.exists(path):
+        # --unzip flattens differently across CLI versions; fall back to opening the archive directly.
+        zipped = os.path.join(STAGE, f"{DS_SLUG}.zip")
+        if os.path.exists(zipped):
+            with zipfile.ZipFile(zipped) as z:
+                z.extract(CREDS, STAGE)
+            path = os.path.join(STAGE, CREDS)
+    if not os.path.exists(path):
+        raise CredentialError(f"{CREDS} not found after download; staged: {sorted(os.listdir(STAGE))}")
+    return path
+
+
+def token():
+    """The HF token as a string. Callers must not print it."""
+    path = os.environ.get("OUNCE100M_CRED_PATH")  # lets a mounting job hand us a path instead
+    if not path:
+        path = _download()
+    with open(path) as f:
+        blob = json.load(f)
+    tok = blob.get("HF_TOKEN", "")
+    if not tok.startswith("hf_") or len(tok) < 20:
+        raise CredentialError("credential file present but HF_TOKEN malformed -- refusing to continue")
+    return tok
+
+
+def install(verify=False):
+    """Put the token where huggingface_hub expects it. Returns a *safe to print* summary only."""
+    import hashlib
+
+    tok = token()
+    os.environ["HF_TOKEN"] = tok
+    os.environ["HUGGING_FACE_HUB_TOKEN"] = tok
+    out = {"source": DS_ID, "length": len(tok),
+           "sha256_prefix": hashlib.sha256(tok.encode()).hexdigest()[:12]}
+    if verify:
+        from huggingface_hub import HfApi
+
+        me = HfApi(token=tok).whoami()
+        out["hub_user"] = me.get("name")
+    return out
+
+
+if __name__ == "__main__":
+    print(json.dumps(install(verify=True), indent=1))

probes/p1_credential_roundtrip.py

@@ -0,0 +1,123 @@
+# Phase 1: prove the credential path end to end, then prove a job can WRITE to the Hub.
+#
+# This is the test that closes D-002. The token arrives via `ounce100m_credentials`, which downloads it
+# from the account's own private Kaggle dataset using the container's built-in Kaggle auth -- so nothing
+# here, and nothing in any kernel source or mirrored repo, contains a credential.
+#
+# The write test matters more than it looks: §3.13's whole cycle (write -> upload -> verify from the Hub
+# -> delete local -> confirm free space) is unimplementable if a job cannot push. It is verified by
+# reading the file back over plain anonymous HTTPS rather than through the client that wrote it, so a
+# library-level false success cannot be mistaken for a working upload path.
+#
+# Zero GPU quota. No token printed anywhere.
+
+import hashlib
+import json
+import os
+import subprocess
+import time
+import urllib.error
+import urllib.request
+
+R = {}
+TEST_REPO = "Cion-lab/ounce100m-authtest-DELETEME"
+
+
+def guard(name, fn):
+    try:
+        R[name] = fn()
+    except Exception as e:
+        R[name] = {"error": f"{type(e).__name__}: {e}"[:300]}
+
+
+guard("credentials_module", lambda: __import__("ounce100m_credentials").install(verify=True))
+
+tok = os.environ.get("HF_TOKEN", "")
+R["token_in_env_after_install"] = {"present": bool(tok), "length": len(tok)}
+
+
+def staging_is_clean():
+    # /kaggle/working files are published as kernel output artifacts; a credential must never land there.
+    listing = sorted(os.listdir("/kaggle/working"))
+    leaked = [f for f in listing if "credential" in f.lower() or "token" in f.lower()]
+    return {"working_listing": listing, "credential_in_working_dir": leaked}
+
+
+guard("artifact_hygiene", staging_is_clean)
+
+
+def write_roundtrip():
+    from huggingface_hub import HfApi, create_repo, delete_repo
+
+    api = HfApi(token=tok)
+    t0 = time.monotonic()
+    # hub 1.11 (the image's version) has no `namespace` kwarg on create_repo -- the owner comes from
+    # the repo_id. Passing namespace= was what broke the earlier attempt (E-013).
+    create_repo(repo_id=TEST_REPO, token=tok, repo_type="model", private=False, exist_ok=True)
+    payload = b"ounce100m credential round-trip probe\n"
+    api.upload_file(path_or_fileobj=payload, path_in_repo="authcheck.txt",
+                    repo_id=TEST_REPO, repo_type="model", commit_message="auth write check")
+    url = f"https://huggingface.co/{TEST_REPO}/resolve/main/authcheck.txt"
+    readback = urllib.request.urlopen(url, timeout=60).read()
+    verified = readback.strip() == payload.strip()
+    sha = hashlib.sha256(readback).hexdigest()[:12]
+    info = api.repo_info(TEST_REPO, repo_type="model")
+    delete_repo(repo_id=TEST_REPO, repo_type="model", token=tok)
+    gone = False
+    try:
+        urllib.request.urlopen(url, timeout=30)
+    except urllib.error.HTTPError as e:
+        gone = e.code in (404, 401, 403)
+    except Exception:
+        pass
+    return {"created": True, "bytes": len(payload), "readback_matches": verified,
+            "readback_sha256_prefix": sha, "revision": getattr(info, "sha", None),
+            "seconds": round(time.monotonic() - t0, 1), "deleted": True,
+            "delete_confirmed_gone": gone}
+
+
+guard("hub_write_roundtrip", write_roundtrip)
+
+# A throughput datapoint for the thing the main run actually does: push a ~200 MB blob and time it.
+# Checkpoint size arithmetic (§Phase 1 open item) is meaningless without knowing the upload rate.
+def bulk_upload():
+    from huggingface_hub import HfApi
+
+    api = HfApi(token=tok)
+    blob = os.urandom(200 * 1024**2)
+    sha = hashlib.sha256(blob).hexdigest()
+    t0 = time.monotonic()
+    api.upload_file(path_or_fileobj=blob, path_in_repo="bulk.bin", repo_id=TEST_REPO,
+                    repo_type="model", commit_message="bulk upload timing probe")
+    up = time.monotonic() - t0
+    url = f"https://huggingface.co/{TEST_REPO}/resolve/main/bulk.bin"
+    t1 = time.monotonic()
+    got = urllib.request.urlopen(url, timeout=300).read()
+    down = time.monotonic() - t1
+    return {"MB": round(len(blob) / 1024**2), "upload_s": round(up, 1),
+            "upload_MB_per_s": round(len(blob) / up / 1024**2, 1),
+            "download_s": round(down, 1), "download_MB_per_s": round(len(got) / down / 1024**2, 1),
+            "sha_matches": hashlib.sha256(got).hexdigest() == sha,
+            "note": "LFS threshold may route this differently than a real multi-GB checkpoint"}
+
+
+guard("bulk_upload_rate", bulk_upload)
+
+
+def cleanup():
+    from huggingface_hub import delete_repo
+
+    delete_repo(repo_id=TEST_REPO, repo_type="model", token=tok)
+    return "deleted"
+
+
+guard("cleanup_delete", cleanup)
+
+R["staging_dir"] = {"path": "/tmp/.ounce100m",
+                    "exists": os.path.isdir("/tmp/.ounce100m"),
+                    "mode": oct(os.stat("/tmp/.ounce100m").st_mode & 0o777)
+                    if os.path.isdir("/tmp/.ounce100m") else None}
+
+print("PROBE_JSON_BEGIN")
+print(json.dumps(R, indent=1, default=str))
+print("PROBE_JSON_END")