fix(jobs): bump bitsandbytes to 0.47 (cu128 binary) so GPU dequant works in HF Job runtime image

scripts/hf_jobs_helper.py

@@ -0,0 +1,74 @@
+"""Tiny helper to query HF Jobs API directly (bypasses CLI rate-limit on /whoami-v2).
+
+Usage:
+    python scripts/hf_jobs_helper.py list           # list recent jobs (newest first)
+    python scripts/hf_jobs_helper.py inspect <id>   # full inspect of one job
+    python scripts/hf_jobs_helper.py logs <id>      # stream logs of one job
+
+Requires HF_TOKEN env var.
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+import urllib.request
+
+API = "https://huggingface.co/api/jobs"
+USER = "Pratham-math"
+
+
+def _auth_headers() -> dict:
+    tok = os.environ.get("HF_TOKEN", "")
+    if not tok:
+        raise SystemExit("HF_TOKEN env var not set")
+    return {"Authorization": f"Bearer {tok}"}
+
+
+def _get(url: str) -> str:
+    req = urllib.request.Request(url, headers=_auth_headers())
+    with urllib.request.urlopen(req, timeout=30) as r:
+        return r.read().decode("utf-8", errors="replace")
+
+
+def list_jobs() -> None:
+    body = _get(f"{API}/{USER}?limit=10")
+    jobs = json.loads(body)
+    for j in jobs:
+        st = (j.get("status") or {}).get("stage", "?")
+        msg = (j.get("status") or {}).get("message", "")
+        flavor = j.get("flavor", "?")
+        created = j.get("createdAt", "?")
+        img = (j.get("dockerImage") or "")[:55]
+        print(f"{j['id']} | {st:>10} | {flavor:>14} | {created} | {img}")
+        if msg:
+            print(f"   msg: {msg}")
+
+
+def inspect(job_id: str) -> None:
+    body = _get(f"{API}/{USER}/{job_id}")
+    print(json.dumps(json.loads(body), indent=2))
+
+
+def logs(job_id: str) -> None:
+    # Use HfApi.fetch_job_logs (SSE-aware) but pass namespace explicitly so it
+    # never calls /whoami-v2 (which is rate-limited).
+    from huggingface_hub import HfApi
+
+    api = HfApi(token=os.environ["HF_TOKEN"])
+    out = sys.stdout.buffer  # write raw bytes to dodge cp1252 on Windows
+    for line in api.fetch_job_logs(job_id=job_id, namespace=USER, follow=False):
+        out.write((line + "\n").encode("utf-8", errors="replace"))
+    out.flush()
+
+
+if __name__ == "__main__":
+    cmd = sys.argv[1] if len(sys.argv) > 1 else "list"
+    if cmd == "list":
+        list_jobs()
+    elif cmd == "inspect":
+        inspect(sys.argv[2])
+    elif cmd == "logs":
+        logs(sys.argv[2])
+    else:
+        raise SystemExit(f"unknown command: {cmd}")

scripts/job_train.sh

@@ -20,7 +20,10 @@ pip install -q hydra-core omegaconf wandb 'huggingface_hub>=0.28' tyro
 # Core ML deps.
 # NOTE: trl 1.2 imports `is_trackio_available` from transformers, which is not
 # present in 4.49.0. Use a newer transformers line in jobs.
-pip install -q transformers==4.56.2 accelerate==1.5.2 peft==0.14.0 bitsandbytes==0.45.1
+# bitsandbytes 0.45.1 only ships up to cuda124 binary; vllm 0.18.0 pulls torch
+# with the cu128 wheel, so we MUST use a bnb release that ships libbitsandbytes_cuda128.so
+# (0.47.0 is the first stable line that does). Confirmed against PyPI Apr 2026.
+pip install -q transformers==4.56.2 accelerate==1.5.2 peft==0.14.0 'bitsandbytes>=0.47.0,<0.48'
 pip install -q datasets==4.7.0
 # Keep TRL pinned for OpenEnv path but bypass resolver deadlock with datasets pin.
 pip install -q --no-deps trl==1.2.0