pin: kernels/phase4_session.py (RepoMissing resume branch, disk gate, PHASE4_PREP_ONLY)

kernels/phase4_session.py

@@ -47,6 +47,7 @@ PLANNING_TOK_PER_S = float(os.environ.get("PLANNING_TOK_PER_S", "7732"))
 SESSION_GPU_HOURS = float(os.environ.get("SESSION_GPU_HOURS", "11.0"))
 QUOTA_LEFT_HOURS = float(os.environ.get("QUOTA_LEFT_HOURS", "30.0"))
 RESERVE_HOURS = float(os.environ.get("RESERVE_HOURS", "0.6"))  # startup, val eval, the last push
+MIN_FREE_GB = float(os.environ.get("MIN_FREE_GB", "8"))       # mix 2.3 GB + two checkpoints + slack
 
 
 def sh(argv, label, timeout=None, env=None):
@@ -141,22 +142,38 @@ def main():
         "shard_dataset.py": ("train/shard_dataset.py",
                              "4111832e79445fb0ffce6a73a02da4c97255d6c7886762682e04a417f2695a78"),
         "hubckpt.py": ("train/hubckpt.py",
-                       "7a250e8232dc99d39108913f169c290e1ef03744217d71bc2dcf833f5d784c53"),
+                       "c8c958417dc48db92f8be5c4b505581c746be9cc10110a358230ac47c480d5bf"),
         "train_ounce100m.py": ("train/train_ounce100m.py",
-                               "c8462cac0cabebc65bcfdcd62be3ed53b03e08120e3a6eb9ca2c4dc235cace42"),
+                               "487edc1be54f03b5f1a312901580cc3e1504c412322ed074ceff869febbfb8bc"),
     })
 
     import ounce100m_credentials
     print("creds:", json.dumps(ounce100m_credentials.install(verify=True)), flush=True)
     rc, out = sh(["bash", "-c", "nvidia-smi --query-gpu=name,memory.used,memory.total "
-                     "--format=csv,noheader; df -h /kaggle/working | tail -1; free -g | head -2"],
+                     "--format=csv,noheader; df -k /kaggle/working | tail -1; free -g | head -2"],
                  "env")
+    # `df -k` not `-h`: the human-readable form prints "318G"/"500M"/"1.5T" and a float() on those raises,
+    # which would take down the session before training even starts.
     free_gb = None
     for line in out.splitlines():
         if "overlay" in line or line.startswith("/dev/"):
             parts = line.split()
             if len(parts) >= 4:
-                free_gb = parts[3]
+                try:
+                    free_gb = float(parts[3]) / 1048576.0
+                except ValueError:
+                    free_gb = None
+    if free_gb is None:
+        print("VERDICT REFUSED_TO_START: could not read the free space on /kaggle/working from "
+              "df output -- refusing to guess whether the mix and two checkpoints will fit", flush=True)
+        raise SystemExit(10)
+    if free_gb < MIN_FREE_GB:
+        print(f"VERDICT REFUSED_TO_START: {free_gb:.1f} GB free on /kaggle/working, this run needs at "
+              f"least {MIN_FREE_GB} GB (2.3 GB mix + ~1.7 GB per checkpoint + the unpacked resume dir). "
+              "A Kaggle GPU instance starts with ~20 GB, so a low number here means something is left "
+              "over from a previous session on this box.", flush=True)
+        raise SystemExit(10)
+    print(f"disk: {free_gb:.1f} GB free", flush=True)
 
     # The Hub pointer is the only truth about where the run is (never local state -- §3.13).
     import hubckpt
@@ -209,6 +226,14 @@ def main():
               "burn quota on work that cannot be resumed from. Raise SESSION_GPU_HOURS or wait for "
               "quota.", flush=True)
         raise SystemExit(8)
+    if os.environ.get("PHASE4_PREP_ONLY") == "1":
+        # Everything a session can get wrong before it bills GPU time has now been checked: the pinned
+        # code hashes, the credentials resolve, the checkpoint pointer is readable, the published mix is
+        # downloadable and large enough, and the plan lands on a checkpoint boundary. A CPU instance can
+        # run all of that for free (E-031's arithmetic is exactly the kind of thing to find there).
+        print("VERDICT PREP_ONLY_OK stop_after_steps", pl["stop_after_steps"],
+              "intervals", pl["intervals_this_session"], flush=True)
+        return
 
     argv = [sys.executable, "-u", "train_ounce100m.py",
             "--root", f"{WORK}/mixroot", "--out", f"{WORK}/run",