Upload train/train_ounce100m.py with huggingface_hub

train/train_ounce100m.py

@@ -232,16 +232,19 @@ def main():
     ap.add_argument("--out", default="/kaggle/working/run")
     ap.add_argument("--hub-repo", default="", help="dataset repo for checkpoints; empty = no upload")
     ap.add_argument("--hub-repo-type", default="dataset", choices=["dataset", "model"])
-    ap.add_argument("--seq-len", type=int, default=2048)
+    # D-011: these ARE the frozen main-run values (measured 7,732-7,828 tok/s at 1024/eager on 2xT4), so
+    # that launching the run does not depend on remembering a flag string.
+    ap.add_argument("--seq-len", type=int, default=1024)
     ap.add_argument("--attn", default="eager", choices=["eager", "sdpa", "flash_attention_2"],
                     help="p0c measured eager beating SDPA on Turing; Gate 3 P2/A-B measured +52 % for "
-                         "eager at seq 1024 and an OOM at 2048. flash_attention_2 needs sm80+ and is not "
-                         "available on a T4")
-    ap.add_argument("--grad-ckpt", action="store_true",
-                    help="trade ~30%% recompute for activation memory; the only route to eager at 2048")
+                         "eager at seq 1024 and an OOM at 2048. flash_attention_2 needs sm80+")
+    ap.add_argument("--grad-ckpt", default=True, action=argparse.BooleanOptionalAction,
+                    help="on by default: it is what keeps peak memory at ~7 GB instead of 12 GB, and the "
+                         "measured rate cost at 1024 was nil (7,828 with GC at m8 vs 7,732 without at m4)")
     ap.add_argument("--layers", type=int, default=0)
     ap.add_argument("--hidden", type=int, default=0)
-    ap.add_argument("--micro-batch", type=int, default=2)
+    ap.add_argument("--micro-batch", type=int, default=4,
+                    help="4 x 1024 x 32 accum x 2 cards = 262,144 tokens/step, the frozen batch size")
     ap.add_argument("--accum", type=int, default=32)
     ap.add_argument("--lr", type=float, default=6e-4)
     ap.add_argument("--tokens", type=int, default=1_000_000_000)