train: preflight -- static-review fixes: absolute-step cursor, hub_listing pagination, cursor required on resume, repo_type everywhere, rank-0 repo create

train/preflight.py

@@ -67,9 +67,13 @@ from transformers import TrainingArguments
 import train_ounce100m as T
 R = {"transformers": transformers.__version__, "torch": torch.__version__}
 cfg = T.build_config(2048)
-R["config"] = {k: getattr(cfg, k) for k in ("hidden_size", "num_hidden_layers", "num_attention_heads",
-                                            "num_key_value_heads", "intermediate_size", "vocab_size",
-                                            "tie_word_embeddings", "max_position_embeddings")}
+R["config"] = {k: getattr(cfg, k, "<absent>") for k in
+               ("hidden_size", "num_hidden_layers", "num_attention_heads", "num_key_value_heads",
+                "intermediate_size", "vocab_size", "tie_word_embeddings", "max_position_embeddings",
+                # rope_theta moved into rope_parameters in newer transformers; if the flat attribute is
+                # gone the value we think we froze may not be the value the model got (verify, not assume)
+                "rope_theta", "rope_parameters", "rms_norm_eps", "attention_dropout", "mlp_bias",
+                "hidden_act")}
 t0 = time.time()
 m = T.LlamaForCausalLM(cfg)
 R["params"] = T.count_params(m)
@@ -77,6 +81,12 @@ R["init_seconds"] = round(time.time() - t0, 1)
 R["shapes_20L"] = T.count_params(T.LlamaForCausalLM(T.build_config(1024, layers=20)))
 c20 = T.build_config(1024, layers=20)
 R["shape20_ffn"] = c20.intermediate_size
+R["shape20_kv"] = c20.num_key_value_heads
+# Regression test for the defect that would have made T1 meaningless: passing --hidden 576 explicitly is
+# the frozen width, so it must NOT fall through to the variant branch and become MHA-9 with a doubled FFN.
+cv = T.build_config(2048, hidden=576)
+R["hidden576_still_frozen"] = (cv.num_key_value_heads == 3 and cv.intermediate_size == 1536
+                               and cv.num_hidden_layers == 22)
 # The exact kwargs the trainer passes. If the image rejects one, this is where we learn it.
 kw = dict(output_dir="/kaggle/working/argcheck", per_device_train_batch_size=2,
           gradient_accumulation_steps=32, learning_rate=6e-4, weight_decay=0.1, adam_beta1=0.9,
@@ -161,6 +171,7 @@ _emit(R)
                            # FFN must stay 1536; a doubled FFN would make test T1 meaningless.
                            and j.get("shapes_20L", {}).get("sum_numel") == 99114048
                            and j.get("shape20_ffn") == 1536 and shape_ok
+                           and j.get("hidden576_still_frozen") is True
                            and (j.get("forward_backward") or {}).get("finite"))
     print("VERDICT PARGS_pass=", R["PARGS_pass"], "blocks=", R["PARGS_n_blocks"], "shape_ok=", shape_ok,
           json.dumps({k: j.get(k) for k in ("transformers", "torch", "kwargs", "params", "shapes_20L",
@@ -294,9 +305,10 @@ print("HUB_JSON_END")
 
 # ------------------------------------------------------------------------ P2..P4 (GPU)
 def torchrun(args, extra, timeout=7200):
-    return sh(["torchrun", "--nproc_per_node=2", "train_ounce100m.py", "--root",
-               WORK + "/mixroot", "--seq-len", str(args.seq_len)] + extra,
-              timeout=timeout, label="torchrun " + " ".join(extra[:6]))
+    # --seq-len is deliberately NOT set here: each caller passes its own, and relying on argparse's
+    # last-wins to undo a duplicate is a defect waiting for a reordering.
+    return sh(["torchrun", "--nproc_per_node=2", "train_ounce100m.py", "--root", WORK + "/mixroot"]
+              + extra, timeout=timeout, label="torchrun " + " ".join(extra[:6]))
 
 
 def p2_throughput(args, R):
@@ -349,7 +361,8 @@ def p3_cold_resume(args, R):
     """Short run -> wipe every local trace -> resume with --resume auto, which must recover the exact
     sample position from the Hub. Loss continuity is the pass condition: a restart that silently
     re-initialised would jump the loss."""
-    common = ["--tokens", str(args.tokens), "--accum", str(args.accum), "--micro-batch", "2",
+    common = ["--seq-len", str(args.seq_len), "--tokens", str(args.tokens),
+              "--accum", str(args.accum), "--micro-batch", "2",
               "--hub-repo", args.ckpt_repo, "--prune", "--log-every", "5"]
     a = torchrun(args, common + ["--max-steps", str(args.steps_a), "--out", WORK + "/p3"],
                  timeout=9000)