train: train_ounce100m -- hubckpt.ensure_repo writes LFS attributes before the first checkpoint push (found by P1)

train/train_ounce100m.py

@@ -243,10 +243,10 @@ def main():
         ounce100m_credentials.install()
         from huggingface_hub import HfApi
         api = HfApi()
-        # Create-if-absent: an upload to a repo nobody made yet is a failure that lands 10 minutes into
-        # the first checkpoint, on a GPU session that is being billed while it waits.
-        api.create_repo(repo_id=args.hub_repo, repo_type=args.hub_repo_type, exist_ok=True,
-                        token=os.environ.get("HF_TOKEN"))
+        # create-if-absent AND LFS patterns: see hubckpt.ensure_repo for the failure this prevents
+        say("ckpt repo:", json.dumps(hubckpt.ensure_repo(args.hub_repo, api,
+                                                         os.environ.get("HF_TOKEN"),
+                                                         args.hub_repo_type)))
         say(f"checkpoint hub target {args.hub_repo} (prune={args.prune}, every {ckpt_every} steps)")
 
     tacc = TrainingArguments(