Robust Hub push + throughput reporting

train_securecoder.py

@@ -729,6 +729,29 @@ def train(args, model, tokenizer, records: list[dict]):
     return trainer, stats, elapsed
 
 
+def _push_adapter(api, model, args):
+    """Unsloth's push_to_hub signature varies between releases (one version
+    rejects ``tokenizer=``), so fall back to uploading the saved folder - the
+    tokenizer files sit next to the adapter and get pushed either way."""
+    try:
+        model.push_to_hub(args.output_repo)
+        return
+    except TypeError as exc:
+        log.warning("push_to_hub rejected our arguments (%s); uploading the folder instead", exc)
+    except Exception as exc:  # noqa: BLE001 - never lose a finished run to an upload quirk
+        log.warning("model.push_to_hub failed (%s); uploading the folder instead", exc)
+    api.upload_folder(folder_path=args.output_dir, repo_id=args.output_repo, repo_type="model")
+
+
+def _push_merged(api, model, tokenizer, args):
+    try:
+        model.push_to_hub_merged(args.merge_repo, tokenizer=tokenizer, save_method="merged_16bit")
+        return
+    except TypeError as exc:
+        log.warning("push_to_hub_merged rejected tokenizer= (%s); retrying without it", exc)
+        model.push_to_hub_merged(args.merge_repo, save_method="merged_16bit")
+
+
 def save_and_push(args, model, tokenizer):
     from huggingface_hub import HfApi
 
@@ -737,13 +760,14 @@ def save_and_push(args, model, tokenizer):
 
     model.save_pretrained(args.output_dir)
     tokenizer.save_pretrained(args.output_dir)
+
     log.info("pushing LoRA adapter to %s", args.output_repo)
-    model.push_to_hub(args.output_repo, tokenizer=tokenizer)
+    _push_adapter(api, model, args)
 
     if args.merge_repo:
         api.create_repo(args.merge_repo, repo_type="model", exist_ok=True, private=args.private)
         log.info("merging to 16-bit and pushing to %s (large upload)", args.merge_repo)
-        model.push_to_hub_merged(args.merge_repo, tokenizer=tokenizer, save_method="merged_16bit")
+        _push_merged(api, model, tokenizer, args)
 
 # --------------------------------------------------------------------------
 # CLI
@@ -845,6 +869,13 @@ def main(argv=None) -> int:
     print("\n" + "=" * 78)
     print(f"DONE  rows={len(records):,}  time={elapsed / 60:.1f} min  "
           f"loss={stats_train.metrics.get('train_loss', float('nan')):.4f}")
+    speed = stats_train.metrics.get("train_samples_per_second") or 0
+    if speed:
+        est_hours = len(records) / speed / 3600
+        print(f"throughput: {speed:.1f} rows/s  ->  a 1-epoch pass over {len(records):,} rows "
+              f"≈ {est_hours:.1f} h at this rate")
+        print(f"cost at $1.80/h (l40sx1): ≈ ${est_hours * 1.80:.0f}   "
+              f"at $2.50/h (a100-large): ≈ ${est_hours * 2.50:.0f}")
     print(f"adapter: https://huggingface.co/{args.output_repo}")
     if args.merge_repo:
         print(f"merged : https://huggingface.co/{args.merge_repo}")