Merge only (GGUF in separate job)

merge_securecoder.py

@@ -1,141 +1,105 @@
-# /// script
-# requires-python = ">=3.10"
-# dependencies = [
-#     "unsloth",
-#     "transformers>=4.57",
-#     "huggingface_hub",
-# ]
-# ///
-"""Merge the SecureCoder LoRA into the base Qwen3-Coder-30B-A3B-Instruct
-checkpoint, upload a 16-bit safetensors repo, then quantise to Q4_K_M GGUF.
-
-Default base: unsloth/Qwen3-Coder-30B-A3B-Instruct
-Default adapter: Taimwe/securecoder-30b-pro
-
-Run on HF Jobs (a100-large has the headroom to load 30B in 16-bit):
-    hf jobs run -d --flavor a100-large --timeout 90m --secrets HF_TOKEN \\
-      ghcr.io/astral-sh/uv:python3.12-bookworm \\
-      uv run --no-project https://huggingface.co/Taimwe/securecoder-scripts/resolve/main/merge_securecoder.py \\
-      -- --adapter Taimwe/securecoder-30b-pro \\
-         --output-repo Taimwe/securecoder-30b-pro-merged \\
-         --gguf-repo Taimwe/securecoder-30b-pro-GGUF
-"""
-
-from __future__ import annotations
-
-import argparse
-import logging
-import os
-import shutil
-import sys
-import time
-
-logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
-log = logging.getLogger("merge")
-
-
-def parse_args() -> argparse.Namespace:
-    p = argparse.ArgumentParser(description="Merge + GGUF + push the SecureCoder LoRA")
-    p.add_argument("--base", default="unsloth/Qwen3-Coder-30B-A3B-Instruct")
-    p.add_argument("--adapter", default="Taimwe/securecoder-30b-pro")
-    p.add_argument("--output-repo", default="Taimwe/securecoder-30b-pro-merged")
-    p.add_argument("--gguf-repo", default=None)
-    p.add_argument("--gguf-quant", default="Q4_K_M")
-    p.add_argument("--private", action="store_true")
-    p.add_argument("--work-dir", default="/data/securecoder-merge")
-    p.add_argument("--max-shard-size", default="5GB")
-    return p.parse_args()
-
-def main() -> int:
-    args = parse_args()
-    token = os.environ.get("HF_TOKEN")
-    if not token:
-        log.error("HF_TOKEN not set")
-        return 1
-
-    import torch
-    from huggingface_hub import HfApi
-    from unsloth import FastLanguageModel
-
-    if not torch.cuda.is_available():
-        log.error("no CUDA - merge needs a GPU")
-        return 1
-    log.info("GPU: %s", torch.cuda.get_device_name(0))
-
-    work = args.work_dir
-    if os.path.exists(work):
-        shutil.rmtree(work)
-    os.makedirs(work, exist_ok=True)
-
-    log.info("loading base %s in 16-bit ...", args.base)
-    started = time.time()
-    model, tokenizer = FastLanguageModel.from_pretrained(
-        model_name=args.base,
-        max_seq_length=2048,
-        dtype=torch.bfloat16,
-        load_in_4bit=False,
-    )
-
-    log.info("loading adapter %s ...", args.adapter)
-    from peft import PeftModel
-    model = PeftModel.from_pretrained(model, args.adapter, token=token)
-    log.info("merging ...")
-    model = model.merge_and_unload()
-    log.info("merge done in %.1f min", (time.time() - started) / 60)
-
-    out_dir = os.path.join(work, "merged")
-    model.save_pretrained(out_dir, safe_serialization=True, max_shard_size=args.max_shard_size)
-    tokenizer.save_pretrained(out_dir)
-    log.info("saved merged model to %s", out_dir)
-
-    api = HfApi(token=token)
-    api.create_repo(args.output_repo, repo_type="model", exist_ok=True, private=args.private)
-    log.info("uploading to %s ...", args.output_repo)
-    api.upload_folder(folder_path=out_dir, repo_id=args.output_repo, repo_type="model",
-                      commit_message="Merge SecureCoder LoRA into base (16-bit)")
-    log.info("merged model live: https://huggingface.co/%s", args.output_repo)
-
-    if args.gguf_repo:
-        log.info("re-loading merged model for GGUF export ...")
-        from unsloth import FastLanguageModel as FLM
-        model, tokenizer = FLM.from_pretrained(
-            model_name=out_dir,
-            max_seq_length=2048,
-            dtype=torch.bfloat16,
-            load_in_4bit=False,
-        )
-        gguf_path = os.path.join(work, "gguf")
-        os.makedirs(gguf_path, exist_ok=True)
-        log.info("quantising to %s ...", args.gguf_quant)
-        try:
-            model.quantize_gguf_model(save_dir=gguf_path, quantization=args.gguf_quant)
-        except Exception as exc:  # noqa: BLE001
-            log.warning("model.quantize_gguf_model failed (%s); falling back to llama-quantize CLI", exc)
-            from huggingface_hub import hf_hub_download
-            from pathlib import Path as _P
-            qbin = hf_hub_download(repo_id="unsloth/llama.cpp", filename="llama-quantize",
-                                   repo_type="model", token=token)
-            import subprocess
-            subprocess.run(["chmod", "+x", qbin], check=False)
-            src = next(_P(out_dir).glob("*.gguf"), None)
-            if src is None:
-                log.error("no GGUF produced by Unsloth quantise pass")
-                return 1
-            subprocess.run([qbin, str(src), str(_P(gguf_path) / src.name), args.gguf_quant], check=True)
-
-        api.create_repo(args.gguf_repo, repo_type="model", exist_ok=True, private=args.private)
-        api.upload_folder(folder_path=gguf_path, repo_id=args.gguf_repo, repo_type="model",
-                          commit_message=f"Add {args.gguf_quant} GGUF export")
-        log.info("GGUF live: https://huggingface.co/%s", args.gguf_repo)
-
-    print("=" * 78)
-    print("MERGE COMPLETE")
-    print(f"  merged: https://huggingface.co/{args.output_repo}")
-    if args.gguf_repo:
-        print(f"  gguf  : https://huggingface.co/{args.gguf_repo}")
-    print("=" * 78)
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "unsloth",
+#     "transformers>=4.57",
+#     "huggingface_hub",
+# ]
+# ///
+"""Merge the SecureCoder LoRA into the base Qwen3-Coder-30B-A3B-Instruct
+checkpoint and upload a 16-bit safetensors repo. GGUF is done in a separate
+job (the 30B-A3B MoE does not fit on one 80 GB card when merge and quantise
+both run in the same process).
+
+Default base:    unsloth/Qwen3-Coder-30B-A3B-Instruct
+Default adapter: Taimwe/securecoder-30b-pro
+
+Run on HF Jobs:
+    hf jobs run -d --flavor a100-large --timeout 90m --secrets HF_TOKEN \\
+      ghcr.io/astral-sh/uv:python3.12-bookworm \\
+      uv run --no-project https://huggingface.co/Taimwe/securecoder-scripts/resolve/main/merge_securecoder.py \\
+      -- --adapter Taimwe/securecoder-30b-pro \\
+         --output-repo Taimwe/securecoder-30b-pro-merged
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+import shutil
+import sys
+import time
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger("merge")
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description="Merge + push the SecureCoder LoRA")
+    p.add_argument("--base", default="unsloth/Qwen3-Coder-30B-A3B-Instruct")
+    p.add_argument("--adapter", default="Taimwe/securecoder-30b-pro")
+    p.add_argument("--output-repo", default="Taimwe/securecoder-30b-pro-merged")
+    p.add_argument("--private", action="store_true")
+    p.add_argument("--work-dir", default="/data/securecoder-merge")
+    p.add_argument("--max-shard-size", default="5GB")
+    return p.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    token = os.environ.get("HF_TOKEN")
+    if not token:
+        log.error("HF_TOKEN not set")
+        return 1
+
+    import torch
+    from huggingface_hub import HfApi
+    from unsloth import FastLanguageModel
+
+    if not torch.cuda.is_available():
+        log.error("no CUDA - merge needs a GPU")
+        return 1
+    log.info("GPU: %s", torch.cuda.get_device_name(0))
+
+    work = args.work_dir
+    if os.path.exists(work):
+        shutil.rmtree(work)
+    os.makedirs(work, exist_ok=True)
+
+    log.info("loading base %s in 16-bit ...", args.base)
+    started = time.time()
+    model, tokenizer = FastLanguageModel.from_pretrained(
+        model_name=args.base,
+        max_seq_length=2048,
+        dtype=torch.bfloat16,
+        load_in_4bit=False,
+    )
+
+    log.info("loading adapter %s ...", args.adapter)
+    from peft import PeftModel
+    model = PeftModel.from_pretrained(model, args.adapter, token=token)
+    log.info("merging ...")
+    model = model.merge_and_unload()
+    log.info("merge done in %.1f min", (time.time() - started) / 60)
+
+    out_dir = os.path.join(work, "merged")
+    model.save_pretrained(out_dir, safe_serialization=True, max_shard_size=args.max_shard_size)
+    tokenizer.save_pretrained(out_dir)
+    log.info("saved merged model to %s", out_dir)
+
+    api = HfApi(token=token)
+    api.create_repo(args.output_repo, repo_type="model", exist_ok=True, private=args.private)
+    log.info("uploading to %s ...", args.output_repo)
+    api.upload_folder(folder_path=out_dir, repo_id=args.output_repo, repo_type="model",
+                      commit_message="Merge SecureCoder LoRA into base (16-bit)")
+    log.info("merged model live: https://huggingface.co/%s", args.output_repo)
+
+    print("=" * 78)
+    print("MERGE COMPLETE")
+    print(f"  merged: https://huggingface.co/{args.output_repo}")
+    print("  next  : run quantise_securecoder.py separately for GGUF")
+    print("=" * 78)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())