File size: 5,513 Bytes
d9dfe3c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | # /// script
# requires-python = ">=3.10"
# dependencies = [
# "unsloth",
# "transformers>=4.57",
# "huggingface_hub",
# ]
# ///
"""Merge the SecureCoder LoRA into the base Qwen3-Coder-30B-A3B-Instruct
checkpoint, upload a 16-bit safetensors repo, then quantise to Q4_K_M GGUF.
Default base: unsloth/Qwen3-Coder-30B-A3B-Instruct
Default adapter: Taimwe/securecoder-30b-pro
Run on HF Jobs (a100-large has the headroom to load 30B in 16-bit):
hf jobs run -d --flavor a100-large --timeout 90m --secrets HF_TOKEN \\
ghcr.io/astral-sh/uv:python3.12-bookworm \\
uv run --no-project https://huggingface.co/Taimwe/securecoder-scripts/resolve/main/merge_securecoder.py \\
-- --adapter Taimwe/securecoder-30b-pro \\
--output-repo Taimwe/securecoder-30b-pro-merged \\
--gguf-repo Taimwe/securecoder-30b-pro-GGUF
"""
from __future__ import annotations
import argparse
import logging
import os
import shutil
import sys
import time
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("merge")
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description="Merge + GGUF + push the SecureCoder LoRA")
p.add_argument("--base", default="unsloth/Qwen3-Coder-30B-A3B-Instruct")
p.add_argument("--adapter", default="Taimwe/securecoder-30b-pro")
p.add_argument("--output-repo", default="Taimwe/securecoder-30b-pro-merged")
p.add_argument("--gguf-repo", default=None)
p.add_argument("--gguf-quant", default="Q4_K_M")
p.add_argument("--private", action="store_true")
p.add_argument("--work-dir", default="/data/securecoder-merge")
p.add_argument("--max-shard-size", default="5GB")
return p.parse_args()
def main() -> int:
args = parse_args()
token = os.environ.get("HF_TOKEN")
if not token:
log.error("HF_TOKEN not set")
return 1
import torch
from huggingface_hub import HfApi
from unsloth import FastLanguageModel
if not torch.cuda.is_available():
log.error("no CUDA - merge needs a GPU")
return 1
log.info("GPU: %s", torch.cuda.get_device_name(0))
work = args.work_dir
if os.path.exists(work):
shutil.rmtree(work)
os.makedirs(work, exist_ok=True)
log.info("loading base %s in 16-bit ...", args.base)
started = time.time()
model, tokenizer = FastLanguageModel.from_pretrained(
model_name=args.base,
max_seq_length=2048,
dtype=torch.bfloat16,
load_in_4bit=False,
)
log.info("loading adapter %s ...", args.adapter)
from peft import PeftModel
model = PeftModel.from_pretrained(model, args.adapter, token=token)
log.info("merging ...")
model = model.merge_and_unload()
log.info("merge done in %.1f min", (time.time() - started) / 60)
out_dir = os.path.join(work, "merged")
model.save_pretrained(out_dir, safe_serialization=True, max_shard_size=args.max_shard_size)
tokenizer.save_pretrained(out_dir)
log.info("saved merged model to %s", out_dir)
api = HfApi(token=token)
api.create_repo(args.output_repo, repo_type="model", exist_ok=True, private=args.private)
log.info("uploading to %s ...", args.output_repo)
api.upload_folder(folder_path=out_dir, repo_id=args.output_repo, repo_type="model",
commit_message="Merge SecureCoder LoRA into base (16-bit)")
log.info("merged model live: https://huggingface.co/%s", args.output_repo)
if args.gguf_repo:
log.info("re-loading merged model for GGUF export ...")
from unsloth import FastLanguageModel as FLM
model, tokenizer = FLM.from_pretrained(
model_name=out_dir,
max_seq_length=2048,
dtype=torch.bfloat16,
load_in_4bit=False,
)
gguf_path = os.path.join(work, "gguf")
os.makedirs(gguf_path, exist_ok=True)
log.info("quantising to %s ...", args.gguf_quant)
try:
model.quantize_gguf_model(save_dir=gguf_path, quantization=args.gguf_quant)
except Exception as exc: # noqa: BLE001
log.warning("model.quantize_gguf_model failed (%s); falling back to llama-quantize CLI", exc)
from huggingface_hub import hf_hub_download
from pathlib import Path as _P
qbin = hf_hub_download(repo_id="unsloth/llama.cpp", filename="llama-quantize",
repo_type="model", token=token)
import subprocess
subprocess.run(["chmod", "+x", qbin], check=False)
src = next(_P(out_dir).glob("*.gguf"), None)
if src is None:
log.error("no GGUF produced by Unsloth quantise pass")
return 1
subprocess.run([qbin, str(src), str(_P(gguf_path) / src.name), args.gguf_quant], check=True)
api.create_repo(args.gguf_repo, repo_type="model", exist_ok=True, private=args.private)
api.upload_folder(folder_path=gguf_path, repo_id=args.gguf_repo, repo_type="model",
commit_message=f"Add {args.gguf_quant} GGUF export")
log.info("GGUF live: https://huggingface.co/%s", args.gguf_repo)
print("=" * 78)
print("MERGE COMPLETE")
print(f" merged: https://huggingface.co/{args.output_repo}")
if args.gguf_repo:
print(f" gguf : https://huggingface.co/{args.gguf_repo}")
print("=" * 78)
return 0
if __name__ == "__main__":
raise SystemExit(main())
|