use one of these scripts for merging loras to Gemma 4 safetensor

PMPF5_finetune_adapter_mergeG4.py

@@ -0,0 +1,77 @@
+import os
+import torch
+from peft import PeftModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import shutil
+
+# ================= CONFIGURATION =================
+BASE_MODEL_PATH = os.getcwd()              # Where the original safetensors are
+ADAPTER_PATH = "finetuned_adapter"         # Where the LoRA files are
+NEW_MODEL_PATH = "My_Finetuned_Model_v1"   # The new subfolder for the result
+# =================================================
+
+def main():
+    # Force CPU processing to save local VRAM during execution
+    os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+    
+    print(f"Loading base model from {BASE_MODEL_PATH}...")
+    
+    # 1. Load the tokenizer and preserve vocabulary
+    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL_PATH, local_files_only=True)
+    vocab_size = tokenizer.vocab_size # Explicitly retrieve 262144 from tokenizer config
+    print(f"Detected tokenizer vocabulary size: {vocab_size}")
+
+    # 2. Load the base model with nested text config routing
+    base_model = AutoModelForCausalLM.from_pretrained(
+        BASE_MODEL_PATH,
+        low_cpu_mem_usage=True,
+        return_dict=True,
+        torch_dtype=torch.bfloat16, # Match model dtype from config
+        device_map="cpu",
+        local_files_only=True
+    )
+
+    # 3. Ensure the base model vocabulary exactly matches the 262144 layout
+    print(f"Aligning base model token embeddings to: {vocab_size}...")
+    base_model.resize_token_embeddings(vocab_size)
+    
+    print(f"Loading adapter from {ADAPTER_PATH}...")
+    # We load the model with custom configuration overrides to force-map 
+    # the adapter directly to the underlying language_model layers.
+    model = PeftModel.from_pretrained(
+        base_model, 
+        ADAPTER_PATH, 
+        adapter_name="default",
+        is_trainable=False,
+    )
+
+    print("Merging weights (this may take a minute)...")
+    # Using merge_and_unload to execute the matrix calculations safely.
+    # We explicitly load state dict first to resolve any path mismatches 
+    # between the base model and the target PEFT adapter.
+    model.base_model.model.load_state_dict(model.base_model.model.state_dict(), strict=False)
+    model = model.merge_and_unload()
+
+    print(f"Saving merged model to {NEW_MODEL_PATH}...")
+    if not os.path.exists(NEW_MODEL_PATH):
+        os.makedirs(NEW_MODEL_PATH)
+
+    model.save_pretrained(
+        NEW_MODEL_PATH, 
+        safe_serialization=True, 
+        max_shard_size="5GB"
+    )
+    
+    print("Saving tokenizer...")
+    tokenizer.save_pretrained(NEW_MODEL_PATH)
+
+    # Copy generation config if it exists
+    gen_config = os.path.join(BASE_MODEL_PATH, "generation_config.json")
+    if os.path.exists(gen_config):
+        shutil.copy(gen_config, os.path.join(NEW_MODEL_PATH, "generation_config.json"))
+
+    print("Merge complete!")
+    print(f"You can now use '{NEW_MODEL_PATH}' in Kobold, Ooba, or MergeKit.")
+
+if __name__ == "__main__":
+    main()

PMPF_finetune_adapter_merge_G4_12B.py

@@ -0,0 +1,117 @@
+import os
+import gc
+import json
+import shutil
+import torch
+from safetensors.torch import load_file, save_file
+
+BASE_MODEL_PATH = os.getcwd()
+ADAPTER_PATH = "finetuned_adapter"
+NEW_MODEL_PATH = "My_Finetuned_Model_v1"
+
+def main():
+    print("=" * 80)
+    print("   GEMMA 4 DIRECT SAFETENSORS MERGER (BYPASSES CLIPPABLELINEAR)")
+    print("=" * 80)
+
+    # 1. Read LoRA Config
+    config_path = os.path.join(ADAPTER_PATH, "adapter_config.json")
+    with open(config_path, "r", encoding="utf-8") as f:
+        adapter_config = json.load(f)
+
+    r = adapter_config.get("r", 32)
+    alpha = adapter_config.get("lora_alpha", 64)
+    scaling = alpha / r
+    print(f"LoRA Config: Rank={r}, Alpha={alpha}, Scaling={scaling}")
+
+    # 2. Load Adapter Tensors
+    adapter_file = os.path.join(ADAPTER_PATH, "adapter_model.safetensors")
+    adapter_tensors = load_file(adapter_file, device="cpu")
+    print(f"Loaded {len(adapter_tensors)} adapter tensors.")
+
+    # 3. Compute Delta Matrices
+    deltas = {}
+    for k in list(adapter_tensors.keys()):
+        if ".lora_A.weight" in k:
+            b_key = k.replace(".lora_A.weight", ".lora_B.weight")
+            if b_key in adapter_tensors:
+                # Raw base target suffix
+                raw_target = k.replace("base_model.model.", "").replace(".lora_A.weight", ".weight")
+                
+                A = adapter_tensors[k].to(torch.float32)  # [r, in_features]
+                B = adapter_tensors[b_key].to(torch.float32)  # [out_features, r]
+                delta = (B @ A) * scaling  # [out_features, in_features]
+                
+                deltas[raw_target] = delta
+
+    print(f"Compiled {len(deltas)} LoRA delta updates.")
+
+    # 4. Stream and Merge Base Shards
+    shard_files = sorted([f for f in os.listdir(BASE_MODEL_PATH) if f.startswith("model") and f.endswith(".safetensors")])
+    if not os.path.exists(NEW_MODEL_PATH):
+        os.makedirs(NEW_MODEL_PATH)
+
+    total_applied = 0
+    for shard_name in shard_files:
+        in_shard = os.path.join(BASE_MODEL_PATH, shard_name)
+        out_shard = os.path.join(NEW_MODEL_PATH, shard_name)
+        
+        base_weights = load_file(in_shard, device="cpu")
+        merged_weights = {}
+        shard_applied = 0
+
+        for k, v in base_weights.items():
+            delta = None
+            
+            # 1. Direct match
+            if k in deltas:
+                delta = deltas[k]
+            else:
+                # 2. Suffix match (resolves language_model prefix differences)
+                for d_k, d_v in deltas.items():
+                    if k.endswith(d_k) or d_k.endswith(k):
+                        delta = d_v
+                        break
+            
+            # 3. Tied weights fallback for embed_tokens / lm_head
+            if delta is None and "embed_tokens" in k:
+                for d_k, d_v in deltas.items():
+                    if "lm_head" in d_k:
+                        delta = d_v
+                        break
+
+            if delta is not None:
+                if delta.shape == v.shape:
+                    merged_weights[k] = (v.to(torch.float32) + delta).to(v.dtype)
+                    shard_applied += 1
+                    total_applied += 1
+                elif delta.T.shape == v.shape:
+                    merged_weights[k] = (v.to(torch.float32) + delta.T).to(v.dtype)
+                    shard_applied += 1
+                    total_applied += 1
+                else:
+                    print(f"⚠️ Shape mismatch for {k}: base {v.shape} vs delta {delta.shape}")
+                    merged_weights[k] = v
+            else:
+                merged_weights[k] = v
+
+        print(f"Shard {shard_name}: Applied {shard_applied} deltas.")
+        save_file(merged_weights, out_shard, metadata={"format": "pt"})
+        del base_weights
+        del merged_weights
+        gc.collect()
+
+    print(f"\n🎉 MERGE COMPLETE! Successfully applied {total_applied} layer deltas.")
+
+    # 5. Copy Tokenizer and Configs
+    print("Copying configuration files...")
+    for item in os.listdir(BASE_MODEL_PATH):
+        if item.endswith((".json", ".jinja", ".txt", ".model", ".tiktoken")):
+            src = os.path.join(BASE_MODEL_PATH, item)
+            dst = os.path.join(NEW_MODEL_PATH, item)
+            shutil.copy(src, dst)
+
+    print(f"Ready for conversion! Point convert_hf_to_gguf.py to: {NEW_MODEL_PATH}")
+
+if __name__ == "__main__":
+    main()