Upload modeling_erk_linear.py with huggingface_hub

modeling_erk_linear.py

@@ -0,0 +1,63 @@
+"""
+Erk-Linear — Erk-14B'nin 8 dikkat katmanini Gated DeltaNet'e damitan %20-lineer hibrit.
+
+Yukleme:
+    # gerekli: pip install torch transformers flash-linear-attention safetensors huggingface_hub
+    from modeling_erk_linear import load_erk_linear
+    model, tokenizer = load_erk_linear()          # Erk-14B tabanini + GDN agirliklarini indirir
+    out = model.generate(**tokenizer("Merhaba", return_tensors="pt").to(model.device))
+
+Model, Qwen3-14B mimarisine dayanir; 8 katmanin softmax dikkati subquadratic Gated DeltaNet ile
+degistirilmis, kalan 32 katman softmax "cipa" olarak korunmustur. Ayrinti: teknik rapor / GitHub.
+"""
+import torch
+import torch.nn as nn
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from safetensors.torch import load_file
+from huggingface_hub import hf_hub_download
+
+BASE_MODEL = "ecloudtech/Erk-14B"     # Qwen3-14B temelli Turkce model
+REPO_ID = "ecloudtech/Erk-Linear"
+GDN_LAYERS = [1, 3, 5, 7, 10, 36, 38, 39]   # %20 lineer, yayilmis yerlesim
+
+
+class _GDNAttention(nn.Module):
+    """Qwen3 self_attn cagri imzasiyla uyumlu Gated DeltaNet sarmalayici."""
+    def __init__(self, gdn):
+        super().__init__()
+        self.gdn = gdn
+
+    def forward(self, hidden_states, *args, **kwargs):
+        out = self.gdn(hidden_states)
+        y = out[0] if isinstance(out, tuple) else out
+        return (y, None)
+
+
+def load_erk_linear(device="cuda", dtype=torch.bfloat16,
+                    base_model=BASE_MODEL, repo_id=REPO_ID):
+    """Erk-Linear hibridini kurar ve (model, tokenizer) doner."""
+    from fla.layers import GatedDeltaNet  # flash-linear-attention
+
+    model = AutoModelForCausalLM.from_pretrained(base_model, torch_dtype=dtype).to(device).eval()
+    H = model.config.hidden_size
+
+    gdn_path = hf_hub_download(repo_id=repo_id, filename="gdn_weights.safetensors")
+    state = load_file(gdn_path)
+
+    for li in GDN_LAYERS:
+        gdn = GatedDeltaNet(hidden_size=H, head_dim=128, num_heads=40,
+                            use_gate=True, use_short_conv=True, mode="chunk")
+        prefix = f"L{li}."
+        layer_sd = {k[len(prefix):]: v for k, v in state.items() if k.startswith(prefix)}
+        gdn.load_state_dict(layer_sd)
+        gdn = gdn.to(device).to(dtype).eval()
+        model.model.layers[li].self_attn = _GDNAttention(gdn).to(device).to(dtype)
+
+    tokenizer = AutoTokenizer.from_pretrained(base_model)
+    return model, tokenizer
+
+
+if __name__ == "__main__":
+    m, t = load_erk_linear()
+    ids = t("Türkiye'nin başkenti", return_tensors="pt").to(m.device)
+    print(t.decode(m.generate(**ids, max_new_tokens=12)[0], skip_special_tokens=True))