Initial verified release: safetensors + modeling + tokenizer (4K YaRN)

config.json

@@ -0,0 +1,27 @@
+{
+  "model_type": "picolm_v3",
+  "auto_map": {
+    "AutoConfig": "configuration_picolm_v3.PicoLMV3Config",
+    "AutoModelForCausalLM": "modeling_picolm_v3.PicoLMV3ForCausalLM"
+  },
+  "vocab_size": 24576,
+  "emb_dim": 128,
+  "dim": 576,
+  "n_layers": 21,
+  "layer_repeat": 2,
+  "n_heads": 9,
+  "n_kv_heads": 3,
+  "head_dim": 64,
+  "intermediate_dim": 1664,
+  "max_seq_len": 2048,
+  "orig_seq_len": 2048,
+  "norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "pad_token_id": 3,
+  "tie_word_embeddings": true,
+  "architectures": [
+    "PicoLMV3ForCausalLM"
+  ]
+}

configuration_picolm_v3.py

@@ -0,0 +1,46 @@
+from transformers import PretrainedConfig
+
+class PicoLMV3Config(PretrainedConfig):
+    model_type = "picolm_v3"
+
+    def __init__(
+        self,
+        vocab_size: int = 24576,
+        emb_dim: int = 128,
+        dim: int = 576,
+        n_layers: int = 21,
+        layer_repeat: int = 2,
+        n_heads: int = 9,
+        n_kv_heads: int = 3,
+        head_dim: int = 64,
+        intermediate_dim: int = 1664,
+        max_seq_len: int = 2048,
+        orig_seq_len: int = 2048,
+        norm_eps: float = 1e-5,
+        rope_theta: float = 10000.0,
+        bos_token_id: int = 0,
+        eos_token_id: int = 2,
+        pad_token_id: int = 3,
+        tie_word_embeddings: bool = True,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.emb_dim = emb_dim
+        self.dim = dim
+        self.n_layers = n_layers
+        self.layer_repeat = layer_repeat
+        self.n_heads = n_heads
+        self.n_kv_heads = n_kv_heads
+        self.head_dim = head_dim
+        self.intermediate_dim = intermediate_dim
+        self.max_seq_len = max_seq_len
+        self.orig_seq_len = orig_seq_len
+        self.norm_eps = norm_eps
+        self.rope_theta = rope_theta
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            pad_token_id=pad_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b64eafc3dbb91f28a139f86bdf153890055207f5cce80dd7023a0950510f9bbf
+size 328962560

modeling_picolm_v3.py

@@ -0,0 +1,143 @@
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import PreTrainedModel, GenerationMixin
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from .configuration_picolm_v3 import PicoLMV3Config
+
+class RMSNorm(nn.Module):
+    def __init__(self, dim: int, eps: float = 1e-5):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(dim))
+    def forward(self, x):
+        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight
+
+def precompute_rope_cis_yarn(head_dim=64, max_seq_len=4096, orig_len=2048, theta=10000.0):
+    half = head_dim // 2
+    scale = max_seq_len / orig_len
+    idx = torch.arange(half, dtype=torch.float64)
+    inv_freq = theta ** (-2.0 * idx / head_dim)
+    
+    if scale > 1.0:
+        def corr_dim(n_rot):
+            return head_dim * math.log(orig_len / (n_rot * 2 * math.pi)) / (2 * math.log(theta))
+        low = max(math.floor(corr_dim(32.0)), 0)
+        high = min(math.ceil(corr_dim(1.0)), half - 1)
+        gamma = ((idx - low) / (high - low)).clamp(0.0, 1.0)
+        inv_freq = inv_freq * (1.0 - gamma) + (inv_freq / scale) * gamma
+        attn_scale = 0.1 * math.log(scale) + 1.0
+    else:
+        attn_scale = 1.0
+
+    t = torch.arange(max_seq_len, dtype=torch.float64)
+    angles = torch.outer(t, inv_freq)
+    cos = (torch.cos(angles) * attn_scale).to(torch.float32)
+    sin = (torch.sin(angles) * attn_scale).to(torch.float32)
+    return cos, sin
+
+def apply_rope(x, cos, sin):
+    B, H, S, D = x.shape
+    cos = cos[:S, :].to(x.device).unsqueeze(0).unsqueeze(0)
+    sin = sin[:S, :].to(x.device).unsqueeze(0).unsqueeze(0)
+    x1, x2 = x[..., : D // 2], x[..., D // 2 :]
+    return torch.cat([x1 * cos - x2 * sin, x1 * sin + x2 * cos], dim=-1)
+
+class Attention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.head_dim = config.head_dim
+        self.n_heads = config.n_heads
+        self.n_kv_heads = config.n_kv_heads
+        self.num_reps = config.n_heads // config.n_kv_heads
+        self.q_proj = nn.Linear(config.dim, config.n_heads * config.head_dim, bias=False)
+        self.k_proj = nn.Linear(config.dim, config.n_kv_heads * config.head_dim, bias=False)
+        self.v_proj = nn.Linear(config.dim, config.n_kv_heads * config.head_dim, bias=False)
+        self.out_proj = nn.Linear(config.n_heads * config.head_dim, config.dim, bias=False)
+        self.q_norm = RMSNorm(config.head_dim, eps=config.norm_eps)
+        self.k_norm = RMSNorm(config.head_dim, eps=config.norm_eps)
+
+    def forward(self, x, cos, sin):
+        B, S, C = x.shape
+        q = apply_rope(self.q_norm(self.q_proj(x).view(B, S, self.n_heads, self.head_dim).transpose(1, 2)), cos, sin)
+        k = apply_rope(self.k_norm(self.k_proj(x).view(B, S, self.n_kv_heads, self.head_dim).transpose(1, 2)), cos, sin)
+        v = self.v_proj(x).view(B, S, self.n_kv_heads, self.head_dim).transpose(1, 2)
+        if self.num_reps > 1:
+            k = k[:, :, None, :, :].expand(B, self.n_kv_heads, self.num_reps, S, self.head_dim).reshape(B, self.n_heads, S, self.head_dim)
+            v = v[:, :, None, :, :].expand(B, self.n_kv_heads, self.num_reps, S, self.head_dim).reshape(B, self.n_heads, S, self.head_dim)
+        attn_out = F.scaled_dot_product_attention(q, k, v, is_causal=True)
+        return self.out_proj(attn_out.transpose(1, 2).contiguous().view(B, S, C))
+
+class SwiGLUMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.gate_proj = nn.Linear(config.dim, config.intermediate_dim, bias=False)
+        self.up_proj = nn.Linear(config.dim, config.intermediate_dim, bias=False)
+        self.down_proj = nn.Linear(config.intermediate_dim, config.dim, bias=False)
+    def forward(self, x):
+        return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
+
+class TransformerBlockV3(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.attn = Attention(config)
+        self.mlp = SwiGLUMLP(config)
+        self.attn_norm_1 = RMSNorm(config.dim, eps=config.norm_eps)
+        self.mlp_norm_1 = RMSNorm(config.dim, eps=config.norm_eps)
+        self.attn_norm_2 = RMSNorm(config.dim, eps=config.norm_eps)
+        self.mlp_norm_2 = RMSNorm(config.dim, eps=config.norm_eps)
+
+    def forward(self, x, cos, sin, pass_idx=0):
+        norm_attn = self.attn_norm_1 if pass_idx == 0 else self.attn_norm_2
+        norm_mlp = self.mlp_norm_1 if pass_idx == 0 else self.mlp_norm_2
+        # V2 HATASI DÜZELTİLDİ: Doğru Kalıntı Yolu
+        x = x + self.attn(norm_attn(x), cos, sin)
+        x = x + self.mlp(norm_mlp(x))
+        return x
+
+class PicoLMV3ForCausalLM(PreTrainedModel, GenerationMixin):
+    config_class = PicoLMV3Config
+    base_model_prefix = "model"
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.config = config
+        self.tok_embeddings = nn.Embedding(config.vocab_size, config.emb_dim)
+        self.emb_proj = nn.Linear(config.emb_dim, config.dim, bias=False)
+        self.blocks = nn.ModuleList([TransformerBlockV3(config) for _ in range(config.n_layers)])
+        self.final_norm = RMSNorm(config.dim, eps=config.norm_eps)
+        cos, sin = precompute_rope_cis_yarn(config.head_dim, config.max_seq_len, config.orig_seq_len, config.rope_theta)
+        self.register_buffer("rope_cos", cos, persistent=False)
+        self.register_buffer("rope_sin", sin, persistent=False)
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.tok_embeddings
+
+    def set_input_embeddings(self, value):
+        self.tok_embeddings = value
+
+    def forward(self, input_ids=None, labels=None, **kwargs):
+        B, S = input_ids.shape
+        x_low = self.tok_embeddings(input_ids)
+        x = self.emb_proj(x_low)
+        cos, sin = self.rope_cos[:S], self.rope_sin[:S]
+
+        for block in self.blocks:
+            for p_idx in range(self.config.layer_repeat):
+                x = block(x, cos, sin, pass_idx=p_idx)
+
+        h = self.final_norm(x)
+        # Factorized Tied Embedding Çıkışı
+        h_low = F.linear(h, self.emb_proj.weight.t())
+        logits = F.linear(h_low, self.tok_embeddings.weight)
+
+        loss = None
+        if labels is not None:
+            loss = F.cross_entropy(logits.reshape(-1, self.config.vocab_size), labels.reshape(-1), ignore_index=-100)
+
+        return CausalLMOutputWithPast(loss=loss, logits=logits)
+
+    def prepare_inputs_for_generation(self, input_ids, **kwargs):
+        return {"input_ids": input_ids}

tokenizer_config.json

@@ -0,0 +1,10 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"
+}