JiRackDeltaNet 70B SFT: bf16, FP8, card

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+sft_final/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+sft_final_fp8/tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,17 +1,15 @@
 ---
-
-license: bsl-1.0
+license: apache-2.0
 library_name: pytorch
 pipeline_tag: text-generation
 tags:
-- gated-deltanet
-- linear-attention
-- hybrid-attention
-- depth-expansion
-- function-calling
-- reasoning
-- fp8
-
+  - gated-deltanet
+  - linear-attention
+  - hybrid-attention
+  - depth-expansion
+  - function-calling
+  - reasoning
+  - fp8
 ---
 
 # JiRackDeltaNet 70B
@@ -21,7 +19,7 @@ Gated DeltaNet / gated full-attention model (Qwen 3.5 layout) built in our own J
 **64 to 128 layers**, and fine-tuned on our own SFT mix (chat, reasoning, code, math, tool calling).
 
 > Status: SFT release. The 64 added layers have been trained only by this SFT run (~45 M tokens), so most of the
-> model's ability still comes from the base model; see [Limitations](#limitations).
+> model's ability still comes from the base model; see Limitations.
 
 ## Model
 
@@ -111,9 +109,9 @@ The repository ships its own PyTorch code; no `transformers` model class is need
 ```bash
 pip install torch safetensors tokenizers jinja2
 # bf16, splits the 128 blocks over all visible GPUs (>= 2 x 80 GB)
-python3 chat_jirack.py --model-dir ./sft_final --no-think --temperature 0
-# FP8 copy (weights stay FP8 in memory, dequantized per layer call; any GPU)
-python3 chat_fp8.py --model-dir ./sft_final_fp8 --no-think --temperature 0
+python3 sft_final/chat_jirack.py --model-dir sft_final --no-think --temperature 0
+# FP8 copy (weights stay FP8 in memory, dequantized per layer call; one 96 GB GPU)
+python3 sft_final_fp8/chat_fp8.py --model-dir sft_final_fp8 --no-think --temperature 0
 ```
 
 Commands in the chat: `/think` (reasoning on / off), `/reset`, `exit`. Prompts use the Qwen 3.5 chat template in
@@ -138,6 +136,6 @@ this is **not verified** yet.
 
 ## License
 
-BSL 1.0. The weights are derived from a Qwen model released under BSL 1.0 . Changes by CMSManhattan
+Apache License 2.0. The weights are derived from a Qwen model released under Apache 2.0. Changes by CMSManhattan
 (Konstantin Grabko): JiRack model code, FFN widening, depth expansion to 128 layers, SFT on the JiRack SFT mix and the
-FP8 conversion. Keep this notice and the license text when redistributing.
+FP8 conversion. Keep this notice and the license text when redistributing.

sft_final/.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

sft_final/JiRackDeltaNet_70b.py

@@ -0,0 +1,766 @@
+"""
+JiRackDeltaNet_27b.py
+=============================
+
+Собственная (hand-rolled) архитектура "JiRack Qwen3.8-27B" -- в стиле
+прежнего DS32B-проекта: один файл, свой TransformerBlock, никакого
+AutoConfig/AutoModelForCausalLM внутри класса архитектуры, никакого
+model_id в конструкторе. Все размерности зашиты константами ниже --
+подтверждены живым инспектом реального checkpoint'а (inspect_qwen38_keys.py,
+851 тензор, сверено).
+
+Математика DeltaNet-рекуррентности, свёртки, гейтинга и partial-rotary
+mRoPE-attention ПОРТИРОВАНА из реального modeling_qwen3_5.py (Apache 2.0,
+HuggingFace/Qwen), а не переизобретена -- чтобы не рисковать тихой
+числовой ошибкой в тонкой рекуррентной формуле. Портирован смысл, но
+код переписан под наши имена/стиль и урезан: без vision-башни, без
+KV-cache классов transformers (для тренировки полный форвард по
+последовательности не нужен; для генерации -- полный recompute на
+каждом шаге, без инкрементального кэша; это сознательное упрощение,
+при желании кэш можно добавить отдельно).
+
+Имена подмодулей (`linear_attn` / `self_attn`, `in_proj_qkv`, `out_proj`,
+`mlp.gate_proj` и т.д.) намеренно СОВПАДАЮТ с именами в реальном
+state_dict Qwen3_5ForCausalLM -- поэтому веса из HF-чекпоинта грузятся
+напрямую через load_state_dict(), без ручного key-mapping.
+
+Файл умеет:
+  - собрать модель (JiRackQwen38ForCausalLM) с нуля, случайные веса;
+  - принять state_dict в формате HF Qwen3_5ForCausalLM (после
+    AutoModelForCausalLM.from_pretrained(...).state_dict() в конвертере)
+    и загрузить его напрямую;
+  - пропатчить нужные nn.Linear -> BitLinear (TERNARIZE_PATTERNS);
+  - сохранить/загрузить свой JiRack-чекпоинт (save_checkpoint/load_checkpoint).
+
+Конвертер (convert_qwen38_to_jirack.py) по-прежнему знает про model_id
+и HF Hub -- но это знание живёт ТОЛЬКО в конвертере, не в архитектуре.
+"""
+
+import os
+import re
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+# =====================================================================
+# 1. CONFIRMED АРХИТЕКТУРА -- жёстко зашитые константы (не AutoConfig)
+# =====================================================================
+
+HIDDEN_SIZE = 5120
+VOCAB_SIZE = 248320
+INTERMEDIATE_SIZE = 26624
+NUM_LAYERS = 128
+RMS_NORM_EPS = 1e-6
+
+# Чередование слоёв: full_attention_interval = 4 ->
+# 3x linear_attention, 1x full_attention, повторяется на все 64 слоя.
+LAYER_TYPES = ["linear_attention"] * 3 + ["full_attention"]
+LAYER_TYPES = (LAYER_TYPES * (NUM_LAYERS // len(LAYER_TYPES) + 1))[:NUM_LAYERS]
+
+# --- Gated DeltaNet (linear_attention) ---
+LINEAR_NUM_KEY_HEADS = 16
+LINEAR_KEY_HEAD_DIM = 128
+LINEAR_NUM_VALUE_HEADS = 48
+LINEAR_VALUE_HEAD_DIM = 128
+LINEAR_CONV_KERNEL_DIM = 4
+DELTA_KEY_DIM = LINEAR_NUM_KEY_HEADS * LINEAR_KEY_HEAD_DIM       # 2048
+DELTA_VALUE_DIM = LINEAR_NUM_VALUE_HEADS * LINEAR_VALUE_HEAD_DIM  # 6144
+DELTA_CONV_DIM = DELTA_KEY_DIM * 2 + DELTA_VALUE_DIM              # 10240
+DELTA_CHUNK_SIZE = 64
+
+# --- Gated Attention (full_attention) ---
+NUM_ATTENTION_HEADS = 24
+NUM_KEY_VALUE_HEADS = 4
+HEAD_DIM = 256
+PARTIAL_ROTARY_FACTOR = 0.25
+ROPE_THETA = 1e7
+MROPE_SECTION = [11, 11, 10]
+ATTENTION_BIAS = False
+
+ARCH_SUMMARY = f"""
+Qwen3.8-27B (qwen3_5), hand-rolled JiRack TransformerBlock, text-only.
+hidden_size={HIDDEN_SIZE}, vocab_size={VOCAB_SIZE},
+intermediate_size={INTERMEDIATE_SIZE}, num_hidden_layers={NUM_LAYERS}
+Чередование (interval=4): 3x linear_attention (DeltaNet), 1x full_attention
+"""
+
+KEPT_FULL_PRECISION = """
+Всегда остаются full precision (не GEMM, либо слишком чувствительны):
+  - dt_bias, A_log                    (сырые Parameters, decay DeltaNet)
+  - linear_attn.in_proj_b/in_proj_a   (по умолчанию, см. GATE_PROJ_PATTERNS)
+  - conv1d                            (depthwise grouped conv, не GEMM)
+  - все нормы: RMSNormGated, q_norm/k_norm, model.norm
+  - embed_tokens, lm_head
+  - mtp.* (вся MTP/NextN-голова)      (draft-слой спекулятивного декода;
+                                       тернаризация убивает acceptance rate,
+                                       комьюнити-GGUF держат её в Q8_0)
+"""
+
+
+# =====================================================================
+# 2. BitLinear -- наш тернарный слой
+# =====================================================================
+
+class BitLinear(nn.Linear):
+    """b1.58-style per-tensor absmean ternary weights, per-token int8
+    activations, непрерывный lambda-warmup (STE)."""
+
+    def __init__(self, in_features, out_features, bias=False):
+        super().__init__(in_features, out_features, bias=bias)
+        self.eps = 1e-5
+        self.register_buffer("lambda_", torch.zeros(()), persistent=True)
+
+    @classmethod
+    def from_linear(cls, lin: nn.Linear) -> "BitLinear":
+        bl = cls(lin.in_features, lin.out_features, bias=lin.bias is not None)
+        bl = bl.to(dtype=lin.weight.dtype, device=lin.weight.device)
+        with torch.no_grad():
+            bl.weight.copy_(lin.weight)
+            if lin.bias is not None:
+                bl.bias.copy_(lin.bias)
+        return bl
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        if not self.training and float(self.lambda_) < 1e-6:
+            return F.linear(x, self.weight, self.bias)
+
+        lam = self.lambda_.to(x.dtype)
+        w = self.weight
+        gamma = w.float().abs().mean().clamp(min=self.eps).to(w.dtype)
+        w_quant = torch.clamp(torch.round(w / gamma), -1.0, 1.0) * gamma
+        w_effective = w + lam * (w_quant - w).detach()
+
+        x_scale = 127.0 / x.abs().max(dim=-1, keepdim=True).values.clamp(min=self.eps)
+        x_quant = torch.clamp(torch.round(x * x_scale), -128.0, 127.0) / x_scale
+        x_effective = x + lam * (x_quant - x).detach()
+
+        return F.linear(x_effective, w_effective, self.bias)
+
+    def set_lambda(self, v: float):
+        self.lambda_.fill_(v)
+
+
+# =====================================================================
+# 3. Карта: что тернаризуем, что оставляем full precision
+# =====================================================================
+
+TERNARIZE_PATTERNS = [
+    r"linear_attn\.in_proj_qkv$",
+    r"linear_attn\.in_proj_z$",
+    r"linear_attn\.out_proj$",
+    r"self_attn\.q_proj$",
+    r"self_attn\.k_proj$",
+    r"self_attn\.v_proj$",
+    r"self_attn\.o_proj$",
+    r"mlp\.gate_proj$",
+    r"mlp\.up_proj$",
+    r"mlp\.down_proj$",
+]
+
+GATE_PROJ_PATTERNS = [
+    r"linear_attn\.in_proj_b$",
+    r"linear_attn\.in_proj_a$",
+]
+
+
+def apply_bitlinear_patch(model: nn.Module, include_gate_proj: bool = False):
+    """Точечная замена nn.Linear -> BitLinear по TERNARIZE_PATTERNS."""
+    patterns = list(TERNARIZE_PATTERNS)
+    if include_gate_proj:
+        patterns += GATE_PROJ_PATTERNS
+    compiled = [re.compile(p) for p in patterns]
+
+    patched = []
+    for name, module in list(model.named_modules()):
+        if not isinstance(module, nn.Linear):
+            continue
+        # MTP/NextN-голова НЕ тернаризуется -- всегда full precision.
+        # Суффиксные паттерны (self_attn.q_proj$ и т.п.) иначе зацепили бы
+        # mtp.layers.0.self_attn.q_proj.
+        if name.startswith("mtp"):
+            continue
+        if any(p.search(name) for p in compiled):
+            parent_name, _, child_name = name.rpartition(".")
+            parent = model.get_submodule(parent_name) if parent_name else model
+            setattr(parent, child_name, BitLinear.from_linear(module))
+            patched.append(name)
+    return patched
+
+
+# =====================================================================
+# 4. Общие блоки (порт математики из реального modeling_qwen3_5.py)
+# =====================================================================
+
+class RMSNorm(nn.Module):
+    """(x * w).to(dtype), w хранится как (weight) с offset +1 -- как в
+    оригинале Qwen3_5RMSNorm (инициализация нулями, эффективный вес 1+w)."""
+
+    def __init__(self, dim: int, eps: float = RMS_NORM_EPS):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.zeros(dim))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        dtype = x.dtype
+        x = x.float()
+        x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+        x = x * (1.0 + self.weight.float())
+        return x.to(dtype)
+
+
+class RMSNormGated(nn.Module):
+    """RMSNorm с гейтингом через SiLU(gate), применяется ПЕРЕД гейтингом
+    (норма считается по hidden_states, гейт применяется после)."""
+
+    def __init__(self, dim: int, eps: float = RMS_NORM_EPS):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(dim))
+        self.eps = eps
+
+    def forward(self, x: torch.Tensor, gate: torch.Tensor) -> torch.Tensor:
+        dtype = x.dtype
+        x = x.float()
+        var = x.pow(2).mean(-1, keepdim=True)
+        x = x * torch.rsqrt(var + self.eps)
+        x = self.weight * x.to(dtype)
+        x = x * F.silu(gate.float())
+        return x.to(dtype)
+
+
+def rotate_half(x: torch.Tensor) -> torch.Tensor:
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2:]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
+    """Partial rotary: только первые cos.shape[-1] каналов вращаются,
+    остальное (q_pass/k_pass) проходит без изменений."""
+    cos = cos.unsqueeze(unsqueeze_dim)
+    sin = sin.unsqueeze(unsqueeze_dim)
+    rotary_dim = cos.shape[-1]
+    q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:]
+    k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:]
+    q_embed = torch.cat([(q_rot * cos) + (rotate_half(q_rot) * sin), q_pass], dim=-1)
+    k_embed = torch.cat([(k_rot * cos) + (rotate_half(k_rot) * sin), k_pass], dim=-1)
+    return q_embed, k_embed
+
+
+class TextRotaryEmbedding(nn.Module):
+    """mRoPE с partial_rotary_factor. Для чистого текста (без vision)
+    3 позиционные "оси" (T,H,W) совпадают -> interleaved mRoPE
+    вырождается в обычный RoPE, но мы всё равно считаем честно тем же
+    путём, что и оригинал, для полной числовой идентичности."""
+
+    def __init__(self, head_dim=HEAD_DIM, partial_rotary_factor=PARTIAL_ROTARY_FACTOR,
+                 theta=ROPE_THETA, mrope_section=MROPE_SECTION):
+        super().__init__()
+        self.rotary_dim = int(head_dim * partial_rotary_factor)
+        inv_freq = 1.0 / (theta ** (torch.arange(0, self.rotary_dim, 2, dtype=torch.float32) / self.rotary_dim))
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self.mrope_section = mrope_section
+
+    def _apply_interleaved_mrope(self, freqs: torch.Tensor) -> torch.Tensor:
+        # freqs: (3, bs, seq, rotary_dim // 2)
+        freqs_t = freqs[0].clone()
+        for dim, offset in enumerate((1, 2), start=1):
+            length = self.mrope_section[dim] * 3
+            idx = slice(offset, length, 3)
+            freqs_t[..., idx] = freqs[dim, ..., idx]
+        return freqs_t
+
+    @torch.no_grad()
+    def forward(self, x: torch.Tensor, position_ids: torch.LongTensor):
+        # position_ids: (bs, seq) -> реплицируем в 3 идентичные оси (text-only)
+        if position_ids.ndim == 2:
+            position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)
+        inv_freq_exp = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1)
+        pos_exp = position_ids[:, :, None, :].float()
+        freqs = (inv_freq_exp @ pos_exp).transpose(2, 3)   # (3, bs, seq, rotary_dim//2)
+        freqs = self._apply_interleaved_mrope(freqs)        # (bs, seq, rotary_dim//2)
+        emb = torch.cat((freqs, freqs), dim=-1)              # (bs, seq, rotary_dim)
+        return emb.cos().to(x.dtype), emb.sin().to(x.dtype)
+
+
+def l2norm(x: torch.Tensor, dim: int = -1, eps: float = 1e-6):
+    return x * torch.rsqrt((x * x).sum(dim=dim, keepdim=True) + eps)
+
+
+def causal_conv1d(hidden_states: torch.Tensor, weight: torch.Tensor, activation: str = "silu"):
+    """hidden_states: (batch, channels, seq_len). Depthwise causal conv,
+    left-padded by kernel_size - 1, затем активация."""
+    channels, seq_len = hidden_states.shape[1], hidden_states.shape[2]
+    padding = weight.shape[-1] - 1
+    out = F.conv1d(hidden_states, weight=weight.unsqueeze(1), bias=None, padding=padding, groups=channels)
+    out = out[:, :, :seq_len]
+    return F.silu(out) if activation == "silu" else out
+
+
+def chunked_gated_delta_rule(query, key, value, g, beta, chunk_size=DELTA_CHUNK_SIZE):
+    """Порт torch_chunk_gated_delta_rule из modeling_qwen3_5.py -- без
+    initial_state/output_final_state (тренировка/полный форвард без
+    инкрементального кэша). query/key/value: (b, h, t, d), g/beta: (b, h, t)."""
+    initial_dtype = query.dtype
+    query = l2norm(query, dim=-1, eps=1e-6)
+    key = l2norm(key, dim=-1, eps=1e-6)
+    query, key, value, beta, g = [x.contiguous().to(torch.float32) for x in (query, key, value, beta, g)]
+
+    b, h, t, k_dim = key.shape
+    v_dim = value.shape[-1]
+    pad = (chunk_size - t % chunk_size) % chunk_size
+    query = F.pad(query, (0, 0, 0, pad))
+    key = F.pad(key, (0, 0, 0, pad))
+    value = F.pad(value, (0, 0, 0, pad))
+    beta = F.pad(beta, (0, pad))
+    g = F.pad(g, (0, pad))
+    total_t = t + pad
+    scale = 1 / (query.shape[-1] ** 0.5)
+    query = query * scale
+
+    v_beta = value * beta.unsqueeze(-1)
+    k_beta = key * beta.unsqueeze(-1)
+    query, key, value, k_beta, v_beta = [
+        x.reshape(x.shape[0], x.shape[1], -1, chunk_size, x.shape[-1]) for x in (query, key, value, k_beta, v_beta)
+    ]
+    g = g.reshape(g.shape[0], g.shape[1], -1, chunk_size)
+    mask = torch.triu(torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=0)
+
+    g = g.cumsum(dim=-1)
+    decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril()
+    attn = -((k_beta @ key.transpose(-1, -2)) * decay_mask).masked_fill(mask, 0)
+    for i in range(1, chunk_size):
+        row = attn[..., i, :i].clone()
+        sub = attn[..., :i, :i].clone()
+        attn[..., i, :i] = row + (row.unsqueeze(-1) * sub).sum(-2)
+    attn = attn + torch.eye(chunk_size, dtype=attn.dtype, device=attn.device)
+    value = attn @ v_beta
+    k_cumdecay = attn @ (k_beta * g.exp().unsqueeze(-1))
+
+    last_state = torch.zeros(b, h, k_dim, v_dim, dtype=value.dtype, device=value.device)
+    core_out = torch.zeros_like(value)
+
+    for i in range(total_t // chunk_size):
+        q_i, k_i, v_i = query[:, :, i], key[:, :, i], value[:, :, i]
+        attn_i = q_i @ k_i.transpose(-1, -2) * decay_mask[:, :, i]
+        v_prime = k_cumdecay[:, :, i] @ last_state
+        v_new = v_i - v_prime
+        attn_inter = (q_i * g[:, :, i, :, None].exp()) @ last_state
+        core_out[:, :, i] = attn_inter + attn_i @ v_new
+        last_state = (
+            last_state * g[:, :, i, -1, None, None].exp()
+            + (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2) @ v_new
+        )
+
+    core_out = core_out.reshape(core_out.shape[0], core_out.shape[1], -1, core_out.shape[-1])
+    core_out = core_out[:, :, :t]
+    core_out = core_out.transpose(1, 2).contiguous().to(initial_dtype)
+    return core_out
+
+
+# =====================================================================
+# 5. Наш собственный TransformerBlock (и подмодули linear_attn/self_attn)
+# =====================================================================
+
+class GatedDeltaNetBlock(nn.Module):
+    """linear_attention слой (Gated DeltaNet). Имена подмодулей совпадают
+    с HF state_dict, чтобы веса грузились напрямую."""
+
+    def __init__(self):
+        super().__init__()
+        self.conv1d = nn.Conv1d(
+            in_channels=DELTA_CONV_DIM, out_channels=DELTA_CONV_DIM, bias=False,
+            kernel_size=LINEAR_CONV_KERNEL_DIM, groups=DELTA_CONV_DIM,
+            padding=LINEAR_CONV_KERNEL_DIM - 1,
+        )
+        self.dt_bias = nn.Parameter(torch.ones(LINEAR_NUM_VALUE_HEADS))
+        A = torch.empty(LINEAR_NUM_VALUE_HEADS).uniform_(0.01, 16)
+        self.A_log = nn.Parameter(torch.log(A))
+        self.norm = RMSNormGated(LINEAR_VALUE_HEAD_DIM, eps=RMS_NORM_EPS)
+        self.out_proj = nn.Linear(DELTA_VALUE_DIM, HIDDEN_SIZE, bias=False)
+        self.in_proj_qkv = nn.Linear(HIDDEN_SIZE, DELTA_KEY_DIM * 2 + DELTA_VALUE_DIM, bias=False)
+        self.in_proj_z = nn.Linear(HIDDEN_SIZE, DELTA_VALUE_DIM, bias=False)
+        self.in_proj_b = nn.Linear(HIDDEN_SIZE, LINEAR_NUM_VALUE_HEADS, bias=False)
+        self.in_proj_a = nn.Linear(HIDDEN_SIZE, LINEAR_NUM_VALUE_HEADS, bias=False)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        bsz, seq_len, _ = hidden_states.shape
+
+        mixed_qkv = self.in_proj_qkv(hidden_states).transpose(1, 2)   # (b, conv_dim, t)
+        z = self.in_proj_z(hidden_states).reshape(bsz, seq_len, -1, LINEAR_VALUE_HEAD_DIM)
+        b = self.in_proj_b(hidden_states)
+        a = self.in_proj_a(hidden_states)
+
+        mixed_qkv = causal_conv1d(mixed_qkv, self.conv1d.weight.squeeze(1), activation="silu")
+        mixed_qkv = mixed_qkv.transpose(1, 2)   # (b, t, conv_dim)
+
+        query, key, value = torch.split(mixed_qkv, [DELTA_KEY_DIM, DELTA_KEY_DIM, DELTA_VALUE_DIM], dim=-1)
+        query = query.reshape(bsz, seq_len, -1, LINEAR_KEY_HEAD_DIM)
+        key = key.reshape(bsz, seq_len, -1, LINEAR_KEY_HEAD_DIM)
+        value = value.reshape(bsz, seq_len, -1, LINEAR_VALUE_HEAD_DIM)
+
+        beta = b.sigmoid()
+        g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
+
+        rep = LINEAR_NUM_VALUE_HEADS // LINEAR_NUM_KEY_HEADS
+        if rep > 1:
+            query = query.repeat_interleave(rep, dim=2)
+            key = key.repeat_interleave(rep, dim=2)
+
+        # -> (b, h, t, d) for the chunked kernel
+        query_h = query.transpose(1, 2)
+        key_h = key.transpose(1, 2)
+        value_h = value.transpose(1, 2)
+        beta_h = beta.transpose(1, 2)
+        g_h = g.transpose(1, 2)
+
+        core_attn_out = chunked_gated_delta_rule(query_h, key_h, value_h, g_h, beta_h)
+
+        core_attn_out = core_attn_out.reshape(-1, LINEAR_VALUE_HEAD_DIM)
+        z_flat = z.reshape(-1, LINEAR_VALUE_HEAD_DIM)
+        core_attn_out = self.norm(core_attn_out, z_flat)
+        core_attn_out = core_attn_out.reshape(bsz, seq_len, -1)
+
+        return self.out_proj(core_attn_out)
+
+
+class AttentionBlock(nn.Module):
+    """full_attention слой (Gated Attention + partial rotary mRoPE)."""
+
+    def __init__(self):
+        super().__init__()
+        self.num_key_value_groups = NUM_ATTENTION_HEADS // NUM_KEY_VALUE_HEADS
+        self.scaling = HEAD_DIM ** -0.5
+        self.q_proj = nn.Linear(HIDDEN_SIZE, NUM_ATTENTION_HEADS * HEAD_DIM * 2, bias=ATTENTION_BIAS)
+        self.k_proj = nn.Linear(HIDDEN_SIZE, NUM_KEY_VALUE_HEADS * HEAD_DIM, bias=ATTENTION_BIAS)
+        self.v_proj = nn.Linear(HIDDEN_SIZE, NUM_KEY_VALUE_HEADS * HEAD_DIM, bias=ATTENTION_BIAS)
+        self.o_proj = nn.Linear(NUM_ATTENTION_HEADS * HEAD_DIM, HIDDEN_SIZE, bias=ATTENTION_BIAS)
+        self.q_norm = RMSNorm(HEAD_DIM, eps=RMS_NORM_EPS)
+        self.k_norm = RMSNorm(HEAD_DIM, eps=RMS_NORM_EPS)
+
+    def forward(self, hidden_states, cos, sin, causal_mask):
+        bsz, seq_len, _ = hidden_states.shape
+        hidden_shape = (bsz, seq_len, -1, HEAD_DIM)
+
+        query_states, gate = torch.chunk(
+            self.q_proj(hidden_states).view(bsz, seq_len, -1, HEAD_DIM * 2), 2, dim=-1
+        )
+        gate = gate.reshape(bsz, seq_len, -1)
+
+        query_states = self.q_norm(query_states.view(hidden_shape)).transpose(1, 2)
+        key_states = self.k_norm(self.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
+        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+        rep = self.num_key_value_groups
+        key_states = key_states.repeat_interleave(rep, dim=1)
+        value_states = value_states.repeat_interleave(rep, dim=1)
+
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) * self.scaling
+        attn_weights = attn_weights + causal_mask
+        attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_output = torch.matmul(attn_weights, value_states)
+        attn_output = attn_output.transpose(1, 2).contiguous().reshape(bsz, seq_len, -1)
+        attn_output = attn_output * torch.sigmoid(gate)
+
+        return self.o_proj(attn_output)
+
+
+class MLPBlock(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.gate_proj = nn.Linear(HIDDEN_SIZE, INTERMEDIATE_SIZE, bias=False)
+        self.up_proj = nn.Linear(HIDDEN_SIZE, INTERMEDIATE_SIZE, bias=False)
+        self.down_proj = nn.Linear(INTERMEDIATE_SIZE, HIDDEN_SIZE, bias=False)
+
+    def forward(self, x):
+        return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
+
+
+class TransformerBlock(nn.Module):
+    """Наш decoder layer -- linear_attn ИЛИ self_attn, в зависимости от
+    layer_type, + общий MLP. Имена атрибутов совпадают с HF state_dict."""
+
+    def __init__(self, layer_idx: int, layer_type: str = None):
+        super().__init__()
+        # layer_type задан явно -> используем его (нужно для MTP-слоя,
+        # который всегда full_attention и лежит ВНЕ LAYER_TYPES).
+        self.block_type = layer_type if layer_type is not None else LAYER_TYPES[layer_idx]
+        if self.block_type == "linear_attention":
+            self.linear_attn = GatedDeltaNetBlock()
+        else:
+            self.self_attn = AttentionBlock()
+        self.mlp = MLPBlock()
+        self.input_layernorm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+        self.post_attention_layernorm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+
+    def forward(self, hidden_states, cos, sin, causal_mask):
+        residual = hidden_states
+        hidden_states = self.input_layernorm(hidden_states)
+        if self.block_type == "linear_attention":
+            hidden_states = self.linear_attn(hidden_states)
+        else:
+            hidden_states = self.self_attn(hidden_states, cos, sin, causal_mask)
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+        return hidden_states
+
+
+# =====================================================================
+# 5b. MTP / NextN-голова (draft-слой спекулятивного декода)
+# =====================================================================
+
+class MTPBlock(nn.Module):
+    """Multi-Token Prediction голова -- ровно как в реальном чекпоинте
+    Qwen3.5/3.8 (mtp_num_hidden_layers=1, mtp_use_dedicated_embeddings=false).
+
+    Имена подмодулей СОВПАДАЮТ с ключами HF state_dict (top-level `mtp.`):
+        mtp.pre_fc_norm_embedding.weight
+        mtp.pre_fc_norm_hidden.weight
+        mtp.fc.weight                       [HIDDEN, 2*HIDDEN]
+        mtp.layers.0.*                      (обычный full_attention decoder)
+        mtp.norm.weight                     (final norm перед общим lm_head)
+
+    embed_tokens и lm_head ОБЩИЕ со стволом (dedicated embeddings = false),
+    здесь их нет. В GGUF это blk.{NUM_LAYERS}.nextn.eh_proj / 
+    shared_head_norm + тензоры draft-слоя.
+
+    Механика (DeepSeek-style, порядок как в vLLM Qwen3NextMTP):
+        h = fc(cat(pre_fc_norm_embedding(emb_next), pre_fc_norm_hidden(h_prev)))
+        h = layers[0](h)          # full attention
+        logits2 = lm_head(norm(h))
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.pre_fc_norm_embedding = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+        self.pre_fc_norm_hidden = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+        self.fc = nn.Linear(2 * HIDDEN_SIZE, HIDDEN_SIZE, bias=False)
+        self.layers = nn.ModuleList([
+            TransformerBlock(layer_idx=NUM_LAYERS, layer_type="full_attention"),
+        ])
+        self.norm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+
+    def forward(self, inputs_embeds, previous_hidden, cos, sin, causal_mask):
+        """inputs_embeds  -- embed_tokens(токены, сдвинутые на +1)
+        previous_hidden   -- hidden ствола ДО финального model.norm
+        (см. контракт: MTP сам применяет pre_fc_norm_hidden)."""
+        h = self.fc(torch.cat(
+            (self.pre_fc_norm_embedding(inputs_embeds),
+             self.pre_fc_norm_hidden(previous_hidden)),
+            dim=-1,
+        ))
+        h = self.layers[0](h, cos, sin, causal_mask)
+        return self.norm(h)
+
+
+# =====================================================================
+# 6. Полная модель
+# =====================================================================
+
+class JiRackQwen38Model(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.embed_tokens = nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE)
+        self.layers = nn.ModuleList([TransformerBlock(i) for i in range(NUM_LAYERS)])
+        self.norm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+        self.rotary_emb = TextRotaryEmbedding()
+
+    def forward(self, input_ids: torch.LongTensor, return_prenorm: bool = False):
+        bsz, seq_len = input_ids.shape
+        hidden_states = self.embed_tokens(input_ids)
+
+        position_ids = torch.arange(seq_len, device=input_ids.device).unsqueeze(0).expand(bsz, -1)
+        cos, sin = self.rotary_emb(hidden_states, position_ids)
+
+        causal_mask = torch.full((seq_len, seq_len), float("-inf"), device=input_ids.device, dtype=torch.float32)
+        causal_mask = torch.triu(causal_mask, diagonal=1)[None, None, :, :].to(hidden_states.dtype)
+
+        for layer in self.layers:
+            hidden_states = layer(hidden_states, cos, sin, causal_mask)
+
+        if return_prenorm:
+            # Для MTP: голова сама применяет pre_fc_norm_hidden, финальный
+            # model.norm сюда НЕ входит (иначе двойная нормализация).
+            return self.norm(hidden_states), hidden_states, (cos, sin, causal_mask)
+        return self.norm(hidden_states)
+
+
+class JiRackQwen38ForCausalLM(nn.Module):
+    """Верхнеуровневая модель. Имена атрибутов (`model`, `lm_head`)
+    совпадают с HF Qwen3_5ForCausalLM -- state_dict грузится напрямую."""
+
+    def __init__(self):
+        super().__init__()
+        self.model = JiRackQwen38Model()
+        self.lm_head = nn.Linear(HIDDEN_SIZE, VOCAB_SIZE, bias=False)
+        # MTP/NextN-голова: top-level атрибут `mtp` -- ключи state_dict
+        # получаются ровно mtp.*, как в реальном HF-чекпоинте.
+        self.mtp = MTPBlock()
+
+    def forward(self, input_ids: torch.LongTensor) -> torch.Tensor:
+        hidden_states = self.model(input_ids)
+        return self.lm_head(hidden_states)
+
+    def mtp_forward(self, input_ids: torch.LongTensor):
+        """Возвращает (logits_t1, logits_t2): обычные логиты ствола и
+        draft-логиты MTP-головы (предсказание токена t+2 из позиции t).
+        Для инференса/валидации MTP; в GGUF-конвертации не участвует."""
+        _, prenorm_hidden, (cos, sin, causal_mask) = self.model(
+            input_ids, return_prenorm=True)
+        logits_t1 = self.lm_head(self.model.norm(prenorm_hidden))
+
+        # Вход MTP -- эмбеддинги токенов, сдвинутых на +1 (следующий токен
+        # уже известен на шаге верификации). Последнюю позицию дублируем,
+        # чтобы сохранить форму (draft для неё не используется).
+        shifted = torch.cat([input_ids[:, 1:], input_ids[:, -1:]], dim=1)
+        emb_next = self.model.embed_tokens(shifted)
+        h2 = self.mtp(emb_next, prenorm_hidden, cos, sin, causal_mask)
+        logits_t2 = self.lm_head(h2)
+        return logits_t1, logits_t2
+
+    # -----------------------------------------------------------------
+    # Заг��узка реальных весов (state_dict уже скачан конвертером через
+    # HF Hub -- сюда просто передаётся словарь тензоров)
+    # -----------------------------------------------------------------
+    def load_hf_state_dict(self, state_dict: dict, strict: bool = False):
+        """Грузит state_dict в формате оригинального Qwen3_5ForCausalLM.
+        mtp.* ключи теперь ГРУЗЯТСЯ в self.mtp (MTPBlock) -- фильтра
+        больше нет. strict=False оставлен из-за служебных lambda_-буферов
+        BitLinear, которых нет в реальном чекпоинте."""
+        missing, unexpected = self.load_state_dict(state_dict, strict=strict)
+        real_missing = [k for k in missing if not k.endswith("lambda_")]
+        if real_missing:
+            print(f"⚠️  {len(real_missing)} missing keys (первые 10): {real_missing[:10]}")
+        if unexpected:
+            print(f"⚠️  {len(unexpected)} unexpected keys (первые 10): {unexpected[:10]}")
+        return missing, unexpected
+
+    # -----------------------------------------------------------------
+    # Подсадка РЕАЛЬНЫХ mtp.* весов из оригинального HF-чекпоинта.
+    # Нужна, потому что старые JiRack .pt сохранены ДО появления MTPBlock
+    # и mtp-весов не содержат (там была бы случайная инициализация).
+    # -----------------------------------------------------------------
+    def load_mtp_from_hf_dir(self, hf_dir: str):
+        """Читает из папки оригинального HF-чекпоинта ТОЛЬКО mtp.* тензоры
+        (по index.json или перебором *.safetensors) и грузит их в self.mtp.
+        Возвращает число загруженных тензоров."""
+        import glob
+        import json as _json
+        from safetensors import safe_open
+
+        index_path = os.path.join(hf_dir, "model.safetensors.index.json")
+        mtp_map = {}  # key -> shard file
+        if os.path.exists(index_path):
+            with open(index_path) as f:
+                weight_map = _json.load(f)["weight_map"]
+            for k, shard in weight_map.items():
+                if k.startswith("mtp."):
+                    mtp_map[k] = os.path.join(hf_dir, shard)
+        else:
+            for shard in glob.glob(os.path.join(hf_dir, "*.safetensors")):
+                with safe_open(shard, framework="pt", device="cpu") as f:
+                    for k in f.keys():
+                        if k.startswith("mtp."):
+                            mtp_map[k] = shard
+
+        if not mtp_map:
+            raise FileNotFoundError(
+                f"В {hf_dir} не найдено ни одного mtp.* тензора. Нужен "
+                f"ОРИГИНАЛЬНЫЙ чекпоинт Qwen3.8-27B (не JiRack-экспорт!). "
+                f"Если оригинальные шарды удалены -- перекачай с HF Hub "
+                f"только шарды с mtp.* по model.safetensors.index.json.")
+
+        own_dtype = next(self.mtp.parameters()).dtype
+        loaded = {}
+        by_shard = {}
+        for k, shard in mtp_map.items():
+            by_shard.setdefault(shard, []).append(k)
+        for shard, keys in by_shard.items():
+            with safe_open(shard, framework="pt", device="cpu") as f:
+                for k in keys:
+                    loaded[k] = f.get_tensor(k).to(own_dtype)
+
+        missing, unexpected = self.load_state_dict(loaded, strict=False)
+        n_ok = len(loaded)
+        print(f"✅ MTP: загружено {n_ok} тензоров из {len(by_shard)} шардов ({hf_dir})")
+        if unexpected:
+            print(f"⚠️  MTP: unexpected keys: {unexpected[:10]}")
+        return n_ok
+
+    # -----------------------------------------------------------------
+    # JiRack-чекпоинт (свой формат, с lambda/patched_linear_names)
+    # -----------------------------------------------------------------
+    def save_checkpoint(self, path: str, lam: float, global_step: int = 0, patched_linear_names=None):
+        torch.save({
+            "model": self.state_dict(),
+            "lambda": lam,
+            "global_step": global_step,
+            "patched_linear_names": patched_linear_names or [],
+        }, path)
+
+    @classmethod
+    def load_checkpoint(cls, path: str, dtype=torch.bfloat16, include_gate_proj: bool = False):
+        ckpt = torch.load(path, map_location="cpu", weights_only=False)
+        model = cls()
+        patched_names = ckpt.get("patched_linear_names", [])
+        if patched_names:
+            for name in patched_names:
+                *parent_path, leaf = name.split(".")
+                parent = model
+                for p in parent_path:
+                    parent = getattr(parent, p)
+                old = getattr(parent, leaf)
+                if not isinstance(old, BitLinear):
+                    setattr(parent, leaf, BitLinear.from_linear(old))
+        else:
+            apply_bitlinear_patch(model, include_gate_proj=include_gate_proj)
+
+        has_mtp_in_ckpt = any(k.startswith("mtp.") for k in ckpt["model"])
+        model.load_state_dict(ckpt["model"], strict=False)
+        model = model.to(dtype)
+
+        if not has_mtp_in_ckpt:
+            print(
+                "⚠️  В этом .pt НЕТ mtp.* весов (чекпоинт сохранён до "
+                "появления MTPBlock). Голова сейчас со СЛУЧАЙНОЙ "
+                "инициализацией! Перед экспортом обязательно вызови "
+                "model.load_mtp_from_hf_dir('<папка оригинального HF-чекпоинта>')."
+            )
+
+        lam = ckpt.get("lambda", 0.0)
+        for m in model.modules():
+            if isinstance(m, BitLinear):
+                m.set_lambda(lam)
+
+        print(f"✅ Загружено. global_step={ckpt.get('global_step', 0)}, lambda={lam}")
+        return model
+
+
+# =====================================================================
+# 7. Демонстрация (без скачивания весов -- только структура)
+# =====================================================================
+
+if __name__ == "__main__":
+    print(ARCH_SUMMARY)
+    print(KEPT_FULL_PRECISION)
+
+    model = JiRackQwen38ForCausalLM()
+    n_params = sum(p.numel() for p in model.parameters())
+    print(f"✅ Модель собрана (случайные веса). Параметров: {n_params / 1e9:.2f}B")
+
+    patched = apply_bitlinear_patch(model)
+    print(f"✅ Заменено подмодулей на BitLinear: {len(patched)}")
+    print("Примеры первых 10:")
+    for n in patched[:10]:
+        print(f"   {n}")
+
+    ids = torch.randint(0, VOCAB_SIZE, (1, 8))
+    with torch.no_grad():
+        logits = model(ids)
+    print(f"✅ Пробный forward (случайные веса) -> logits shape: {tuple(logits.shape)}")

sft_final/audit_depth.json

@@ -0,0 +1,14 @@
+{
+  "kind": "depth",
+  "src": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded",
+  "out": "/mnt/nfs_share/JiRackDeltaNet_70b/expanded",
+  "tensors": 1714,
+  "ops": {
+    "copy": 1586,
+    "zero": 128
+  },
+  "errors": [],
+  "n_errors": 0,
+  "verdict": "PASS",
+  "minutes": 9.8
+}

sft_final/chat_jirack.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+# =============================================================================
+# chat_jirack.py - chat with JiRackDeltaNet (27B / 36B / 70B) from a safetensors folder.
+#
+#   python3 chat_jirack.py --model-dir /mnt/nfs_share/JiRackDeltaNet_70b/expanded
+#   python3 chat_jirack.py --model-dir /mnt/nfs_share/JiRackDeltaNet_27b_cp      # 27B
+#
+# - model code: JiRackDeltaNet_*.py from the same folder (or --model-file)
+# - several GPUs: the TransformerBlocks are split across them evenly (70B bf16 = 140 GB,
+#   so at least 2x80 GB); 1 GPU if it fits; CPU works but is VERY slow for 70B
+# - answer streams token by token, stops on <|im_end|> / <|endoftext|>
+# - commands: /reset (clear history), /think (thinking on/off), exit
+# =============================================================================
+import argparse
+import glob
+import importlib.util
+import itertools
+import json
+import os
+import sys
+import time
+
+import torch
+import torch.nn as nn
+
+INIT_FNS = ("uniform_", "normal_", "trunc_normal_", "kaiming_uniform_", "kaiming_normal_",
+            "xavier_uniform_", "xavier_normal_", "orthogonal_")
+
+
+def find_model_file(d):
+    files = sorted(glob.glob(os.path.join(d, "JiRackDeltaNet_*.py")))
+    if not files:
+        sys.exit(f"❌ no JiRackDeltaNet_*.py in {d}; pass --model-file")
+    return files[-1]
+
+
+def load_module(path):
+    spec = importlib.util.spec_from_file_location("jirack_chat_model", path)
+    mod = importlib.util.module_from_spec(spec)
+    sys.modules["jirack_chat_model"] = mod
+    spec.loader.exec_module(mod)
+    return mod
+
+
+def build(mod):
+    """bf16 skeleton without random init (all weights are loaded afterwards) - as in deltanet_gate.py."""
+    saved = {n: getattr(nn.init, n) for n in INIT_FNS if hasattr(nn.init, n)}
+    for n in saved:
+        setattr(nn.init, n, lambda t, *a, **k: t)
+    torch.set_default_dtype(torch.bfloat16)
+    try:
+        model = mod.JiRackQwen38ForCausalLM()
+    finally:
+        torch.set_default_dtype(torch.float32)
+        for n, f in saved.items():
+            setattr(nn.init, n, f)
+    return model.eval()
+
+
+def load_weights(model, d):
+    from safetensors import safe_open
+    idx = os.path.join(d, "model.safetensors.index.json")
+    files = sorted(set(json.load(open(idx))["weight_map"].values())) if os.path.exists(idx) else \
+        sorted(os.path.basename(p) for p in glob.glob(os.path.join(d, "*.safetensors")))
+    sd = model.state_dict()
+    params = {n for n, _ in model.named_parameters()}
+    loaded = set()
+    t0 = time.time()
+    with torch.no_grad():
+        for i, f in enumerate(files):
+            with safe_open(os.path.join(d, f), framework="pt", device="cpu") as h:
+                for k in h.keys():
+                    if k in sd:
+                        sd[k].copy_(h.get_tensor(k))
+                        loaded.add(k)
+            print(f"   {i + 1}/{len(files)} {f}  {time.time() - t0:.0f} s", flush=True)
+    missing = sorted(params - loaded)
+    if missing:
+        sys.exit(f"❌ {len(missing)} parameters not in {d}: {missing[:5]}")
+
+
+class ChatTokenizer:
+    """tokenizer.json (tokenizers) + chat template (jinja2) - no transformers needed, so the
+    tokenizer_config.json format (e.g. extra_special_tokens as a list) does not matter."""
+
+    def __init__(self, d):
+        import jinja2
+        from jinja2.sandbox import ImmutableSandboxedEnvironment
+        from tokenizers import Tokenizer
+        self.tk = Tokenizer.from_file(os.path.join(d, "tokenizer.json"))
+        cfg = {}
+        if os.path.exists(os.path.join(d, "tokenizer_config.json")):
+            cfg = json.load(open(os.path.join(d, "tokenizer_config.json"), encoding="utf-8"))
+        path = os.path.join(d, "chat_template.jinja")
+        text = open(path, encoding="utf-8").read() if os.path.exists(path) else cfg.get("chat_template")
+        if isinstance(text, list):
+            text = next(t["template"] for t in text if t.get("name") == "default")
+        if not text:
+            sys.exit(f"❌ no chat template in {d}")
+
+        def raise_exception(msg):
+            raise jinja2.exceptions.TemplateError(msg)
+
+        def tojson(x, ensure_ascii=False, indent=None, separators=None, sort_keys=False):
+            return json.dumps(x, ensure_ascii=ensure_ascii, indent=indent, separators=separators,
+                              sort_keys=sort_keys)
+
+        env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True)
+        env.filters["tojson"] = tojson
+        env.globals["raise_exception"] = raise_exception
+        self.tmpl = env.from_string(text)
+        s = lambda v: v.get("content") if isinstance(v, dict) else v
+        self.bos, self.eos = s(cfg.get("bos_token")) or "", s(cfg.get("eos_token")) or ""
+        self.special = [t.content for t in self.tk.get_added_tokens_decoder().values() if t.special]
+        self.stop_ids = {i for i in (self.tk.token_to_id("<|im_end|>"), self.tk.token_to_id("<|endoftext|>"),
+                                     self.tk.token_to_id(self.eos) if self.eos else None) if i is not None}
+
+    def prompt_ids(self, msgs, think):
+        text = self.tmpl.render(messages=msgs, add_generation_prompt=True, enable_thinking=think,
+                                bos_token=self.bos, eos_token=self.eos)
+        return self.tk.encode(text, add_special_tokens=False).ids
+
+    def decode(self, ids):
+        return self.tk.decode(ids, skip_special_tokens=False)
+
+
+def _to(x, dev):
+    if torch.is_tensor(x):
+        return x.to(dev)
+    if isinstance(x, (tuple, list)):
+        return type(x)(_to(v, dev) for v in x)
+    if isinstance(x, dict):
+        return {k: _to(v, dev) for k, v in x.items()}
+    return x
+
+
+def place(model, devices):
+    """Blocks split evenly over the GPUs (moved there directly - the whole model never sits on one
+    GPU); everything else on the first GPU. Pre-hooks move each module's inputs to its device,
+    so the model code needs no changes."""
+    blocks = [m for m in model.modules() if type(m).__name__ == "TransformerBlock"]
+    in_block = {id(x) for b in blocks for x in b.modules()}
+    for i, b in enumerate(blocks):
+        dev = devices[i * len(devices) // len(blocks)]
+        b.to(dev)
+        if len(devices) > 1:
+            b.register_forward_pre_hook(lambda m, a, k, dev=dev: (_to(a, dev), _to(k, dev)), with_kwargs=True)
+    for m in model.modules():                             # embeddings / norms / lm_head / mtp / rotary
+        if id(m) in in_block:
+            continue
+        m._apply(lambda t: t.to(devices[0]), recurse=False)
+        if len(devices) > 1 and not list(m.children()):
+            m.register_forward_pre_hook(lambda mm, a, k: (_to(a, devices[0]), _to(k, devices[0])),
+                                        with_kwargs=True)
+    return devices[0]
+
+
+def _cast(x, dtype):
+    if torch.is_tensor(x):
+        return x.to(dtype) if x.is_floating_point() and x.dtype != dtype else x
+    if isinstance(x, (tuple, list)):
+        return type(x)(_cast(v, dtype) for v in x)
+    if isinstance(x, dict):
+        return {k: _cast(v, dtype) for k, v in x.items()}
+    return x
+
+
+def fp32_blocks(model):
+    """CPU (AVX-only Xeon has no fast bf16 matmul): every TransformerBlock and lm_head runs with
+    fp32 weights/inputs, converted on entry and back on exit - stored weights stay bf16 (RAM),
+    same as the 'fp32blocks' compute of deltanet_gate.py."""
+    def pre(mod, args, kwargs):
+        saved = []
+        for t in itertools.chain(mod.parameters(), mod.buffers()):
+            if t.is_floating_point() and t.dtype != torch.float32:
+                saved.append((t, t.data))
+                t.data = t.data.float()
+        mod._jr_saved = saved
+        return _cast(args, torch.float32), _cast(kwargs, torch.float32)
+
+    def post(mod, args, out, back=torch.bfloat16):
+        for t, d in mod._jr_saved:
+            t.data = d
+        mod._jr_saved = []
+        return _cast(out, back) if back is not None else out
+
+    for m in model.modules():
+        if type(m).__name__ == "TransformerBlock":
+            m.register_forward_pre_hook(pre, with_kwargs=True)
+            m.register_forward_hook(post)
+    head = getattr(model, "lm_head", None)
+    if isinstance(head, nn.Module):
+        head.register_forward_pre_hook(pre, with_kwargs=True)
+        head.register_forward_hook(lambda m, a, o: post(m, a, o, None))
+
+
+def logits_of(out):
+    if isinstance(out, (tuple, list)):
+        return out[0]
+    return out.logits if hasattr(out, "logits") else out
+
+
+def sample(logits, temperature, top_p):
+    if temperature <= 0:
+        return int(logits.argmax())
+    probs = torch.softmax(logits.float() / temperature, dim=-1)
+    sp, si = probs.sort(descending=True)
+    cut = (sp.cumsum(-1) - sp) > top_p
+    sp[cut] = 0
+    return int(si[torch.multinomial(sp / sp.sum(), 1)])
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument("--model-dir", default="/mnt/nfs_share/JiRackDeltaNet_70b/expanded")
+    p.add_argument("--model-file", default=None)
+    p.add_argument("--tokenizer-dir", default=None, help="default: --model-dir")
+    p.add_argument("--max-new", type=int, default=2048)
+    p.add_argument("--temperature", type=float, default=0.6, help="0 = greedy")
+    p.add_argument("--top-p", type=float, default=0.95)
+    p.add_argument("--system", default=None, help="file with a system prompt")
+    p.add_argument("--no-think", action="store_true", help="start with thinking off")
+    p.add_argument("--cpu", action="store_true")
+    p.add_argument("--cpu-bf16", action="store_true", help="CPU: plain bf16 instead of fp32 blocks")
+    a = p.parse_args()
+
+    tok = ChatTokenizer(a.tokenizer_dir or a.model_dir)
+    stop = tok.stop_ids
+    print(f"stop tokens {sorted(stop)}", flush=True)
+
+    mf = a.model_file or find_model_file(a.model_dir)
+    print(f"model file {mf}\nweights    {a.model_dir}", flush=True)
+    model = build(load_module(mf))
+    load_weights(model, a.model_dir)
+    n = sum(q.numel() for q in model.parameters())
+    if a.cpu or not torch.cuda.is_available():
+        dev = torch.device("cpu")
+        try:
+            torch.backends.nnpack.set_flags(False)       # silences "Could not initialize NNPACK"
+        except Exception:
+            pass
+        if not a.cpu_bf16:
+            fp32_blocks(model)
+        print(f"⚠️  CPU: {n / 1e9:.1f}B params, compute {'bf16' if a.cpu_bf16 else 'fp32 blocks'} - "
+              f"every token is a full forward pass, expect minutes per token. Ctrl+C stops the answer.")
+    else:
+        devs = [torch.device("cuda", i) for i in range(torch.cuda.device_count())]
+        dev = place(model, devs)
+        print(f"on {len(devs)} GPU(s): {n / 1e9:.1f}B params, "
+              f"{[round(torch.cuda.memory_allocated(d) / 2 ** 30) for d in devs]} GB")
+
+    system = open(a.system, encoding="utf-8").read().strip() if a.system else None
+    think = not a.no_think
+    history = []
+    print("\nReady. /reset - clear history, /think - thinking on/off, exit - quit")
+    while True:
+        try:
+            user = input("\nYou: ").strip()
+        except (EOFError, KeyboardInterrupt):
+            break
+        if not user:
+            continue
+        if user.lower() in ("exit", "quit"):
+            break
+        if user == "/reset":
+            history = []
+            print("(history cleared)")
+            continue
+        if user == "/think":
+            think = not think
+            print(f"(thinking {'on' if think else 'off'})")
+            continue
+        history.append({"role": "user", "content": user})
+        msgs = ([{"role": "system", "content": system}] if system else []) + history
+        ids = torch.tensor([tok.prompt_ids(msgs, think)], dtype=torch.long, device=dev)
+        start = ids.shape[1]
+        print("\nAssistant: ", end="", flush=True)
+        printed, t0 = "", time.time()
+        try:
+            with torch.no_grad():
+                for _ in range(a.max_new):
+                    nxt = sample(logits_of(model(ids))[0, -1], a.temperature, a.top_p)
+                    if nxt in stop:
+                        break
+                    ids = torch.cat([ids, torch.tensor([[nxt]], device=ids.device)], dim=1)
+                    text = tok.decode(ids[0, start:].tolist())
+                    if not text.endswith("�"):           # wait for a complete UTF-8 character
+                        print(text[len(printed):], end="", flush=True)
+                        printed = text
+        except KeyboardInterrupt:
+            print(" [stopped]", end="")
+        gen = ids.shape[1] - start
+        raw = tok.decode(ids[0, start:].tolist())
+        print(f"\n   [{gen} tokens, {gen / max(time.time() - t0, 1e-6):.2f} tok/s]")
+        # Qwen: earlier turns go back into the history without the thinking part
+        reply = raw.split("</think>")[-1]
+        for t in tok.special:
+            reply = reply.replace(t, "")
+        history.append({"role": "assistant", "content": reply.strip()})
+
+
+if __name__ == "__main__":
+    main()

sft_final/chat_template.jinja

@@ -0,0 +1,171 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- set reasoning_instructions = '' %}
+{%- if enable_thinking is defined and enable_thinking is true %}
+    {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}
+    {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}
+        {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}
+    {%- endif %}
+    {%- if resolved_reasoning_effort == 'xhigh' %}
+        {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}
+    {%- elif resolved_reasoning_effort == 'low' %}
+        {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}
+    {%- endif %}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {%- if reasoning_instructions %}
+        {{- reasoning_instructions + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '')  + content + '<|im_end|>\n' }}
+        {%- elif reasoning_instructions %}
+            {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
+        {%- endif %}
+    {%- elif reasoning_instructions %}
+        {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined and tool_call.arguments != '' %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is true %}
+        {{- '<think>\n' }}
+    {%- else %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}
+

sft_final/config.json

@@ -0,0 +1,178 @@
+{
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "attn_output_gate": true,
+  "bos_token_id": 248044,
+  "dtype": "bfloat16",
+  "eos_token_id": 248044,
+  "full_attention_interval": 4,
+  "head_dim": 256,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 26624,
+  "layer_types": [
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention",
+    "linear_attention",
+    "linear_attention",
+    "linear_attention",
+    "full_attention"
+  ],
+  "linear_conv_kernel_dim": 4,
+  "linear_key_head_dim": 128,
+  "linear_num_key_heads": 16,
+  "linear_num_value_heads": 48,
+  "linear_value_head_dim": 128,
+  "mamba_ssm_dtype": "float32",
+  "max_position_embeddings": 262144,
+  "model_type": "qwen3_5_text",
+  "mtp_num_hidden_layers": 1,
+  "mtp_use_dedicated_embeddings": false,
+  "num_attention_heads": 24,
+  "num_hidden_layers": 128,
+  "num_key_value_heads": 4,
+  "output_gate_type": "swish",
+  "pad_token_id": null,
+  "partial_rotary_factor": 0.25,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "mrope_interleaved": true,
+    "mrope_section": [
+      11,
+      11,
+      10
+    ],
+    "partial_rotary_factor": 0.25,
+    "rope_theta": 10000000,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "use_cache": true,
+  "vocab_size": 248320,
+  "architectures": [
+    "Qwen3_5ForCausalLM"
+  ]
+}

sft_final/gate_full.json

@@ -0,0 +1,81 @@
+{
+  "verdict": "PASS",
+  "expected": "exact",
+  "mini": false,
+  "compute": "fp32blocks",
+  "a": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded",
+  "b": "/mnt/nfs_share/JiRackDeltaNet_70b/expanded",
+  "layers_run": [
+    64,
+    128
+  ],
+  "intermediate": [
+    26624,
+    26624
+  ],
+  "prompts": [
+    "The capital of France is",
+    "def fibonacci(n):\n    ",
+    "Столица России —"
+  ],
+  "ids": [
+    [
+      760,
+      6511,
+      314,
+      9338,
+      369
+    ],
+    [
+      727,
+      73111,
+      1393,
+      1590,
+      198,
+      257
+    ],
+    [
+      212396,
+      168314,
+      170312,
+      1892
+    ]
+  ],
+  "rows": [
+    {
+      "T": 5,
+      "identical": true,
+      "max_abs": 0.0,
+      "mean_abs": 0.0,
+      "max_kl": 0.0,
+      "top1_agree": 5
+    },
+    {
+      "T": 6,
+      "identical": true,
+      "max_abs": 0.0,
+      "mean_abs": 0.0,
+      "max_kl": 0.0,
+      "top1_agree": 6
+    },
+    {
+      "T": 4,
+      "identical": true,
+      "max_abs": 0.0,
+      "mean_abs": 0.0,
+      "max_kl": 0.0,
+      "top1_agree": 4
+    }
+  ],
+  "a_info": {
+    "cached": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded/gate_logits_fp32blocks_L64.npz"
+  },
+  "b_info": {
+    "loaded": 1714,
+    "skipped": 0,
+    "unexpected": [],
+    "missing_buffers": [],
+    "seconds": 2016
+  },
+  "torch": "2.13.0+cu130"
+}

sft_final/gate_logits_fp32blocks_L128.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a589ae8136832e11c0c408e43a560cf110f011e3ecef3b60852db63eb256d025
+size 14900556

sft_final/gate_logits_fp32blocks_L8.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8975717cc37cabfaffcea30896d5aaefceb526cb2d0b1a0b993830d57358e29e
+size 14900556

sft_final/gate_mini.json

@@ -0,0 +1,81 @@
+{
+  "verdict": "PASS",
+  "expected": "exact",
+  "mini": true,
+  "compute": "fp32blocks",
+  "a": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded",
+  "b": "/mnt/nfs_share/JiRackDeltaNet_70b/expanded",
+  "layers_run": [
+    4,
+    8
+  ],
+  "intermediate": [
+    26624,
+    26624
+  ],
+  "prompts": [
+    "The capital of France is",
+    "def fibonacci(n):\n    ",
+    "Столица России —"
+  ],
+  "ids": [
+    [
+      760,
+      6511,
+      314,
+      9338,
+      369
+    ],
+    [
+      727,
+      73111,
+      1393,
+      1590,
+      198,
+      257
+    ],
+    [
+      212396,
+      168314,
+      170312,
+      1892
+    ]
+  ],
+  "rows": [
+    {
+      "T": 5,
+      "identical": true,
+      "max_abs": 0.0,
+      "mean_abs": 0.0,
+      "max_kl": 0.0,
+      "top1_agree": 5
+    },
+    {
+      "T": 6,
+      "identical": true,
+      "max_abs": 0.0,
+      "mean_abs": 0.0,
+      "max_kl": 0.0,
+      "top1_agree": 6
+    },
+    {
+      "T": 4,
+      "identical": true,
+      "max_abs": 0.0,
+      "mean_abs": 0.0,
+      "max_kl": 0.0,
+      "top1_agree": 4
+    }
+  ],
+  "a_info": {
+    "cached": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded/gate_logits_fp32blocks_L4.npz"
+  },
+  "b_info": {
+    "loaded": 124,
+    "skipped": 1590,
+    "unexpected": [],
+    "missing_buffers": [],
+    "seconds": 138
+  },
+  "torch": "2.13.0+cu130"
+}

sft_final/gguf.txt

@@ -0,0 +1,35 @@
+# 1. Конвертация в bf16 (как договорились)
+python convert_hf_to_gguf.py /mnt/nfs_share/Qwen3_8 \
+    --outfile /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.gguf --outtype bf16
+
+# 2. Сборка llama-quantize (один раз)
+cd /mnt/nfs_share/llama.cpp
+cmake -B build
+cmake --build build -j
+
+# 3. Квантование в Q4_K_M
+./build/bin/llama-quantize \
+    /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.gguf \
+    /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.Q4_K_M.gguf \
+    Q4_K_M
+
+
+
+cd /mnt/nfs_share/llama.cpp
+cmake -B build -DGGML_CUDA=ON  # для NVIDIA
+make -C build
+
+
+/mnt/nfs_share/llama.cpp/build/bin/llama-cli \
+     -m /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.Q4_K_M.gguf \
+     -n 512 \
+     -c 4096 \
+     -p "Explain quantum computing in simple terms:"
+
+
+
+
+
+/mnt/nfs_share/llama.cpp/build/bin/llama-server \
+     -m /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.Q4_K_M.gguf \
+     --port 8000

sft_final/jirack_sft_state.json

@@ -0,0 +1,9 @@
+{
+  "global_step": 374,
+  "lr": 1e-05,
+  "max_steps": 374,
+  "shards": "0",
+  "tok_micro": 8192,
+  "accum": 2,
+  "init_dir": "/home/ubuntu/deltanet70b"
+}

sft_final/model-00001-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1360d4af0ce009aaa30a0159a667ddc59298e927a8cdf820f015db8d17eba0a
+size 5146507008

sft_final/model-00002-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b88380f4538c1d5a023e89b795fb9e3aa7d1887ea8e14edd20ee451bc55a075
+size 5204245040

sft_final/model-00003-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07ee192c49b4fa41f8236a3b92153820c42cc4bc4ee8cfdb985eeded7cd48127
+size 5226281296

sft_final/model-00004-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55e52de05145920a220e89bb564adf50e43ea99597e6a566dcc5744b9066807c
+size 5226281352

sft_final/model-00005-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebb1ecb964ee82f89f796796689e767c876440672c90f6a1c0793405c42cd2f0
+size 5226281352

sft_final/model-00006-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57ca557f3b558c080e8bde31c87e4214099995e408df8d9d207ef5260d05f70c
+size 5204245104

sft_final/model-00007-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4bd5929c8cb98f744f8d0d6a82adc3c21960dcfc357c8db9ba11d10cbcdd2ba
+size 5226281352

sft_final/model-00008-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de56d111c6e08a02a5d7c863adf0c05980c4efa23e742113caacda1d248eb304
+size 5226281352

sft_final/model-00009-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cb7e2b4989b3be6bfd0ba94bd84c14b06356cb76aa199e9f788af3ce12ac73c
+size 5226281352

sft_final/model-00010-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd4c4509fdf30a4404d16d7d3c734eb28bc8bafe1864a1759b9c0dc33e66da62
+size 5204245104

sft_final/model-00011-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce3bc4cc3bbf0f02ae4ee3f703df6fc824ffc1a55086421f8c7ca7f8d00bdd3c
+size 5226281352

sft_final/model-00012-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fecbfb15b8e92fd41b22d3509a178572764b6ffd39b4142dc190e466e925dfc
+size 5226281352

sft_final/model-00013-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27236071639417aaa92d756239fc403f2b6ab72d3fef247a614a2f681f107796
+size 5226281352

sft_final/model-00014-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47c992e281a5387c19de30b86c344b3d3b0649ab2d11977b9c294d49cee53ed1
+size 5204245104

sft_final/model-00015-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d7e1f039d216da37c40e655775534c52210ca7bb08b90d902459be52b39959d
+size 5226281352

sft_final/model-00016-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcb1983d57449ce9af7f702cf5db51cedae41df04c418acdc2c7e78852a8af9b
+size 5226281352

sft_final/model-00017-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0fb91fdf474b9f2f0b7e404b304a126047fb9da957934a70ff658a107d9ecfd
+size 5226281352

sft_final/model-00018-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd5eb70f5171fb1dceba0d0277bfc5af75fa895952bd646717bfd6a25958f017
+size 5204245104

sft_final/model-00019-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49e77b4426051b3fc8c7c18da83d48288f3ca4b694fc687a92d903fdd5a07063
+size 5226281352

sft_final/model-00020-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fe115d9440d1732381ee44bbc07e142d5871c1e288bcdf3598273261daace87
+size 5226281352

sft_final/model-00021-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22652bf288a10fa233fcf7adcc9d02d4309643badbd0c8cce1274c0cf9ec715b
+size 5226281360

sft_final/model-00022-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6db8017e7414d546907bd8ebb90458e9932b4b8d5dfcbe071ecf4f5d982aa289
+size 5204245168

sft_final/model-00023-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50cc1a1b337270c327cb90566b1fd99c9c3e42f08e96f148fca6c1f8c432c7c1
+size 5226281424

sft_final/model-00024-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24b94858756a2c0ad41faf3f33fc6a15e3406844288896336cd12112e79755ac
+size 5226281424

sft_final/model-00025-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca3ab652f04c9b441244c00fe1a50304309c6a7486c2f37cafd0ceb22ae86208
+size 5226281424

sft_final/model-00026-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:360f47926cc9428ee23e4fb75ea1911ed4f0970a19679ce5ed80338f15409328
+size 5204245168

sft_final/model-00027-of-00027.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16fe0a0184cdf5468f0e406cba58990cc930932519bbd5bd5ac3163261ffb41b
+size 4220603664

sft_final/system_prompt.txt

@@ -0,0 +1,6 @@
+You are JiRack, an assistant that answers immediately and to the point.
+Never use a reasoning block wrapped in think tags.
+Do not show internal reasoning, drafts, or step-by-step thought process.
+Give the final, ready answer right away, with no preliminary explanation of your reasoning.
+If you feel the need to think through the question, do it silently, internally,
+and output only the final answer to the user.

sft_final/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7befaf450030f2fc58d7ea6b3941216ec6cccd85115bf7c2bd5d127f2632cff1
+size 20010936

sft_final/tokenizer_config.json

@@ -0,0 +1,148 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|image|>",
+    "<|video|>",
+    "<|sound|>",
+    "<|voice|>",
+    "<|listening|>",
+    "<|vision|>",
+    "<|mood_happy|>",
+    "<|mood_sad|>",
+    "<|mood_angry|>",
+    "<|mood_neutral|>",
+    "<fim_prefix>",
+    "<fim_middle>",
+    "<fim_suffix>",
+    "<|action_start|>",
+    "<|action_end|>",
+    "<|trajectory_start|>",
+    "<|trajectory_end|>",
+    "<|joint_start|>",
+    "<|joint_end|>",
+    "<|sensor_start|>",
+    "<|sensor_end|>",
+    "<|command_start|>",
+    "<|command_end|>",
+    "<|state_start|>",
+    "<|state_end|>",
+    "<|pose|>",
+    "<|velocity|>",
+    "<|force|>",
+    "<|torque|>",
+    "<|gripper|>",
+    "<|navigation|>",
+    "<|obstacle|>",
+    "<|task_start|>",
+    "<|task_end|>",
+    "<|plan_start|>",
+    "<|plan_end|>",
+    "<|behavior_start|>",
+    "<|behavior_end|>",
+    "<|skill_start|>",
+    "<|skill_end|>",
+    "<|motor|>",
+    "<|servo|>",
+    "<|imu|>",
+    "<|lidar|>",
+    "<|camera|>",
+    "<|depth|>",
+    "<|waypoint|>",
+    "<|path|>",
+    "<|collision|>",
+    "<|grasp|>",
+    "<|release|>",
+    "<|homing|>",
+    "<|emergency_stop|>",
+    "<|calibration|>",
+    "<|manipulation|>",
+    "<|locomotion|>",
+    "<|feedback|>",
+    "<|control_loop|>",
+    "<|language|>",
+    "<|tool_call_start|>",
+    "<|tool_call_end|>",
+    "<|tool_result_start|>",
+    "<|tool_result_end|>",
+    "__SCIENCE__",
+    "__CODING__",
+    "__STOCK_EXCHANGE__",
+    "__MEDICINE__",
+    "__GOVERNMENT__",
+    "__NEWS__",
+    "__GENERAL__",
+    "__MATERIAL_SCIENCE__",
+    "__ELECTRONICS__",
+    "__MICROELECTRONICS__",
+    "__ENGINEERING__",
+    "__ROBOTICS__",
+    "__ENERGY__",
+    "__AUTOMOTIVE__",
+    "__AVIATION__",
+    "__MATH__",
+    "__PYTHON__",
+    "__C__",
+    "__CPP__",
+    "__C_SHARP__",
+    "__JAVA__",
+    "__JAVASCRIPT__",
+    "__TYPESCRIPT__",
+    "__RUST__",
+    "__GO__",
+    "__RUBY__",
+    "__PHP__",
+    "__SWIFT__",
+    "__KOTLIN__",
+    "__BASH__",
+    "__SQL__",
+    "__ASSEMBLY__",
+    "__PHILOSOPHY__",
+    "__LITERATURE__",
+    "__SOCIOLOGY__",
+    "__PSYCHOLOGY__",
+    "__POLITICAL_SCIENCE__",
+    "__CULTURAL_STUDIES__",
+    "__ETHNOGRAPHY__",
+    "__HUMAN_RIGHTS__",
+    "__COMPLIANCE__",
+    "__MILITARY__",
+    "__BANKING__",
+    "__OIL_INDUSTRY__",
+    "__LIGHT_INDUSTRY__",
+    "__NATURE__",
+    "__OCEAN__",
+    "__SPORT__",
+    "__CULINARY__",
+    "__TRAVEL__",
+    "__HOBBY__"
+  ],
+  "image_token": "<|image_pad|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}

sft_final_fp8/.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

sft_final_fp8/JiRackDeltaNet_70b.py

@@ -0,0 +1,766 @@
+"""
+JiRackDeltaNet_27b.py
+=============================
+
+Собственная (hand-rolled) архитектура "JiRack Qwen3.8-27B" -- в стиле
+прежнего DS32B-проекта: один файл, свой TransformerBlock, никакого
+AutoConfig/AutoModelForCausalLM внутри класса архитектуры, никакого
+model_id в конструкторе. Все размерности зашиты константами ниже --
+подтверждены живым инспектом реального checkpoint'а (inspect_qwen38_keys.py,
+851 тензор, сверено).
+
+Математика DeltaNet-рекуррентности, свёртки, гейтинга и partial-rotary
+mRoPE-attention ПОРТИРОВАНА из реального modeling_qwen3_5.py (Apache 2.0,
+HuggingFace/Qwen), а не переизобретена -- чтобы не рисковать тихой
+числовой ошибкой в тонкой рекуррентной формуле. Портирован смысл, но
+код переписан под наши имена/стиль и урезан: без vision-башни, без
+KV-cache классов transformers (для тренировки полный форвард по
+последовательности не нужен; для генерации -- полный recompute на
+каждом шаге, без инкрементального кэша; это сознательное упрощение,
+при желании кэш можно добавить отдельно).
+
+Имена подмодулей (`linear_attn` / `self_attn`, `in_proj_qkv`, `out_proj`,
+`mlp.gate_proj` и т.д.) намеренно СОВПАДАЮТ с именами в реальном
+state_dict Qwen3_5ForCausalLM -- поэтому веса из HF-чекпоинта грузятся
+напрямую через load_state_dict(), без ручного key-mapping.
+
+Файл умеет:
+  - собрать модель (JiRackQwen38ForCausalLM) с нуля, случайные веса;
+  - принять state_dict в формате HF Qwen3_5ForCausalLM (после
+    AutoModelForCausalLM.from_pretrained(...).state_dict() в конвертере)
+    и загрузить его напрямую;
+  - пропатчить нужные nn.Linear -> BitLinear (TERNARIZE_PATTERNS);
+  - сохранить/загрузить свой JiRack-чекпоинт (save_checkpoint/load_checkpoint).
+
+Конвертер (convert_qwen38_to_jirack.py) по-прежнему знает про model_id
+и HF Hub -- но это знание живёт ТОЛЬКО в конвертере, не в архитектуре.
+"""
+
+import os
+import re
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+# =====================================================================
+# 1. CONFIRMED АРХИТЕКТУРА -- жёстко зашитые константы (не AutoConfig)
+# =====================================================================
+
+HIDDEN_SIZE = 5120
+VOCAB_SIZE = 248320
+INTERMEDIATE_SIZE = 26624
+NUM_LAYERS = 128
+RMS_NORM_EPS = 1e-6
+
+# Чередование слоёв: full_attention_interval = 4 ->
+# 3x linear_attention, 1x full_attention, повторяется на все 64 слоя.
+LAYER_TYPES = ["linear_attention"] * 3 + ["full_attention"]
+LAYER_TYPES = (LAYER_TYPES * (NUM_LAYERS // len(LAYER_TYPES) + 1))[:NUM_LAYERS]
+
+# --- Gated DeltaNet (linear_attention) ---
+LINEAR_NUM_KEY_HEADS = 16
+LINEAR_KEY_HEAD_DIM = 128
+LINEAR_NUM_VALUE_HEADS = 48
+LINEAR_VALUE_HEAD_DIM = 128
+LINEAR_CONV_KERNEL_DIM = 4
+DELTA_KEY_DIM = LINEAR_NUM_KEY_HEADS * LINEAR_KEY_HEAD_DIM       # 2048
+DELTA_VALUE_DIM = LINEAR_NUM_VALUE_HEADS * LINEAR_VALUE_HEAD_DIM  # 6144
+DELTA_CONV_DIM = DELTA_KEY_DIM * 2 + DELTA_VALUE_DIM              # 10240
+DELTA_CHUNK_SIZE = 64
+
+# --- Gated Attention (full_attention) ---
+NUM_ATTENTION_HEADS = 24
+NUM_KEY_VALUE_HEADS = 4
+HEAD_DIM = 256
+PARTIAL_ROTARY_FACTOR = 0.25
+ROPE_THETA = 1e7
+MROPE_SECTION = [11, 11, 10]
+ATTENTION_BIAS = False
+
+ARCH_SUMMARY = f"""
+Qwen3.8-27B (qwen3_5), hand-rolled JiRack TransformerBlock, text-only.
+hidden_size={HIDDEN_SIZE}, vocab_size={VOCAB_SIZE},
+intermediate_size={INTERMEDIATE_SIZE}, num_hidden_layers={NUM_LAYERS}
+Чередование (interval=4): 3x linear_attention (DeltaNet), 1x full_attention
+"""
+
+KEPT_FULL_PRECISION = """
+Всегда остаются full precision (не GEMM, либо слишком чувствительны):
+  - dt_bias, A_log                    (сырые Parameters, decay DeltaNet)
+  - linear_attn.in_proj_b/in_proj_a   (по умолчанию, см. GATE_PROJ_PATTERNS)
+  - conv1d                            (depthwise grouped conv, не GEMM)
+  - все нормы: RMSNormGated, q_norm/k_norm, model.norm
+  - embed_tokens, lm_head
+  - mtp.* (вся MTP/NextN-голова)      (draft-слой спекулятивного декода;
+                                       тернаризация убивает acceptance rate,
+                                       комьюнити-GGUF держат её в Q8_0)
+"""
+
+
+# =====================================================================
+# 2. BitLinear -- наш тернарный слой
+# =====================================================================
+
+class BitLinear(nn.Linear):
+    """b1.58-style per-tensor absmean ternary weights, per-token int8
+    activations, непрерывный lambda-warmup (STE)."""
+
+    def __init__(self, in_features, out_features, bias=False):
+        super().__init__(in_features, out_features, bias=bias)
+        self.eps = 1e-5
+        self.register_buffer("lambda_", torch.zeros(()), persistent=True)
+
+    @classmethod
+    def from_linear(cls, lin: nn.Linear) -> "BitLinear":
+        bl = cls(lin.in_features, lin.out_features, bias=lin.bias is not None)
+        bl = bl.to(dtype=lin.weight.dtype, device=lin.weight.device)
+        with torch.no_grad():
+            bl.weight.copy_(lin.weight)
+            if lin.bias is not None:
+                bl.bias.copy_(lin.bias)
+        return bl
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        if not self.training and float(self.lambda_) < 1e-6:
+            return F.linear(x, self.weight, self.bias)
+
+        lam = self.lambda_.to(x.dtype)
+        w = self.weight
+        gamma = w.float().abs().mean().clamp(min=self.eps).to(w.dtype)
+        w_quant = torch.clamp(torch.round(w / gamma), -1.0, 1.0) * gamma
+        w_effective = w + lam * (w_quant - w).detach()
+
+        x_scale = 127.0 / x.abs().max(dim=-1, keepdim=True).values.clamp(min=self.eps)
+        x_quant = torch.clamp(torch.round(x * x_scale), -128.0, 127.0) / x_scale
+        x_effective = x + lam * (x_quant - x).detach()
+
+        return F.linear(x_effective, w_effective, self.bias)
+
+    def set_lambda(self, v: float):
+        self.lambda_.fill_(v)
+
+
+# =====================================================================
+# 3. Карта: что тернаризуем, что оставляем full precision
+# =====================================================================
+
+TERNARIZE_PATTERNS = [
+    r"linear_attn\.in_proj_qkv$",
+    r"linear_attn\.in_proj_z$",
+    r"linear_attn\.out_proj$",
+    r"self_attn\.q_proj$",
+    r"self_attn\.k_proj$",
+    r"self_attn\.v_proj$",
+    r"self_attn\.o_proj$",
+    r"mlp\.gate_proj$",
+    r"mlp\.up_proj$",
+    r"mlp\.down_proj$",
+]
+
+GATE_PROJ_PATTERNS = [
+    r"linear_attn\.in_proj_b$",
+    r"linear_attn\.in_proj_a$",
+]
+
+
+def apply_bitlinear_patch(model: nn.Module, include_gate_proj: bool = False):
+    """Точечная замена nn.Linear -> BitLinear по TERNARIZE_PATTERNS."""
+    patterns = list(TERNARIZE_PATTERNS)
+    if include_gate_proj:
+        patterns += GATE_PROJ_PATTERNS
+    compiled = [re.compile(p) for p in patterns]
+
+    patched = []
+    for name, module in list(model.named_modules()):
+        if not isinstance(module, nn.Linear):
+            continue
+        # MTP/NextN-голова НЕ тернаризуется -- всегда full precision.
+        # Суффиксные паттерны (self_attn.q_proj$ и т.п.) иначе зацепили бы
+        # mtp.layers.0.self_attn.q_proj.
+        if name.startswith("mtp"):
+            continue
+        if any(p.search(name) for p in compiled):
+            parent_name, _, child_name = name.rpartition(".")
+            parent = model.get_submodule(parent_name) if parent_name else model
+            setattr(parent, child_name, BitLinear.from_linear(module))
+            patched.append(name)
+    return patched
+
+
+# =====================================================================
+# 4. Общие блоки (порт математики из реального modeling_qwen3_5.py)
+# =====================================================================
+
+class RMSNorm(nn.Module):
+    """(x * w).to(dtype), w хранится как (weight) с offset +1 -- как в
+    оригинале Qwen3_5RMSNorm (инициализация нулями, эффективный вес 1+w)."""
+
+    def __init__(self, dim: int, eps: float = RMS_NORM_EPS):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.zeros(dim))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        dtype = x.dtype
+        x = x.float()
+        x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+        x = x * (1.0 + self.weight.float())
+        return x.to(dtype)
+
+
+class RMSNormGated(nn.Module):
+    """RMSNorm с гейтингом через SiLU(gate), применяется ПЕРЕД гейтингом
+    (норма считается по hidden_states, гейт применяется после)."""
+
+    def __init__(self, dim: int, eps: float = RMS_NORM_EPS):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(dim))
+        self.eps = eps
+
+    def forward(self, x: torch.Tensor, gate: torch.Tensor) -> torch.Tensor:
+        dtype = x.dtype
+        x = x.float()
+        var = x.pow(2).mean(-1, keepdim=True)
+        x = x * torch.rsqrt(var + self.eps)
+        x = self.weight * x.to(dtype)
+        x = x * F.silu(gate.float())
+        return x.to(dtype)
+
+
+def rotate_half(x: torch.Tensor) -> torch.Tensor:
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2:]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
+    """Partial rotary: только первые cos.shape[-1] каналов вращаются,
+    остальное (q_pass/k_pass) проходит без изменений."""
+    cos = cos.unsqueeze(unsqueeze_dim)
+    sin = sin.unsqueeze(unsqueeze_dim)
+    rotary_dim = cos.shape[-1]
+    q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:]
+    k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:]
+    q_embed = torch.cat([(q_rot * cos) + (rotate_half(q_rot) * sin), q_pass], dim=-1)
+    k_embed = torch.cat([(k_rot * cos) + (rotate_half(k_rot) * sin), k_pass], dim=-1)
+    return q_embed, k_embed
+
+
+class TextRotaryEmbedding(nn.Module):
+    """mRoPE с partial_rotary_factor. Для чистого текста (без vision)
+    3 позиционные "оси" (T,H,W) совпадают -> interleaved mRoPE
+    вырождается в обычный RoPE, но мы всё равно считаем честно тем же
+    путём, что и оригинал, для полной числовой идентичности."""
+
+    def __init__(self, head_dim=HEAD_DIM, partial_rotary_factor=PARTIAL_ROTARY_FACTOR,
+                 theta=ROPE_THETA, mrope_section=MROPE_SECTION):
+        super().__init__()
+        self.rotary_dim = int(head_dim * partial_rotary_factor)
+        inv_freq = 1.0 / (theta ** (torch.arange(0, self.rotary_dim, 2, dtype=torch.float32) / self.rotary_dim))
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self.mrope_section = mrope_section
+
+    def _apply_interleaved_mrope(self, freqs: torch.Tensor) -> torch.Tensor:
+        # freqs: (3, bs, seq, rotary_dim // 2)
+        freqs_t = freqs[0].clone()
+        for dim, offset in enumerate((1, 2), start=1):
+            length = self.mrope_section[dim] * 3
+            idx = slice(offset, length, 3)
+            freqs_t[..., idx] = freqs[dim, ..., idx]
+        return freqs_t
+
+    @torch.no_grad()
+    def forward(self, x: torch.Tensor, position_ids: torch.LongTensor):
+        # position_ids: (bs, seq) -> реплицируем в 3 идентичные оси (text-only)
+        if position_ids.ndim == 2:
+            position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)
+        inv_freq_exp = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1)
+        pos_exp = position_ids[:, :, None, :].float()
+        freqs = (inv_freq_exp @ pos_exp).transpose(2, 3)   # (3, bs, seq, rotary_dim//2)
+        freqs = self._apply_interleaved_mrope(freqs)        # (bs, seq, rotary_dim//2)
+        emb = torch.cat((freqs, freqs), dim=-1)              # (bs, seq, rotary_dim)
+        return emb.cos().to(x.dtype), emb.sin().to(x.dtype)
+
+
+def l2norm(x: torch.Tensor, dim: int = -1, eps: float = 1e-6):
+    return x * torch.rsqrt((x * x).sum(dim=dim, keepdim=True) + eps)
+
+
+def causal_conv1d(hidden_states: torch.Tensor, weight: torch.Tensor, activation: str = "silu"):
+    """hidden_states: (batch, channels, seq_len). Depthwise causal conv,
+    left-padded by kernel_size - 1, затем активация."""
+    channels, seq_len = hidden_states.shape[1], hidden_states.shape[2]
+    padding = weight.shape[-1] - 1
+    out = F.conv1d(hidden_states, weight=weight.unsqueeze(1), bias=None, padding=padding, groups=channels)
+    out = out[:, :, :seq_len]
+    return F.silu(out) if activation == "silu" else out
+
+
+def chunked_gated_delta_rule(query, key, value, g, beta, chunk_size=DELTA_CHUNK_SIZE):
+    """Порт torch_chunk_gated_delta_rule из modeling_qwen3_5.py -- без
+    initial_state/output_final_state (тренировка/полный форвард без
+    инкрементального кэша). query/key/value: (b, h, t, d), g/beta: (b, h, t)."""
+    initial_dtype = query.dtype
+    query = l2norm(query, dim=-1, eps=1e-6)
+    key = l2norm(key, dim=-1, eps=1e-6)
+    query, key, value, beta, g = [x.contiguous().to(torch.float32) for x in (query, key, value, beta, g)]
+
+    b, h, t, k_dim = key.shape
+    v_dim = value.shape[-1]
+    pad = (chunk_size - t % chunk_size) % chunk_size
+    query = F.pad(query, (0, 0, 0, pad))
+    key = F.pad(key, (0, 0, 0, pad))
+    value = F.pad(value, (0, 0, 0, pad))
+    beta = F.pad(beta, (0, pad))
+    g = F.pad(g, (0, pad))
+    total_t = t + pad
+    scale = 1 / (query.shape[-1] ** 0.5)
+    query = query * scale
+
+    v_beta = value * beta.unsqueeze(-1)
+    k_beta = key * beta.unsqueeze(-1)
+    query, key, value, k_beta, v_beta = [
+        x.reshape(x.shape[0], x.shape[1], -1, chunk_size, x.shape[-1]) for x in (query, key, value, k_beta, v_beta)
+    ]
+    g = g.reshape(g.shape[0], g.shape[1], -1, chunk_size)
+    mask = torch.triu(torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=0)
+
+    g = g.cumsum(dim=-1)
+    decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril()
+    attn = -((k_beta @ key.transpose(-1, -2)) * decay_mask).masked_fill(mask, 0)
+    for i in range(1, chunk_size):
+        row = attn[..., i, :i].clone()
+        sub = attn[..., :i, :i].clone()
+        attn[..., i, :i] = row + (row.unsqueeze(-1) * sub).sum(-2)
+    attn = attn + torch.eye(chunk_size, dtype=attn.dtype, device=attn.device)
+    value = attn @ v_beta
+    k_cumdecay = attn @ (k_beta * g.exp().unsqueeze(-1))
+
+    last_state = torch.zeros(b, h, k_dim, v_dim, dtype=value.dtype, device=value.device)
+    core_out = torch.zeros_like(value)
+
+    for i in range(total_t // chunk_size):
+        q_i, k_i, v_i = query[:, :, i], key[:, :, i], value[:, :, i]
+        attn_i = q_i @ k_i.transpose(-1, -2) * decay_mask[:, :, i]
+        v_prime = k_cumdecay[:, :, i] @ last_state
+        v_new = v_i - v_prime
+        attn_inter = (q_i * g[:, :, i, :, None].exp()) @ last_state
+        core_out[:, :, i] = attn_inter + attn_i @ v_new
+        last_state = (
+            last_state * g[:, :, i, -1, None, None].exp()
+            + (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2) @ v_new
+        )
+
+    core_out = core_out.reshape(core_out.shape[0], core_out.shape[1], -1, core_out.shape[-1])
+    core_out = core_out[:, :, :t]
+    core_out = core_out.transpose(1, 2).contiguous().to(initial_dtype)
+    return core_out
+
+
+# =====================================================================
+# 5. Наш собственный TransformerBlock (и подмодули linear_attn/self_attn)
+# =====================================================================
+
+class GatedDeltaNetBlock(nn.Module):
+    """linear_attention слой (Gated DeltaNet). Имена подмодулей совпадают
+    с HF state_dict, чтобы веса грузились напрямую."""
+
+    def __init__(self):
+        super().__init__()
+        self.conv1d = nn.Conv1d(
+            in_channels=DELTA_CONV_DIM, out_channels=DELTA_CONV_DIM, bias=False,
+            kernel_size=LINEAR_CONV_KERNEL_DIM, groups=DELTA_CONV_DIM,
+            padding=LINEAR_CONV_KERNEL_DIM - 1,
+        )
+        self.dt_bias = nn.Parameter(torch.ones(LINEAR_NUM_VALUE_HEADS))
+        A = torch.empty(LINEAR_NUM_VALUE_HEADS).uniform_(0.01, 16)
+        self.A_log = nn.Parameter(torch.log(A))
+        self.norm = RMSNormGated(LINEAR_VALUE_HEAD_DIM, eps=RMS_NORM_EPS)
+        self.out_proj = nn.Linear(DELTA_VALUE_DIM, HIDDEN_SIZE, bias=False)
+        self.in_proj_qkv = nn.Linear(HIDDEN_SIZE, DELTA_KEY_DIM * 2 + DELTA_VALUE_DIM, bias=False)
+        self.in_proj_z = nn.Linear(HIDDEN_SIZE, DELTA_VALUE_DIM, bias=False)
+        self.in_proj_b = nn.Linear(HIDDEN_SIZE, LINEAR_NUM_VALUE_HEADS, bias=False)
+        self.in_proj_a = nn.Linear(HIDDEN_SIZE, LINEAR_NUM_VALUE_HEADS, bias=False)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        bsz, seq_len, _ = hidden_states.shape
+
+        mixed_qkv = self.in_proj_qkv(hidden_states).transpose(1, 2)   # (b, conv_dim, t)
+        z = self.in_proj_z(hidden_states).reshape(bsz, seq_len, -1, LINEAR_VALUE_HEAD_DIM)
+        b = self.in_proj_b(hidden_states)
+        a = self.in_proj_a(hidden_states)
+
+        mixed_qkv = causal_conv1d(mixed_qkv, self.conv1d.weight.squeeze(1), activation="silu")
+        mixed_qkv = mixed_qkv.transpose(1, 2)   # (b, t, conv_dim)
+
+        query, key, value = torch.split(mixed_qkv, [DELTA_KEY_DIM, DELTA_KEY_DIM, DELTA_VALUE_DIM], dim=-1)
+        query = query.reshape(bsz, seq_len, -1, LINEAR_KEY_HEAD_DIM)
+        key = key.reshape(bsz, seq_len, -1, LINEAR_KEY_HEAD_DIM)
+        value = value.reshape(bsz, seq_len, -1, LINEAR_VALUE_HEAD_DIM)
+
+        beta = b.sigmoid()
+        g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
+
+        rep = LINEAR_NUM_VALUE_HEADS // LINEAR_NUM_KEY_HEADS
+        if rep > 1:
+            query = query.repeat_interleave(rep, dim=2)
+            key = key.repeat_interleave(rep, dim=2)
+
+        # -> (b, h, t, d) for the chunked kernel
+        query_h = query.transpose(1, 2)
+        key_h = key.transpose(1, 2)
+        value_h = value.transpose(1, 2)
+        beta_h = beta.transpose(1, 2)
+        g_h = g.transpose(1, 2)
+
+        core_attn_out = chunked_gated_delta_rule(query_h, key_h, value_h, g_h, beta_h)
+
+        core_attn_out = core_attn_out.reshape(-1, LINEAR_VALUE_HEAD_DIM)
+        z_flat = z.reshape(-1, LINEAR_VALUE_HEAD_DIM)
+        core_attn_out = self.norm(core_attn_out, z_flat)
+        core_attn_out = core_attn_out.reshape(bsz, seq_len, -1)
+
+        return self.out_proj(core_attn_out)
+
+
+class AttentionBlock(nn.Module):
+    """full_attention слой (Gated Attention + partial rotary mRoPE)."""
+
+    def __init__(self):
+        super().__init__()
+        self.num_key_value_groups = NUM_ATTENTION_HEADS // NUM_KEY_VALUE_HEADS
+        self.scaling = HEAD_DIM ** -0.5
+        self.q_proj = nn.Linear(HIDDEN_SIZE, NUM_ATTENTION_HEADS * HEAD_DIM * 2, bias=ATTENTION_BIAS)
+        self.k_proj = nn.Linear(HIDDEN_SIZE, NUM_KEY_VALUE_HEADS * HEAD_DIM, bias=ATTENTION_BIAS)
+        self.v_proj = nn.Linear(HIDDEN_SIZE, NUM_KEY_VALUE_HEADS * HEAD_DIM, bias=ATTENTION_BIAS)
+        self.o_proj = nn.Linear(NUM_ATTENTION_HEADS * HEAD_DIM, HIDDEN_SIZE, bias=ATTENTION_BIAS)
+        self.q_norm = RMSNorm(HEAD_DIM, eps=RMS_NORM_EPS)
+        self.k_norm = RMSNorm(HEAD_DIM, eps=RMS_NORM_EPS)
+
+    def forward(self, hidden_states, cos, sin, causal_mask):
+        bsz, seq_len, _ = hidden_states.shape
+        hidden_shape = (bsz, seq_len, -1, HEAD_DIM)
+
+        query_states, gate = torch.chunk(
+            self.q_proj(hidden_states).view(bsz, seq_len, -1, HEAD_DIM * 2), 2, dim=-1
+        )
+        gate = gate.reshape(bsz, seq_len, -1)
+
+        query_states = self.q_norm(query_states.view(hidden_shape)).transpose(1, 2)
+        key_states = self.k_norm(self.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
+        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+        rep = self.num_key_value_groups
+        key_states = key_states.repeat_interleave(rep, dim=1)
+        value_states = value_states.repeat_interleave(rep, dim=1)
+
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) * self.scaling
+        attn_weights = attn_weights + causal_mask
+        attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_output = torch.matmul(attn_weights, value_states)
+        attn_output = attn_output.transpose(1, 2).contiguous().reshape(bsz, seq_len, -1)
+        attn_output = attn_output * torch.sigmoid(gate)
+
+        return self.o_proj(attn_output)
+
+
+class MLPBlock(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.gate_proj = nn.Linear(HIDDEN_SIZE, INTERMEDIATE_SIZE, bias=False)
+        self.up_proj = nn.Linear(HIDDEN_SIZE, INTERMEDIATE_SIZE, bias=False)
+        self.down_proj = nn.Linear(INTERMEDIATE_SIZE, HIDDEN_SIZE, bias=False)
+
+    def forward(self, x):
+        return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
+
+
+class TransformerBlock(nn.Module):
+    """Наш decoder layer -- linear_attn ИЛИ self_attn, в зависимости от
+    layer_type, + общий MLP. Имена атрибутов совпадают с HF state_dict."""
+
+    def __init__(self, layer_idx: int, layer_type: str = None):
+        super().__init__()
+        # layer_type задан явно -> используем его (нужно для MTP-слоя,
+        # который всегда full_attention и лежит ВНЕ LAYER_TYPES).
+        self.block_type = layer_type if layer_type is not None else LAYER_TYPES[layer_idx]
+        if self.block_type == "linear_attention":
+            self.linear_attn = GatedDeltaNetBlock()
+        else:
+            self.self_attn = AttentionBlock()
+        self.mlp = MLPBlock()
+        self.input_layernorm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+        self.post_attention_layernorm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+
+    def forward(self, hidden_states, cos, sin, causal_mask):
+        residual = hidden_states
+        hidden_states = self.input_layernorm(hidden_states)
+        if self.block_type == "linear_attention":
+            hidden_states = self.linear_attn(hidden_states)
+        else:
+            hidden_states = self.self_attn(hidden_states, cos, sin, causal_mask)
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+        return hidden_states
+
+
+# =====================================================================
+# 5b. MTP / NextN-голова (draft-слой спекулятивного декода)
+# =====================================================================
+
+class MTPBlock(nn.Module):
+    """Multi-Token Prediction голова -- ровно как в реальном чекпоинте
+    Qwen3.5/3.8 (mtp_num_hidden_layers=1, mtp_use_dedicated_embeddings=false).
+
+    Имена подмодулей СОВПАДАЮТ с ключами HF state_dict (top-level `mtp.`):
+        mtp.pre_fc_norm_embedding.weight
+        mtp.pre_fc_norm_hidden.weight
+        mtp.fc.weight                       [HIDDEN, 2*HIDDEN]
+        mtp.layers.0.*                      (обычный full_attention decoder)
+        mtp.norm.weight                     (final norm перед общим lm_head)
+
+    embed_tokens и lm_head ОБЩИЕ со стволом (dedicated embeddings = false),
+    здесь их нет. В GGUF это blk.{NUM_LAYERS}.nextn.eh_proj / 
+    shared_head_norm + тензоры draft-слоя.
+
+    Механика (DeepSeek-style, порядок как в vLLM Qwen3NextMTP):
+        h = fc(cat(pre_fc_norm_embedding(emb_next), pre_fc_norm_hidden(h_prev)))
+        h = layers[0](h)          # full attention
+        logits2 = lm_head(norm(h))
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.pre_fc_norm_embedding = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+        self.pre_fc_norm_hidden = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+        self.fc = nn.Linear(2 * HIDDEN_SIZE, HIDDEN_SIZE, bias=False)
+        self.layers = nn.ModuleList([
+            TransformerBlock(layer_idx=NUM_LAYERS, layer_type="full_attention"),
+        ])
+        self.norm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+
+    def forward(self, inputs_embeds, previous_hidden, cos, sin, causal_mask):
+        """inputs_embeds  -- embed_tokens(токены, сдвинутые на +1)
+        previous_hidden   -- hidden ствола ДО финального model.norm
+        (см. контракт: MTP сам применяет pre_fc_norm_hidden)."""
+        h = self.fc(torch.cat(
+            (self.pre_fc_norm_embedding(inputs_embeds),
+             self.pre_fc_norm_hidden(previous_hidden)),
+            dim=-1,
+        ))
+        h = self.layers[0](h, cos, sin, causal_mask)
+        return self.norm(h)
+
+
+# =====================================================================
+# 6. Полная модель
+# =====================================================================
+
+class JiRackQwen38Model(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.embed_tokens = nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE)
+        self.layers = nn.ModuleList([TransformerBlock(i) for i in range(NUM_LAYERS)])
+        self.norm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+        self.rotary_emb = TextRotaryEmbedding()
+
+    def forward(self, input_ids: torch.LongTensor, return_prenorm: bool = False):
+        bsz, seq_len = input_ids.shape
+        hidden_states = self.embed_tokens(input_ids)
+
+        position_ids = torch.arange(seq_len, device=input_ids.device).unsqueeze(0).expand(bsz, -1)
+        cos, sin = self.rotary_emb(hidden_states, position_ids)
+
+        causal_mask = torch.full((seq_len, seq_len), float("-inf"), device=input_ids.device, dtype=torch.float32)
+        causal_mask = torch.triu(causal_mask, diagonal=1)[None, None, :, :].to(hidden_states.dtype)
+
+        for layer in self.layers:
+            hidden_states = layer(hidden_states, cos, sin, causal_mask)
+
+        if return_prenorm:
+            # Для MTP: голова сама применяет pre_fc_norm_hidden, финальный
+            # model.norm сюда НЕ входит (иначе двойная нормализация).
+            return self.norm(hidden_states), hidden_states, (cos, sin, causal_mask)
+        return self.norm(hidden_states)
+
+
+class JiRackQwen38ForCausalLM(nn.Module):
+    """Верхнеуровневая модель. Имена атрибутов (`model`, `lm_head`)
+    совпадают с HF Qwen3_5ForCausalLM -- state_dict грузится напрямую."""
+
+    def __init__(self):
+        super().__init__()
+        self.model = JiRackQwen38Model()
+        self.lm_head = nn.Linear(HIDDEN_SIZE, VOCAB_SIZE, bias=False)
+        # MTP/NextN-голова: top-level атрибут `mtp` -- ключи state_dict
+        # получаются ровно mtp.*, как в реальном HF-чекпоинте.
+        self.mtp = MTPBlock()
+
+    def forward(self, input_ids: torch.LongTensor) -> torch.Tensor:
+        hidden_states = self.model(input_ids)
+        return self.lm_head(hidden_states)
+
+    def mtp_forward(self, input_ids: torch.LongTensor):
+        """Возвращает (logits_t1, logits_t2): обычные логиты ствола и
+        draft-логиты MTP-головы (предсказание токена t+2 из позиции t).
+        Для инференса/валидации MTP; в GGUF-конвертации не участвует."""
+        _, prenorm_hidden, (cos, sin, causal_mask) = self.model(
+            input_ids, return_prenorm=True)
+        logits_t1 = self.lm_head(self.model.norm(prenorm_hidden))
+
+        # Вход MTP -- эмбеддинги токенов, сдвинутых на +1 (следующий токен
+        # уже известен на шаге верификации). Последнюю позицию дублируем,
+        # чтобы сохранить форму (draft для неё не используется).
+        shifted = torch.cat([input_ids[:, 1:], input_ids[:, -1:]], dim=1)
+        emb_next = self.model.embed_tokens(shifted)
+        h2 = self.mtp(emb_next, prenorm_hidden, cos, sin, causal_mask)
+        logits_t2 = self.lm_head(h2)
+        return logits_t1, logits_t2
+
+    # -----------------------------------------------------------------
+    # Заг��узка реальных весов (state_dict уже скачан конвертером через
+    # HF Hub -- сюда просто передаётся словарь тензоров)
+    # -----------------------------------------------------------------
+    def load_hf_state_dict(self, state_dict: dict, strict: bool = False):
+        """Грузит state_dict в формате оригинального Qwen3_5ForCausalLM.
+        mtp.* ключи теперь ГРУЗЯТСЯ в self.mtp (MTPBlock) -- фильтра
+        больше нет. strict=False оставлен из-за служебных lambda_-буферов
+        BitLinear, которых нет в реальном чекпоинте."""
+        missing, unexpected = self.load_state_dict(state_dict, strict=strict)
+        real_missing = [k for k in missing if not k.endswith("lambda_")]
+        if real_missing:
+            print(f"⚠️  {len(real_missing)} missing keys (первые 10): {real_missing[:10]}")
+        if unexpected:
+            print(f"⚠️  {len(unexpected)} unexpected keys (первые 10): {unexpected[:10]}")
+        return missing, unexpected
+
+    # -----------------------------------------------------------------
+    # Подсадка РЕАЛЬНЫХ mtp.* весов из оригинального HF-чекпоинта.
+    # Нужна, потому что старые JiRack .pt сохранены ДО появления MTPBlock
+    # и mtp-весов не содержат (там была бы случайная инициализация).
+    # -----------------------------------------------------------------
+    def load_mtp_from_hf_dir(self, hf_dir: str):
+        """Читает из папки оригинального HF-чекпоинта ТОЛЬКО mtp.* тензоры
+        (по index.json или перебором *.safetensors) и грузит их в self.mtp.
+        Возвращает число загруженных тензоров."""
+        import glob
+        import json as _json
+        from safetensors import safe_open
+
+        index_path = os.path.join(hf_dir, "model.safetensors.index.json")
+        mtp_map = {}  # key -> shard file
+        if os.path.exists(index_path):
+            with open(index_path) as f:
+                weight_map = _json.load(f)["weight_map"]
+            for k, shard in weight_map.items():
+                if k.startswith("mtp."):
+                    mtp_map[k] = os.path.join(hf_dir, shard)
+        else:
+            for shard in glob.glob(os.path.join(hf_dir, "*.safetensors")):
+                with safe_open(shard, framework="pt", device="cpu") as f:
+                    for k in f.keys():
+                        if k.startswith("mtp."):
+                            mtp_map[k] = shard
+
+        if not mtp_map:
+            raise FileNotFoundError(
+                f"В {hf_dir} не найдено ни одного mtp.* тензора. Нужен "
+                f"ОРИГИНАЛЬНЫЙ чекпоинт Qwen3.8-27B (не JiRack-экспорт!). "
+                f"Если оригинальные шарды удалены -- перекачай с HF Hub "
+                f"только шарды с mtp.* по model.safetensors.index.json.")
+
+        own_dtype = next(self.mtp.parameters()).dtype
+        loaded = {}
+        by_shard = {}
+        for k, shard in mtp_map.items():
+            by_shard.setdefault(shard, []).append(k)
+        for shard, keys in by_shard.items():
+            with safe_open(shard, framework="pt", device="cpu") as f:
+                for k in keys:
+                    loaded[k] = f.get_tensor(k).to(own_dtype)
+
+        missing, unexpected = self.load_state_dict(loaded, strict=False)
+        n_ok = len(loaded)
+        print(f"✅ MTP: загружено {n_ok} тензоров из {len(by_shard)} шардов ({hf_dir})")
+        if unexpected:
+            print(f"⚠️  MTP: unexpected keys: {unexpected[:10]}")
+        return n_ok
+
+    # -----------------------------------------------------------------
+    # JiRack-чекпоинт (свой формат, с lambda/patched_linear_names)
+    # -----------------------------------------------------------------
+    def save_checkpoint(self, path: str, lam: float, global_step: int = 0, patched_linear_names=None):
+        torch.save({
+            "model": self.state_dict(),
+            "lambda": lam,
+            "global_step": global_step,
+            "patched_linear_names": patched_linear_names or [],
+        }, path)
+
+    @classmethod
+    def load_checkpoint(cls, path: str, dtype=torch.bfloat16, include_gate_proj: bool = False):
+        ckpt = torch.load(path, map_location="cpu", weights_only=False)
+        model = cls()
+        patched_names = ckpt.get("patched_linear_names", [])
+        if patched_names:
+            for name in patched_names:
+                *parent_path, leaf = name.split(".")
+                parent = model
+                for p in parent_path:
+                    parent = getattr(parent, p)
+                old = getattr(parent, leaf)
+                if not isinstance(old, BitLinear):
+                    setattr(parent, leaf, BitLinear.from_linear(old))
+        else:
+            apply_bitlinear_patch(model, include_gate_proj=include_gate_proj)
+
+        has_mtp_in_ckpt = any(k.startswith("mtp.") for k in ckpt["model"])
+        model.load_state_dict(ckpt["model"], strict=False)
+        model = model.to(dtype)
+
+        if not has_mtp_in_ckpt:
+            print(
+                "⚠️  В этом .pt НЕТ mtp.* весов (чекпоинт сохранён до "
+                "появления MTPBlock). Голова сейчас со СЛУЧАЙНОЙ "
+                "инициализацией! Перед экспортом обязательно вызови "
+                "model.load_mtp_from_hf_dir('<папка оригинального HF-чекпоинта>')."
+            )
+
+        lam = ckpt.get("lambda", 0.0)
+        for m in model.modules():
+            if isinstance(m, BitLinear):
+                m.set_lambda(lam)
+
+        print(f"✅ Загружено. global_step={ckpt.get('global_step', 0)}, lambda={lam}")
+        return model
+
+
+# =====================================================================
+# 7. Демонстрация (без скачивания весов -- только структура)
+# =====================================================================
+
+if __name__ == "__main__":
+    print(ARCH_SUMMARY)
+    print(KEPT_FULL_PRECISION)
+
+    model = JiRackQwen38ForCausalLM()
+    n_params = sum(p.numel() for p in model.parameters())
+    print(f"✅ Модель собрана (случайные веса). Параметров: {n_params / 1e9:.2f}B")
+
+    patched = apply_bitlinear_patch(model)
+    print(f"✅ Заменено подмодулей на BitLinear: {len(patched)}")
+    print("Примеры первых 10:")
+    for n in patched[:10]:
+        print(f"   {n}")
+
+    ids = torch.randint(0, VOCAB_SIZE, (1, 8))
+    with torch.no_grad():
+        logits = model(ids)
+    print(f"✅ Пробный forward (случайные веса) -> logits shape: {tuple(logits.shape)}")

sft_final_fp8/audit_depth.json

@@ -0,0 +1,14 @@
+{
+  "kind": "depth",
+  "src": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded",
+  "out": "/mnt/nfs_share/JiRackDeltaNet_70b/expanded",
+  "tensors": 1714,
+  "ops": {
+    "copy": 1586,
+    "zero": 128
+  },
+  "errors": [],
+  "n_errors": 0,
+  "verdict": "PASS",
+  "minutes": 9.8
+}

sft_final_fp8/chat_fp8.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+# =============================================================================
+# chat_fp8.py - chat_jirack.py for the FP8 block-quantized copy (convert_fp8_block.py).
+#   python3 chat_fp8.py --model-dir /home/ubuntu/deltanet70b_sft/final_fp8 --no-think --temperature 0
+# Same commands/options as chat_jirack.py (it is imported from the same folder).
+# The quantized GEMM weights stay FP8 in memory (~72 GB for 70B) and are dequantized
+# per layer call (weight * 128x128 block scale -> bf16), so it runs on any GPU
+# (A100, H100, Blackwell); no FP8 tensor-core kernels are used.
+# =============================================================================
+import argparse
+import json
+import os
+import sys
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import chat_jirack as cj  # noqa: E402
+
+B = 128
+
+
+class FP8Linear(nn.Module):
+    def __init__(self, in_features, out_features):
+        super().__init__()
+        assert in_features % B == 0 and out_features % B == 0, (in_features, out_features)
+        self.in_features, self.out_features = in_features, out_features
+        self.register_buffer("weight", torch.empty(out_features, in_features, dtype=torch.float8_e4m3fn))
+        self.register_buffer("weight_scale_inv", torch.empty(out_features // B, in_features // B,
+                                                             dtype=torch.float32))
+
+    def forward(self, x):
+        R, C = self.out_features, self.in_features
+        w = self.weight.to(x.dtype).view(R // B, B, C // B, B) * self.weight_scale_inv.to(x.dtype)[:, None, :, None]
+        return F.linear(x, w.view(R, C))
+
+
+def _model_dir():
+    p = argparse.ArgumentParser(add_help=False)
+    p.add_argument("--model-dir", default="/mnt/nfs_share/JiRackDeltaNet_70b/expanded")
+    return p.parse_known_args()[0].model_dir
+
+
+def build_fp8(mod):
+    d = _model_dir()
+    wm = json.load(open(os.path.join(d, "model.safetensors.index.json")))["weight_map"]
+    qnames = {k[:-len(".weight_scale_inv")] for k in wm if k.endswith(".weight_scale_inv")}
+    if not qnames:
+        sys.exit(f"❌ no *.weight_scale_inv in {d} - not an FP8 copy, use chat_jirack.py")
+    torch.set_default_dtype(torch.bfloat16)
+    try:
+        with torch.device("meta"):
+            model = mod.JiRackQwen38ForCausalLM()
+            for name in qnames:
+                parent, _, leaf = name.rpartition(".")
+                pm = model.get_submodule(parent)
+                old = getattr(pm, leaf)
+                setattr(pm, leaf, FP8Linear(old.in_features, old.out_features))
+    finally:
+        torch.set_default_dtype(torch.float32)
+    model.to_empty(device="cpu")
+    for m in model.modules():                       # non-persistent buffer, not in the checkpoint
+        if isinstance(m, mod.TextRotaryEmbedding):
+            m.inv_freq = 1.0 / (mod.ROPE_THETA ** (torch.arange(0, m.rotary_dim, 2, dtype=torch.float32)
+                                                   / m.rotary_dim))
+    print(f"🧮 FP8: {len(qnames)} linear layers kept in FP8 (block {B}x{B})", flush=True)
+    return model.eval()
+
+
+cj.build = build_fp8
+
+if __name__ == "__main__":
+    cj.main()