diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..0d572b98da8d70f949173d164f51ef7ca0cec307 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
+sft_final/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+sft_final_fp8/tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/README.md b/README.md
index fa66110fc391a2f18974c4fed42163bb389b7b9a..b4f535d1fe3cfdbe6a1f5b43f1b9120a711f4d47 100644
--- a/README.md
+++ b/README.md
@@ -1,17 +1,15 @@
---
-
-license: bsl-1.0
+license: apache-2.0
library_name: pytorch
pipeline_tag: text-generation
tags:
-- gated-deltanet
-- linear-attention
-- hybrid-attention
-- depth-expansion
-- function-calling
-- reasoning
-- fp8
-
+ - gated-deltanet
+ - linear-attention
+ - hybrid-attention
+ - depth-expansion
+ - function-calling
+ - reasoning
+ - fp8
---
# JiRackDeltaNet 70B
@@ -21,7 +19,7 @@ Gated DeltaNet / gated full-attention model (Qwen 3.5 layout) built in our own J
**64 to 128 layers**, and fine-tuned on our own SFT mix (chat, reasoning, code, math, tool calling).
> Status: SFT release. The 64 added layers have been trained only by this SFT run (~45 M tokens), so most of the
-> model's ability still comes from the base model; see [Limitations](#limitations).
+> model's ability still comes from the base model; see Limitations.
## Model
@@ -111,9 +109,9 @@ The repository ships its own PyTorch code; no `transformers` model class is need
```bash
pip install torch safetensors tokenizers jinja2
# bf16, splits the 128 blocks over all visible GPUs (>= 2 x 80 GB)
-python3 chat_jirack.py --model-dir ./sft_final --no-think --temperature 0
-# FP8 copy (weights stay FP8 in memory, dequantized per layer call; any GPU)
-python3 chat_fp8.py --model-dir ./sft_final_fp8 --no-think --temperature 0
+python3 sft_final/chat_jirack.py --model-dir sft_final --no-think --temperature 0
+# FP8 copy (weights stay FP8 in memory, dequantized per layer call; one 96 GB GPU)
+python3 sft_final_fp8/chat_fp8.py --model-dir sft_final_fp8 --no-think --temperature 0
```
Commands in the chat: `/think` (reasoning on / off), `/reset`, `exit`. Prompts use the Qwen 3.5 chat template in
@@ -138,6 +136,6 @@ this is **not verified** yet.
## License
-BSL 1.0. The weights are derived from a Qwen model released under BSL 1.0 . Changes by CMSManhattan
+Apache License 2.0. The weights are derived from a Qwen model released under Apache 2.0. Changes by CMSManhattan
(Konstantin Grabko): JiRack model code, FFN widening, depth expansion to 128 layers, SFT on the JiRack SFT mix and the
-FP8 conversion. Keep this notice and the license text when redistributing.
\ No newline at end of file
+FP8 conversion. Keep this notice and the license text when redistributing.
diff --git a/sft_final/.gitattributes b/sft_final/.gitattributes
new file mode 100644
index 0000000000000000000000000000000000000000..52373fe24473b1aa44333d318f578ae6bf04b49b
--- /dev/null
+++ b/sft_final/.gitattributes
@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/sft_final/JiRackDeltaNet_70b.py b/sft_final/JiRackDeltaNet_70b.py
new file mode 100644
index 0000000000000000000000000000000000000000..c099cd18bd19038eef7c5dd2e59d0897bcd7d962
--- /dev/null
+++ b/sft_final/JiRackDeltaNet_70b.py
@@ -0,0 +1,766 @@
+"""
+JiRackDeltaNet_27b.py
+=============================
+
+Собственная (hand-rolled) архитектура "JiRack Qwen3.8-27B" -- в стиле
+прежнего DS32B-проекта: один файл, свой TransformerBlock, никакого
+AutoConfig/AutoModelForCausalLM внутри класса архитектуры, никакого
+model_id в конструкторе. Все размерности зашиты константами ниже --
+подтверждены живым инспектом реального checkpoint'а (inspect_qwen38_keys.py,
+851 тензор, сверено).
+
+Математика DeltaNet-рекуррентности, свёртки, гейтинга и partial-rotary
+mRoPE-attention ПОРТИРОВАНА из реального modeling_qwen3_5.py (Apache 2.0,
+HuggingFace/Qwen), а не переизобретена -- чтобы не рисковать тихой
+числовой ошибкой в тонкой рекуррентной формуле. Портирован смысл, но
+код переписан под наши имена/стиль и урезан: без vision-башни, без
+KV-cache классов transformers (для тренировки полный форвард по
+последовательности не нужен; для генерации -- полный recompute на
+каждом шаге, без инкрементального кэша; это сознательное упрощение,
+при желании кэш можно добавить отдельно).
+
+Имена подмодулей (`linear_attn` / `self_attn`, `in_proj_qkv`, `out_proj`,
+`mlp.gate_proj` и т.д.) намеренно СОВПАДАЮТ с именами в реальном
+state_dict Qwen3_5ForCausalLM -- поэтому веса из HF-чекпоинта грузятся
+напрямую через load_state_dict(), без ручного key-mapping.
+
+Файл умеет:
+ - собрать модель (JiRackQwen38ForCausalLM) с нуля, случайные веса;
+ - принять state_dict в формате HF Qwen3_5ForCausalLM (после
+ AutoModelForCausalLM.from_pretrained(...).state_dict() в конвертере)
+ и загрузить его напрямую;
+ - пропатчить нужные nn.Linear -> BitLinear (TERNARIZE_PATTERNS);
+ - сохранить/загрузить свой JiRack-чекпоинт (save_checkpoint/load_checkpoint).
+
+Конвертер (convert_qwen38_to_jirack.py) по-прежнему знает про model_id
+и HF Hub -- но это знание живёт ТОЛЬКО в конвертере, не в архитектуре.
+"""
+
+import os
+import re
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+# =====================================================================
+# 1. CONFIRMED АРХИТЕКТУРА -- жёстко зашитые константы (не AutoConfig)
+# =====================================================================
+
+HIDDEN_SIZE = 5120
+VOCAB_SIZE = 248320
+INTERMEDIATE_SIZE = 26624
+NUM_LAYERS = 128
+RMS_NORM_EPS = 1e-6
+
+# Чередование слоёв: full_attention_interval = 4 ->
+# 3x linear_attention, 1x full_attention, повторяется на все 64 слоя.
+LAYER_TYPES = ["linear_attention"] * 3 + ["full_attention"]
+LAYER_TYPES = (LAYER_TYPES * (NUM_LAYERS // len(LAYER_TYPES) + 1))[:NUM_LAYERS]
+
+# --- Gated DeltaNet (linear_attention) ---
+LINEAR_NUM_KEY_HEADS = 16
+LINEAR_KEY_HEAD_DIM = 128
+LINEAR_NUM_VALUE_HEADS = 48
+LINEAR_VALUE_HEAD_DIM = 128
+LINEAR_CONV_KERNEL_DIM = 4
+DELTA_KEY_DIM = LINEAR_NUM_KEY_HEADS * LINEAR_KEY_HEAD_DIM # 2048
+DELTA_VALUE_DIM = LINEAR_NUM_VALUE_HEADS * LINEAR_VALUE_HEAD_DIM # 6144
+DELTA_CONV_DIM = DELTA_KEY_DIM * 2 + DELTA_VALUE_DIM # 10240
+DELTA_CHUNK_SIZE = 64
+
+# --- Gated Attention (full_attention) ---
+NUM_ATTENTION_HEADS = 24
+NUM_KEY_VALUE_HEADS = 4
+HEAD_DIM = 256
+PARTIAL_ROTARY_FACTOR = 0.25
+ROPE_THETA = 1e7
+MROPE_SECTION = [11, 11, 10]
+ATTENTION_BIAS = False
+
+ARCH_SUMMARY = f"""
+Qwen3.8-27B (qwen3_5), hand-rolled JiRack TransformerBlock, text-only.
+hidden_size={HIDDEN_SIZE}, vocab_size={VOCAB_SIZE},
+intermediate_size={INTERMEDIATE_SIZE}, num_hidden_layers={NUM_LAYERS}
+Чередование (interval=4): 3x linear_attention (DeltaNet), 1x full_attention
+"""
+
+KEPT_FULL_PRECISION = """
+Всегда остаются full precision (не GEMM, либо слишком чувствительны):
+ - dt_bias, A_log (сырые Parameters, decay DeltaNet)
+ - linear_attn.in_proj_b/in_proj_a (по умолчанию, см. GATE_PROJ_PATTERNS)
+ - conv1d (depthwise grouped conv, не GEMM)
+ - все нормы: RMSNormGated, q_norm/k_norm, model.norm
+ - embed_tokens, lm_head
+ - mtp.* (вся MTP/NextN-голова) (draft-слой спекулятивного декода;
+ тернаризация убивает acceptance rate,
+ комьюнити-GGUF держат её в Q8_0)
+"""
+
+
+# =====================================================================
+# 2. BitLinear -- наш тернарный слой
+# =====================================================================
+
+class BitLinear(nn.Linear):
+ """b1.58-style per-tensor absmean ternary weights, per-token int8
+ activations, непрерывный lambda-warmup (STE)."""
+
+ def __init__(self, in_features, out_features, bias=False):
+ super().__init__(in_features, out_features, bias=bias)
+ self.eps = 1e-5
+ self.register_buffer("lambda_", torch.zeros(()), persistent=True)
+
+ @classmethod
+ def from_linear(cls, lin: nn.Linear) -> "BitLinear":
+ bl = cls(lin.in_features, lin.out_features, bias=lin.bias is not None)
+ bl = bl.to(dtype=lin.weight.dtype, device=lin.weight.device)
+ with torch.no_grad():
+ bl.weight.copy_(lin.weight)
+ if lin.bias is not None:
+ bl.bias.copy_(lin.bias)
+ return bl
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ if not self.training and float(self.lambda_) < 1e-6:
+ return F.linear(x, self.weight, self.bias)
+
+ lam = self.lambda_.to(x.dtype)
+ w = self.weight
+ gamma = w.float().abs().mean().clamp(min=self.eps).to(w.dtype)
+ w_quant = torch.clamp(torch.round(w / gamma), -1.0, 1.0) * gamma
+ w_effective = w + lam * (w_quant - w).detach()
+
+ x_scale = 127.0 / x.abs().max(dim=-1, keepdim=True).values.clamp(min=self.eps)
+ x_quant = torch.clamp(torch.round(x * x_scale), -128.0, 127.0) / x_scale
+ x_effective = x + lam * (x_quant - x).detach()
+
+ return F.linear(x_effective, w_effective, self.bias)
+
+ def set_lambda(self, v: float):
+ self.lambda_.fill_(v)
+
+
+# =====================================================================
+# 3. Карта: что тернаризуем, что оставляем full precision
+# =====================================================================
+
+TERNARIZE_PATTERNS = [
+ r"linear_attn\.in_proj_qkv$",
+ r"linear_attn\.in_proj_z$",
+ r"linear_attn\.out_proj$",
+ r"self_attn\.q_proj$",
+ r"self_attn\.k_proj$",
+ r"self_attn\.v_proj$",
+ r"self_attn\.o_proj$",
+ r"mlp\.gate_proj$",
+ r"mlp\.up_proj$",
+ r"mlp\.down_proj$",
+]
+
+GATE_PROJ_PATTERNS = [
+ r"linear_attn\.in_proj_b$",
+ r"linear_attn\.in_proj_a$",
+]
+
+
+def apply_bitlinear_patch(model: nn.Module, include_gate_proj: bool = False):
+ """Точечная замена nn.Linear -> BitLinear по TERNARIZE_PATTERNS."""
+ patterns = list(TERNARIZE_PATTERNS)
+ if include_gate_proj:
+ patterns += GATE_PROJ_PATTERNS
+ compiled = [re.compile(p) for p in patterns]
+
+ patched = []
+ for name, module in list(model.named_modules()):
+ if not isinstance(module, nn.Linear):
+ continue
+ # MTP/NextN-голова НЕ тернаризуется -- всегда full precision.
+ # Суффиксные паттерны (self_attn.q_proj$ и т.п.) иначе зацепили бы
+ # mtp.layers.0.self_attn.q_proj.
+ if name.startswith("mtp"):
+ continue
+ if any(p.search(name) for p in compiled):
+ parent_name, _, child_name = name.rpartition(".")
+ parent = model.get_submodule(parent_name) if parent_name else model
+ setattr(parent, child_name, BitLinear.from_linear(module))
+ patched.append(name)
+ return patched
+
+
+# =====================================================================
+# 4. Общие блоки (порт математики из реального modeling_qwen3_5.py)
+# =====================================================================
+
+class RMSNorm(nn.Module):
+ """(x * w).to(dtype), w хранится как (weight) с offset +1 -- как в
+ оригинале Qwen3_5RMSNorm (инициализация нулями, эффективный вес 1+w)."""
+
+ def __init__(self, dim: int, eps: float = RMS_NORM_EPS):
+ super().__init__()
+ self.eps = eps
+ self.weight = nn.Parameter(torch.zeros(dim))
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ dtype = x.dtype
+ x = x.float()
+ x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+ x = x * (1.0 + self.weight.float())
+ return x.to(dtype)
+
+
+class RMSNormGated(nn.Module):
+ """RMSNorm с гейтингом через SiLU(gate), применяется ПЕРЕД гейтингом
+ (норма считается по hidden_states, гейт применяется после)."""
+
+ def __init__(self, dim: int, eps: float = RMS_NORM_EPS):
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(dim))
+ self.eps = eps
+
+ def forward(self, x: torch.Tensor, gate: torch.Tensor) -> torch.Tensor:
+ dtype = x.dtype
+ x = x.float()
+ var = x.pow(2).mean(-1, keepdim=True)
+ x = x * torch.rsqrt(var + self.eps)
+ x = self.weight * x.to(dtype)
+ x = x * F.silu(gate.float())
+ return x.to(dtype)
+
+
+def rotate_half(x: torch.Tensor) -> torch.Tensor:
+ x1 = x[..., : x.shape[-1] // 2]
+ x2 = x[..., x.shape[-1] // 2:]
+ return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
+ """Partial rotary: только первые cos.shape[-1] каналов вращаются,
+ остальное (q_pass/k_pass) проходит без изменений."""
+ cos = cos.unsqueeze(unsqueeze_dim)
+ sin = sin.unsqueeze(unsqueeze_dim)
+ rotary_dim = cos.shape[-1]
+ q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:]
+ k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:]
+ q_embed = torch.cat([(q_rot * cos) + (rotate_half(q_rot) * sin), q_pass], dim=-1)
+ k_embed = torch.cat([(k_rot * cos) + (rotate_half(k_rot) * sin), k_pass], dim=-1)
+ return q_embed, k_embed
+
+
+class TextRotaryEmbedding(nn.Module):
+ """mRoPE с partial_rotary_factor. Для чистого текста (без vision)
+ 3 позиционные "оси" (T,H,W) совпадают -> interleaved mRoPE
+ вырождается в обычный RoPE, но мы всё равно считаем честно тем же
+ путём, что и оригинал, для полной числовой идентичности."""
+
+ def __init__(self, head_dim=HEAD_DIM, partial_rotary_factor=PARTIAL_ROTARY_FACTOR,
+ theta=ROPE_THETA, mrope_section=MROPE_SECTION):
+ super().__init__()
+ self.rotary_dim = int(head_dim * partial_rotary_factor)
+ inv_freq = 1.0 / (theta ** (torch.arange(0, self.rotary_dim, 2, dtype=torch.float32) / self.rotary_dim))
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
+ self.mrope_section = mrope_section
+
+ def _apply_interleaved_mrope(self, freqs: torch.Tensor) -> torch.Tensor:
+ # freqs: (3, bs, seq, rotary_dim // 2)
+ freqs_t = freqs[0].clone()
+ for dim, offset in enumerate((1, 2), start=1):
+ length = self.mrope_section[dim] * 3
+ idx = slice(offset, length, 3)
+ freqs_t[..., idx] = freqs[dim, ..., idx]
+ return freqs_t
+
+ @torch.no_grad()
+ def forward(self, x: torch.Tensor, position_ids: torch.LongTensor):
+ # position_ids: (bs, seq) -> реплицируем в 3 идентичные оси (text-only)
+ if position_ids.ndim == 2:
+ position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)
+ inv_freq_exp = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1)
+ pos_exp = position_ids[:, :, None, :].float()
+ freqs = (inv_freq_exp @ pos_exp).transpose(2, 3) # (3, bs, seq, rotary_dim//2)
+ freqs = self._apply_interleaved_mrope(freqs) # (bs, seq, rotary_dim//2)
+ emb = torch.cat((freqs, freqs), dim=-1) # (bs, seq, rotary_dim)
+ return emb.cos().to(x.dtype), emb.sin().to(x.dtype)
+
+
+def l2norm(x: torch.Tensor, dim: int = -1, eps: float = 1e-6):
+ return x * torch.rsqrt((x * x).sum(dim=dim, keepdim=True) + eps)
+
+
+def causal_conv1d(hidden_states: torch.Tensor, weight: torch.Tensor, activation: str = "silu"):
+ """hidden_states: (batch, channels, seq_len). Depthwise causal conv,
+ left-padded by kernel_size - 1, затем активация."""
+ channels, seq_len = hidden_states.shape[1], hidden_states.shape[2]
+ padding = weight.shape[-1] - 1
+ out = F.conv1d(hidden_states, weight=weight.unsqueeze(1), bias=None, padding=padding, groups=channels)
+ out = out[:, :, :seq_len]
+ return F.silu(out) if activation == "silu" else out
+
+
+def chunked_gated_delta_rule(query, key, value, g, beta, chunk_size=DELTA_CHUNK_SIZE):
+ """Порт torch_chunk_gated_delta_rule из modeling_qwen3_5.py -- без
+ initial_state/output_final_state (тренировка/полный форвард без
+ инкрементального кэша). query/key/value: (b, h, t, d), g/beta: (b, h, t)."""
+ initial_dtype = query.dtype
+ query = l2norm(query, dim=-1, eps=1e-6)
+ key = l2norm(key, dim=-1, eps=1e-6)
+ query, key, value, beta, g = [x.contiguous().to(torch.float32) for x in (query, key, value, beta, g)]
+
+ b, h, t, k_dim = key.shape
+ v_dim = value.shape[-1]
+ pad = (chunk_size - t % chunk_size) % chunk_size
+ query = F.pad(query, (0, 0, 0, pad))
+ key = F.pad(key, (0, 0, 0, pad))
+ value = F.pad(value, (0, 0, 0, pad))
+ beta = F.pad(beta, (0, pad))
+ g = F.pad(g, (0, pad))
+ total_t = t + pad
+ scale = 1 / (query.shape[-1] ** 0.5)
+ query = query * scale
+
+ v_beta = value * beta.unsqueeze(-1)
+ k_beta = key * beta.unsqueeze(-1)
+ query, key, value, k_beta, v_beta = [
+ x.reshape(x.shape[0], x.shape[1], -1, chunk_size, x.shape[-1]) for x in (query, key, value, k_beta, v_beta)
+ ]
+ g = g.reshape(g.shape[0], g.shape[1], -1, chunk_size)
+ mask = torch.triu(torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=0)
+
+ g = g.cumsum(dim=-1)
+ decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril()
+ attn = -((k_beta @ key.transpose(-1, -2)) * decay_mask).masked_fill(mask, 0)
+ for i in range(1, chunk_size):
+ row = attn[..., i, :i].clone()
+ sub = attn[..., :i, :i].clone()
+ attn[..., i, :i] = row + (row.unsqueeze(-1) * sub).sum(-2)
+ attn = attn + torch.eye(chunk_size, dtype=attn.dtype, device=attn.device)
+ value = attn @ v_beta
+ k_cumdecay = attn @ (k_beta * g.exp().unsqueeze(-1))
+
+ last_state = torch.zeros(b, h, k_dim, v_dim, dtype=value.dtype, device=value.device)
+ core_out = torch.zeros_like(value)
+
+ for i in range(total_t // chunk_size):
+ q_i, k_i, v_i = query[:, :, i], key[:, :, i], value[:, :, i]
+ attn_i = q_i @ k_i.transpose(-1, -2) * decay_mask[:, :, i]
+ v_prime = k_cumdecay[:, :, i] @ last_state
+ v_new = v_i - v_prime
+ attn_inter = (q_i * g[:, :, i, :, None].exp()) @ last_state
+ core_out[:, :, i] = attn_inter + attn_i @ v_new
+ last_state = (
+ last_state * g[:, :, i, -1, None, None].exp()
+ + (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2) @ v_new
+ )
+
+ core_out = core_out.reshape(core_out.shape[0], core_out.shape[1], -1, core_out.shape[-1])
+ core_out = core_out[:, :, :t]
+ core_out = core_out.transpose(1, 2).contiguous().to(initial_dtype)
+ return core_out
+
+
+# =====================================================================
+# 5. Наш собственный TransformerBlock (и подмодули linear_attn/self_attn)
+# =====================================================================
+
+class GatedDeltaNetBlock(nn.Module):
+ """linear_attention слой (Gated DeltaNet). Имена подмодулей совпадают
+ с HF state_dict, чтобы веса грузились напрямую."""
+
+ def __init__(self):
+ super().__init__()
+ self.conv1d = nn.Conv1d(
+ in_channels=DELTA_CONV_DIM, out_channels=DELTA_CONV_DIM, bias=False,
+ kernel_size=LINEAR_CONV_KERNEL_DIM, groups=DELTA_CONV_DIM,
+ padding=LINEAR_CONV_KERNEL_DIM - 1,
+ )
+ self.dt_bias = nn.Parameter(torch.ones(LINEAR_NUM_VALUE_HEADS))
+ A = torch.empty(LINEAR_NUM_VALUE_HEADS).uniform_(0.01, 16)
+ self.A_log = nn.Parameter(torch.log(A))
+ self.norm = RMSNormGated(LINEAR_VALUE_HEAD_DIM, eps=RMS_NORM_EPS)
+ self.out_proj = nn.Linear(DELTA_VALUE_DIM, HIDDEN_SIZE, bias=False)
+ self.in_proj_qkv = nn.Linear(HIDDEN_SIZE, DELTA_KEY_DIM * 2 + DELTA_VALUE_DIM, bias=False)
+ self.in_proj_z = nn.Linear(HIDDEN_SIZE, DELTA_VALUE_DIM, bias=False)
+ self.in_proj_b = nn.Linear(HIDDEN_SIZE, LINEAR_NUM_VALUE_HEADS, bias=False)
+ self.in_proj_a = nn.Linear(HIDDEN_SIZE, LINEAR_NUM_VALUE_HEADS, bias=False)
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ bsz, seq_len, _ = hidden_states.shape
+
+ mixed_qkv = self.in_proj_qkv(hidden_states).transpose(1, 2) # (b, conv_dim, t)
+ z = self.in_proj_z(hidden_states).reshape(bsz, seq_len, -1, LINEAR_VALUE_HEAD_DIM)
+ b = self.in_proj_b(hidden_states)
+ a = self.in_proj_a(hidden_states)
+
+ mixed_qkv = causal_conv1d(mixed_qkv, self.conv1d.weight.squeeze(1), activation="silu")
+ mixed_qkv = mixed_qkv.transpose(1, 2) # (b, t, conv_dim)
+
+ query, key, value = torch.split(mixed_qkv, [DELTA_KEY_DIM, DELTA_KEY_DIM, DELTA_VALUE_DIM], dim=-1)
+ query = query.reshape(bsz, seq_len, -1, LINEAR_KEY_HEAD_DIM)
+ key = key.reshape(bsz, seq_len, -1, LINEAR_KEY_HEAD_DIM)
+ value = value.reshape(bsz, seq_len, -1, LINEAR_VALUE_HEAD_DIM)
+
+ beta = b.sigmoid()
+ g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
+
+ rep = LINEAR_NUM_VALUE_HEADS // LINEAR_NUM_KEY_HEADS
+ if rep > 1:
+ query = query.repeat_interleave(rep, dim=2)
+ key = key.repeat_interleave(rep, dim=2)
+
+ # -> (b, h, t, d) for the chunked kernel
+ query_h = query.transpose(1, 2)
+ key_h = key.transpose(1, 2)
+ value_h = value.transpose(1, 2)
+ beta_h = beta.transpose(1, 2)
+ g_h = g.transpose(1, 2)
+
+ core_attn_out = chunked_gated_delta_rule(query_h, key_h, value_h, g_h, beta_h)
+
+ core_attn_out = core_attn_out.reshape(-1, LINEAR_VALUE_HEAD_DIM)
+ z_flat = z.reshape(-1, LINEAR_VALUE_HEAD_DIM)
+ core_attn_out = self.norm(core_attn_out, z_flat)
+ core_attn_out = core_attn_out.reshape(bsz, seq_len, -1)
+
+ return self.out_proj(core_attn_out)
+
+
+class AttentionBlock(nn.Module):
+ """full_attention слой (Gated Attention + partial rotary mRoPE)."""
+
+ def __init__(self):
+ super().__init__()
+ self.num_key_value_groups = NUM_ATTENTION_HEADS // NUM_KEY_VALUE_HEADS
+ self.scaling = HEAD_DIM ** -0.5
+ self.q_proj = nn.Linear(HIDDEN_SIZE, NUM_ATTENTION_HEADS * HEAD_DIM * 2, bias=ATTENTION_BIAS)
+ self.k_proj = nn.Linear(HIDDEN_SIZE, NUM_KEY_VALUE_HEADS * HEAD_DIM, bias=ATTENTION_BIAS)
+ self.v_proj = nn.Linear(HIDDEN_SIZE, NUM_KEY_VALUE_HEADS * HEAD_DIM, bias=ATTENTION_BIAS)
+ self.o_proj = nn.Linear(NUM_ATTENTION_HEADS * HEAD_DIM, HIDDEN_SIZE, bias=ATTENTION_BIAS)
+ self.q_norm = RMSNorm(HEAD_DIM, eps=RMS_NORM_EPS)
+ self.k_norm = RMSNorm(HEAD_DIM, eps=RMS_NORM_EPS)
+
+ def forward(self, hidden_states, cos, sin, causal_mask):
+ bsz, seq_len, _ = hidden_states.shape
+ hidden_shape = (bsz, seq_len, -1, HEAD_DIM)
+
+ query_states, gate = torch.chunk(
+ self.q_proj(hidden_states).view(bsz, seq_len, -1, HEAD_DIM * 2), 2, dim=-1
+ )
+ gate = gate.reshape(bsz, seq_len, -1)
+
+ query_states = self.q_norm(query_states.view(hidden_shape)).transpose(1, 2)
+ key_states = self.k_norm(self.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
+ value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+ rep = self.num_key_value_groups
+ key_states = key_states.repeat_interleave(rep, dim=1)
+ value_states = value_states.repeat_interleave(rep, dim=1)
+
+ attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) * self.scaling
+ attn_weights = attn_weights + causal_mask
+ attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+ attn_output = torch.matmul(attn_weights, value_states)
+ attn_output = attn_output.transpose(1, 2).contiguous().reshape(bsz, seq_len, -1)
+ attn_output = attn_output * torch.sigmoid(gate)
+
+ return self.o_proj(attn_output)
+
+
+class MLPBlock(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate_proj = nn.Linear(HIDDEN_SIZE, INTERMEDIATE_SIZE, bias=False)
+ self.up_proj = nn.Linear(HIDDEN_SIZE, INTERMEDIATE_SIZE, bias=False)
+ self.down_proj = nn.Linear(INTERMEDIATE_SIZE, HIDDEN_SIZE, bias=False)
+
+ def forward(self, x):
+ return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
+
+
+class TransformerBlock(nn.Module):
+ """Наш decoder layer -- linear_attn ИЛИ self_attn, в зависимости от
+ layer_type, + общий MLP. Имена атрибутов совпадают с HF state_dict."""
+
+ def __init__(self, layer_idx: int, layer_type: str = None):
+ super().__init__()
+ # layer_type задан явно -> используем его (нужно для MTP-слоя,
+ # который всегда full_attention и лежит ВНЕ LAYER_TYPES).
+ self.block_type = layer_type if layer_type is not None else LAYER_TYPES[layer_idx]
+ if self.block_type == "linear_attention":
+ self.linear_attn = GatedDeltaNetBlock()
+ else:
+ self.self_attn = AttentionBlock()
+ self.mlp = MLPBlock()
+ self.input_layernorm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+ self.post_attention_layernorm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+
+ def forward(self, hidden_states, cos, sin, causal_mask):
+ residual = hidden_states
+ hidden_states = self.input_layernorm(hidden_states)
+ if self.block_type == "linear_attention":
+ hidden_states = self.linear_attn(hidden_states)
+ else:
+ hidden_states = self.self_attn(hidden_states, cos, sin, causal_mask)
+ hidden_states = residual + hidden_states
+
+ residual = hidden_states
+ hidden_states = self.post_attention_layernorm(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + hidden_states
+ return hidden_states
+
+
+# =====================================================================
+# 5b. MTP / NextN-голова (draft-слой спекулятивного декода)
+# =====================================================================
+
+class MTPBlock(nn.Module):
+ """Multi-Token Prediction голова -- ровно как в реальном чекпоинте
+ Qwen3.5/3.8 (mtp_num_hidden_layers=1, mtp_use_dedicated_embeddings=false).
+
+ Имена подмодулей СОВПАДАЮТ с ключами HF state_dict (top-level `mtp.`):
+ mtp.pre_fc_norm_embedding.weight
+ mtp.pre_fc_norm_hidden.weight
+ mtp.fc.weight [HIDDEN, 2*HIDDEN]
+ mtp.layers.0.* (обычный full_attention decoder)
+ mtp.norm.weight (final norm перед общим lm_head)
+
+ embed_tokens и lm_head ОБЩИЕ со стволом (dedicated embeddings = false),
+ здесь их нет. В GGUF это blk.{NUM_LAYERS}.nextn.eh_proj /
+ shared_head_norm + тензоры draft-слоя.
+
+ Механика (DeepSeek-style, порядок как в vLLM Qwen3NextMTP):
+ h = fc(cat(pre_fc_norm_embedding(emb_next), pre_fc_norm_hidden(h_prev)))
+ h = layers[0](h) # full attention
+ logits2 = lm_head(norm(h))
+ """
+
+ def __init__(self):
+ super().__init__()
+ self.pre_fc_norm_embedding = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+ self.pre_fc_norm_hidden = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+ self.fc = nn.Linear(2 * HIDDEN_SIZE, HIDDEN_SIZE, bias=False)
+ self.layers = nn.ModuleList([
+ TransformerBlock(layer_idx=NUM_LAYERS, layer_type="full_attention"),
+ ])
+ self.norm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+
+ def forward(self, inputs_embeds, previous_hidden, cos, sin, causal_mask):
+ """inputs_embeds -- embed_tokens(токены, сдвинутые на +1)
+ previous_hidden -- hidden ствола ДО финального model.norm
+ (см. контракт: MTP сам применяет pre_fc_norm_hidden)."""
+ h = self.fc(torch.cat(
+ (self.pre_fc_norm_embedding(inputs_embeds),
+ self.pre_fc_norm_hidden(previous_hidden)),
+ dim=-1,
+ ))
+ h = self.layers[0](h, cos, sin, causal_mask)
+ return self.norm(h)
+
+
+# =====================================================================
+# 6. Полная модель
+# =====================================================================
+
+class JiRackQwen38Model(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.embed_tokens = nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE)
+ self.layers = nn.ModuleList([TransformerBlock(i) for i in range(NUM_LAYERS)])
+ self.norm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+ self.rotary_emb = TextRotaryEmbedding()
+
+ def forward(self, input_ids: torch.LongTensor, return_prenorm: bool = False):
+ bsz, seq_len = input_ids.shape
+ hidden_states = self.embed_tokens(input_ids)
+
+ position_ids = torch.arange(seq_len, device=input_ids.device).unsqueeze(0).expand(bsz, -1)
+ cos, sin = self.rotary_emb(hidden_states, position_ids)
+
+ causal_mask = torch.full((seq_len, seq_len), float("-inf"), device=input_ids.device, dtype=torch.float32)
+ causal_mask = torch.triu(causal_mask, diagonal=1)[None, None, :, :].to(hidden_states.dtype)
+
+ for layer in self.layers:
+ hidden_states = layer(hidden_states, cos, sin, causal_mask)
+
+ if return_prenorm:
+ # Для MTP: голова сама применяет pre_fc_norm_hidden, финальный
+ # model.norm сюда НЕ входит (иначе двойная нормализация).
+ return self.norm(hidden_states), hidden_states, (cos, sin, causal_mask)
+ return self.norm(hidden_states)
+
+
+class JiRackQwen38ForCausalLM(nn.Module):
+ """Верхнеуровневая модель. Имена атрибутов (`model`, `lm_head`)
+ совпадают с HF Qwen3_5ForCausalLM -- state_dict грузится напрямую."""
+
+ def __init__(self):
+ super().__init__()
+ self.model = JiRackQwen38Model()
+ self.lm_head = nn.Linear(HIDDEN_SIZE, VOCAB_SIZE, bias=False)
+ # MTP/NextN-голова: top-level атрибут `mtp` -- ключи state_dict
+ # получаются ровно mtp.*, как в реальном HF-чекпоинте.
+ self.mtp = MTPBlock()
+
+ def forward(self, input_ids: torch.LongTensor) -> torch.Tensor:
+ hidden_states = self.model(input_ids)
+ return self.lm_head(hidden_states)
+
+ def mtp_forward(self, input_ids: torch.LongTensor):
+ """Возвращает (logits_t1, logits_t2): обычные логиты ствола и
+ draft-логиты MTP-головы (предсказание токена t+2 из позиции t).
+ Для инференса/валидации MTP; в GGUF-конвертации не участвует."""
+ _, prenorm_hidden, (cos, sin, causal_mask) = self.model(
+ input_ids, return_prenorm=True)
+ logits_t1 = self.lm_head(self.model.norm(prenorm_hidden))
+
+ # Вход MTP -- эмбеддинги токенов, сдвинутых на +1 (следующий токен
+ # уже известен на шаге верификации). Последнюю позицию дублируем,
+ # чтобы сохранить форму (draft для неё не используется).
+ shifted = torch.cat([input_ids[:, 1:], input_ids[:, -1:]], dim=1)
+ emb_next = self.model.embed_tokens(shifted)
+ h2 = self.mtp(emb_next, prenorm_hidden, cos, sin, causal_mask)
+ logits_t2 = self.lm_head(h2)
+ return logits_t1, logits_t2
+
+ # -----------------------------------------------------------------
+ # Загрузка реальных весов (state_dict уже скачан конвертером через
+ # HF Hub -- сюда просто передаётся словарь тензоров)
+ # -----------------------------------------------------------------
+ def load_hf_state_dict(self, state_dict: dict, strict: bool = False):
+ """Грузит state_dict в формате оригинального Qwen3_5ForCausalLM.
+ mtp.* ключи теперь ГРУЗЯТСЯ в self.mtp (MTPBlock) -- фильтра
+ больше нет. strict=False оставлен из-за служебных lambda_-буферов
+ BitLinear, которых нет в реальном чекпоинте."""
+ missing, unexpected = self.load_state_dict(state_dict, strict=strict)
+ real_missing = [k for k in missing if not k.endswith("lambda_")]
+ if real_missing:
+ print(f"⚠️ {len(real_missing)} missing keys (первые 10): {real_missing[:10]}")
+ if unexpected:
+ print(f"⚠️ {len(unexpected)} unexpected keys (первые 10): {unexpected[:10]}")
+ return missing, unexpected
+
+ # -----------------------------------------------------------------
+ # Подсадка РЕАЛЬНЫХ mtp.* весов из оригинального HF-чекпоинта.
+ # Нужна, потому что старые JiRack .pt сохранены ДО появления MTPBlock
+ # и mtp-весов не содержат (там была бы случайная инициализация).
+ # -----------------------------------------------------------------
+ def load_mtp_from_hf_dir(self, hf_dir: str):
+ """Читает из папки оригинального HF-чекпоинта ТОЛЬКО mtp.* тензоры
+ (по index.json или перебором *.safetensors) и грузит их в self.mtp.
+ Возвращает число загруженных тензоров."""
+ import glob
+ import json as _json
+ from safetensors import safe_open
+
+ index_path = os.path.join(hf_dir, "model.safetensors.index.json")
+ mtp_map = {} # key -> shard file
+ if os.path.exists(index_path):
+ with open(index_path) as f:
+ weight_map = _json.load(f)["weight_map"]
+ for k, shard in weight_map.items():
+ if k.startswith("mtp."):
+ mtp_map[k] = os.path.join(hf_dir, shard)
+ else:
+ for shard in glob.glob(os.path.join(hf_dir, "*.safetensors")):
+ with safe_open(shard, framework="pt", device="cpu") as f:
+ for k in f.keys():
+ if k.startswith("mtp."):
+ mtp_map[k] = shard
+
+ if not mtp_map:
+ raise FileNotFoundError(
+ f"В {hf_dir} не найдено ни одного mtp.* тензора. Нужен "
+ f"ОРИГИНАЛЬНЫЙ чекпоинт Qwen3.8-27B (не JiRack-экспорт!). "
+ f"Если оригинальные шарды удалены -- перекачай с HF Hub "
+ f"только шарды с mtp.* по model.safetensors.index.json.")
+
+ own_dtype = next(self.mtp.parameters()).dtype
+ loaded = {}
+ by_shard = {}
+ for k, shard in mtp_map.items():
+ by_shard.setdefault(shard, []).append(k)
+ for shard, keys in by_shard.items():
+ with safe_open(shard, framework="pt", device="cpu") as f:
+ for k in keys:
+ loaded[k] = f.get_tensor(k).to(own_dtype)
+
+ missing, unexpected = self.load_state_dict(loaded, strict=False)
+ n_ok = len(loaded)
+ print(f"✅ MTP: загружено {n_ok} тензоров из {len(by_shard)} шардов ({hf_dir})")
+ if unexpected:
+ print(f"⚠️ MTP: unexpected keys: {unexpected[:10]}")
+ return n_ok
+
+ # -----------------------------------------------------------------
+ # JiRack-чекпоинт (свой формат, с lambda/patched_linear_names)
+ # -----------------------------------------------------------------
+ def save_checkpoint(self, path: str, lam: float, global_step: int = 0, patched_linear_names=None):
+ torch.save({
+ "model": self.state_dict(),
+ "lambda": lam,
+ "global_step": global_step,
+ "patched_linear_names": patched_linear_names or [],
+ }, path)
+
+ @classmethod
+ def load_checkpoint(cls, path: str, dtype=torch.bfloat16, include_gate_proj: bool = False):
+ ckpt = torch.load(path, map_location="cpu", weights_only=False)
+ model = cls()
+ patched_names = ckpt.get("patched_linear_names", [])
+ if patched_names:
+ for name in patched_names:
+ *parent_path, leaf = name.split(".")
+ parent = model
+ for p in parent_path:
+ parent = getattr(parent, p)
+ old = getattr(parent, leaf)
+ if not isinstance(old, BitLinear):
+ setattr(parent, leaf, BitLinear.from_linear(old))
+ else:
+ apply_bitlinear_patch(model, include_gate_proj=include_gate_proj)
+
+ has_mtp_in_ckpt = any(k.startswith("mtp.") for k in ckpt["model"])
+ model.load_state_dict(ckpt["model"], strict=False)
+ model = model.to(dtype)
+
+ if not has_mtp_in_ckpt:
+ print(
+ "⚠️ В этом .pt НЕТ mtp.* весов (чекпоинт сохранён до "
+ "появления MTPBlock). Голова сейчас со СЛУЧАЙНОЙ "
+ "инициализацией! Перед экспортом обязательно вызови "
+ "model.load_mtp_from_hf_dir('<папка оригинального HF-чекпоинта>')."
+ )
+
+ lam = ckpt.get("lambda", 0.0)
+ for m in model.modules():
+ if isinstance(m, BitLinear):
+ m.set_lambda(lam)
+
+ print(f"✅ Загружено. global_step={ckpt.get('global_step', 0)}, lambda={lam}")
+ return model
+
+
+# =====================================================================
+# 7. Демонстрация (без скачивания весов -- только структура)
+# =====================================================================
+
+if __name__ == "__main__":
+ print(ARCH_SUMMARY)
+ print(KEPT_FULL_PRECISION)
+
+ model = JiRackQwen38ForCausalLM()
+ n_params = sum(p.numel() for p in model.parameters())
+ print(f"✅ Модель собрана (случайные веса). Параметров: {n_params / 1e9:.2f}B")
+
+ patched = apply_bitlinear_patch(model)
+ print(f"✅ Заменено подмодулей на BitLinear: {len(patched)}")
+ print("Примеры первых 10:")
+ for n in patched[:10]:
+ print(f" {n}")
+
+ ids = torch.randint(0, VOCAB_SIZE, (1, 8))
+ with torch.no_grad():
+ logits = model(ids)
+ print(f"✅ Пробный forward (случайные веса) -> logits shape: {tuple(logits.shape)}")
diff --git a/sft_final/audit_depth.json b/sft_final/audit_depth.json
new file mode 100644
index 0000000000000000000000000000000000000000..420de2a404139630bbe23cc934c6f7fbae0b572d
--- /dev/null
+++ b/sft_final/audit_depth.json
@@ -0,0 +1,14 @@
+{
+ "kind": "depth",
+ "src": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded",
+ "out": "/mnt/nfs_share/JiRackDeltaNet_70b/expanded",
+ "tensors": 1714,
+ "ops": {
+ "copy": 1586,
+ "zero": 128
+ },
+ "errors": [],
+ "n_errors": 0,
+ "verdict": "PASS",
+ "minutes": 9.8
+}
\ No newline at end of file
diff --git a/sft_final/chat_jirack.py b/sft_final/chat_jirack.py
new file mode 100644
index 0000000000000000000000000000000000000000..b528fc3d015a027a33f5abf2c60d1dcd3b7d1711
--- /dev/null
+++ b/sft_final/chat_jirack.py
@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+# =============================================================================
+# chat_jirack.py - chat with JiRackDeltaNet (27B / 36B / 70B) from a safetensors folder.
+#
+# python3 chat_jirack.py --model-dir /mnt/nfs_share/JiRackDeltaNet_70b/expanded
+# python3 chat_jirack.py --model-dir /mnt/nfs_share/JiRackDeltaNet_27b_cp # 27B
+#
+# - model code: JiRackDeltaNet_*.py from the same folder (or --model-file)
+# - several GPUs: the TransformerBlocks are split across them evenly (70B bf16 = 140 GB,
+# so at least 2x80 GB); 1 GPU if it fits; CPU works but is VERY slow for 70B
+# - answer streams token by token, stops on <|im_end|> / <|endoftext|>
+# - commands: /reset (clear history), /think (thinking on/off), exit
+# =============================================================================
+import argparse
+import glob
+import importlib.util
+import itertools
+import json
+import os
+import sys
+import time
+
+import torch
+import torch.nn as nn
+
+INIT_FNS = ("uniform_", "normal_", "trunc_normal_", "kaiming_uniform_", "kaiming_normal_",
+ "xavier_uniform_", "xavier_normal_", "orthogonal_")
+
+
+def find_model_file(d):
+ files = sorted(glob.glob(os.path.join(d, "JiRackDeltaNet_*.py")))
+ if not files:
+ sys.exit(f"❌ no JiRackDeltaNet_*.py in {d}; pass --model-file")
+ return files[-1]
+
+
+def load_module(path):
+ spec = importlib.util.spec_from_file_location("jirack_chat_model", path)
+ mod = importlib.util.module_from_spec(spec)
+ sys.modules["jirack_chat_model"] = mod
+ spec.loader.exec_module(mod)
+ return mod
+
+
+def build(mod):
+ """bf16 skeleton without random init (all weights are loaded afterwards) - as in deltanet_gate.py."""
+ saved = {n: getattr(nn.init, n) for n in INIT_FNS if hasattr(nn.init, n)}
+ for n in saved:
+ setattr(nn.init, n, lambda t, *a, **k: t)
+ torch.set_default_dtype(torch.bfloat16)
+ try:
+ model = mod.JiRackQwen38ForCausalLM()
+ finally:
+ torch.set_default_dtype(torch.float32)
+ for n, f in saved.items():
+ setattr(nn.init, n, f)
+ return model.eval()
+
+
+def load_weights(model, d):
+ from safetensors import safe_open
+ idx = os.path.join(d, "model.safetensors.index.json")
+ files = sorted(set(json.load(open(idx))["weight_map"].values())) if os.path.exists(idx) else \
+ sorted(os.path.basename(p) for p in glob.glob(os.path.join(d, "*.safetensors")))
+ sd = model.state_dict()
+ params = {n for n, _ in model.named_parameters()}
+ loaded = set()
+ t0 = time.time()
+ with torch.no_grad():
+ for i, f in enumerate(files):
+ with safe_open(os.path.join(d, f), framework="pt", device="cpu") as h:
+ for k in h.keys():
+ if k in sd:
+ sd[k].copy_(h.get_tensor(k))
+ loaded.add(k)
+ print(f" {i + 1}/{len(files)} {f} {time.time() - t0:.0f} s", flush=True)
+ missing = sorted(params - loaded)
+ if missing:
+ sys.exit(f"❌ {len(missing)} parameters not in {d}: {missing[:5]}")
+
+
+class ChatTokenizer:
+ """tokenizer.json (tokenizers) + chat template (jinja2) - no transformers needed, so the
+ tokenizer_config.json format (e.g. extra_special_tokens as a list) does not matter."""
+
+ def __init__(self, d):
+ import jinja2
+ from jinja2.sandbox import ImmutableSandboxedEnvironment
+ from tokenizers import Tokenizer
+ self.tk = Tokenizer.from_file(os.path.join(d, "tokenizer.json"))
+ cfg = {}
+ if os.path.exists(os.path.join(d, "tokenizer_config.json")):
+ cfg = json.load(open(os.path.join(d, "tokenizer_config.json"), encoding="utf-8"))
+ path = os.path.join(d, "chat_template.jinja")
+ text = open(path, encoding="utf-8").read() if os.path.exists(path) else cfg.get("chat_template")
+ if isinstance(text, list):
+ text = next(t["template"] for t in text if t.get("name") == "default")
+ if not text:
+ sys.exit(f"❌ no chat template in {d}")
+
+ def raise_exception(msg):
+ raise jinja2.exceptions.TemplateError(msg)
+
+ def tojson(x, ensure_ascii=False, indent=None, separators=None, sort_keys=False):
+ return json.dumps(x, ensure_ascii=ensure_ascii, indent=indent, separators=separators,
+ sort_keys=sort_keys)
+
+ env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True)
+ env.filters["tojson"] = tojson
+ env.globals["raise_exception"] = raise_exception
+ self.tmpl = env.from_string(text)
+ s = lambda v: v.get("content") if isinstance(v, dict) else v
+ self.bos, self.eos = s(cfg.get("bos_token")) or "", s(cfg.get("eos_token")) or ""
+ self.special = [t.content for t in self.tk.get_added_tokens_decoder().values() if t.special]
+ self.stop_ids = {i for i in (self.tk.token_to_id("<|im_end|>"), self.tk.token_to_id("<|endoftext|>"),
+ self.tk.token_to_id(self.eos) if self.eos else None) if i is not None}
+
+ def prompt_ids(self, msgs, think):
+ text = self.tmpl.render(messages=msgs, add_generation_prompt=True, enable_thinking=think,
+ bos_token=self.bos, eos_token=self.eos)
+ return self.tk.encode(text, add_special_tokens=False).ids
+
+ def decode(self, ids):
+ return self.tk.decode(ids, skip_special_tokens=False)
+
+
+def _to(x, dev):
+ if torch.is_tensor(x):
+ return x.to(dev)
+ if isinstance(x, (tuple, list)):
+ return type(x)(_to(v, dev) for v in x)
+ if isinstance(x, dict):
+ return {k: _to(v, dev) for k, v in x.items()}
+ return x
+
+
+def place(model, devices):
+ """Blocks split evenly over the GPUs (moved there directly - the whole model never sits on one
+ GPU); everything else on the first GPU. Pre-hooks move each module's inputs to its device,
+ so the model code needs no changes."""
+ blocks = [m for m in model.modules() if type(m).__name__ == "TransformerBlock"]
+ in_block = {id(x) for b in blocks for x in b.modules()}
+ for i, b in enumerate(blocks):
+ dev = devices[i * len(devices) // len(blocks)]
+ b.to(dev)
+ if len(devices) > 1:
+ b.register_forward_pre_hook(lambda m, a, k, dev=dev: (_to(a, dev), _to(k, dev)), with_kwargs=True)
+ for m in model.modules(): # embeddings / norms / lm_head / mtp / rotary
+ if id(m) in in_block:
+ continue
+ m._apply(lambda t: t.to(devices[0]), recurse=False)
+ if len(devices) > 1 and not list(m.children()):
+ m.register_forward_pre_hook(lambda mm, a, k: (_to(a, devices[0]), _to(k, devices[0])),
+ with_kwargs=True)
+ return devices[0]
+
+
+def _cast(x, dtype):
+ if torch.is_tensor(x):
+ return x.to(dtype) if x.is_floating_point() and x.dtype != dtype else x
+ if isinstance(x, (tuple, list)):
+ return type(x)(_cast(v, dtype) for v in x)
+ if isinstance(x, dict):
+ return {k: _cast(v, dtype) for k, v in x.items()}
+ return x
+
+
+def fp32_blocks(model):
+ """CPU (AVX-only Xeon has no fast bf16 matmul): every TransformerBlock and lm_head runs with
+ fp32 weights/inputs, converted on entry and back on exit - stored weights stay bf16 (RAM),
+ same as the 'fp32blocks' compute of deltanet_gate.py."""
+ def pre(mod, args, kwargs):
+ saved = []
+ for t in itertools.chain(mod.parameters(), mod.buffers()):
+ if t.is_floating_point() and t.dtype != torch.float32:
+ saved.append((t, t.data))
+ t.data = t.data.float()
+ mod._jr_saved = saved
+ return _cast(args, torch.float32), _cast(kwargs, torch.float32)
+
+ def post(mod, args, out, back=torch.bfloat16):
+ for t, d in mod._jr_saved:
+ t.data = d
+ mod._jr_saved = []
+ return _cast(out, back) if back is not None else out
+
+ for m in model.modules():
+ if type(m).__name__ == "TransformerBlock":
+ m.register_forward_pre_hook(pre, with_kwargs=True)
+ m.register_forward_hook(post)
+ head = getattr(model, "lm_head", None)
+ if isinstance(head, nn.Module):
+ head.register_forward_pre_hook(pre, with_kwargs=True)
+ head.register_forward_hook(lambda m, a, o: post(m, a, o, None))
+
+
+def logits_of(out):
+ if isinstance(out, (tuple, list)):
+ return out[0]
+ return out.logits if hasattr(out, "logits") else out
+
+
+def sample(logits, temperature, top_p):
+ if temperature <= 0:
+ return int(logits.argmax())
+ probs = torch.softmax(logits.float() / temperature, dim=-1)
+ sp, si = probs.sort(descending=True)
+ cut = (sp.cumsum(-1) - sp) > top_p
+ sp[cut] = 0
+ return int(si[torch.multinomial(sp / sp.sum(), 1)])
+
+
+def main():
+ p = argparse.ArgumentParser()
+ p.add_argument("--model-dir", default="/mnt/nfs_share/JiRackDeltaNet_70b/expanded")
+ p.add_argument("--model-file", default=None)
+ p.add_argument("--tokenizer-dir", default=None, help="default: --model-dir")
+ p.add_argument("--max-new", type=int, default=2048)
+ p.add_argument("--temperature", type=float, default=0.6, help="0 = greedy")
+ p.add_argument("--top-p", type=float, default=0.95)
+ p.add_argument("--system", default=None, help="file with a system prompt")
+ p.add_argument("--no-think", action="store_true", help="start with thinking off")
+ p.add_argument("--cpu", action="store_true")
+ p.add_argument("--cpu-bf16", action="store_true", help="CPU: plain bf16 instead of fp32 blocks")
+ a = p.parse_args()
+
+ tok = ChatTokenizer(a.tokenizer_dir or a.model_dir)
+ stop = tok.stop_ids
+ print(f"stop tokens {sorted(stop)}", flush=True)
+
+ mf = a.model_file or find_model_file(a.model_dir)
+ print(f"model file {mf}\nweights {a.model_dir}", flush=True)
+ model = build(load_module(mf))
+ load_weights(model, a.model_dir)
+ n = sum(q.numel() for q in model.parameters())
+ if a.cpu or not torch.cuda.is_available():
+ dev = torch.device("cpu")
+ try:
+ torch.backends.nnpack.set_flags(False) # silences "Could not initialize NNPACK"
+ except Exception:
+ pass
+ if not a.cpu_bf16:
+ fp32_blocks(model)
+ print(f"⚠️ CPU: {n / 1e9:.1f}B params, compute {'bf16' if a.cpu_bf16 else 'fp32 blocks'} - "
+ f"every token is a full forward pass, expect minutes per token. Ctrl+C stops the answer.")
+ else:
+ devs = [torch.device("cuda", i) for i in range(torch.cuda.device_count())]
+ dev = place(model, devs)
+ print(f"on {len(devs)} GPU(s): {n / 1e9:.1f}B params, "
+ f"{[round(torch.cuda.memory_allocated(d) / 2 ** 30) for d in devs]} GB")
+
+ system = open(a.system, encoding="utf-8").read().strip() if a.system else None
+ think = not a.no_think
+ history = []
+ print("\nReady. /reset - clear history, /think - thinking on/off, exit - quit")
+ while True:
+ try:
+ user = input("\nYou: ").strip()
+ except (EOFError, KeyboardInterrupt):
+ break
+ if not user:
+ continue
+ if user.lower() in ("exit", "quit"):
+ break
+ if user == "/reset":
+ history = []
+ print("(history cleared)")
+ continue
+ if user == "/think":
+ think = not think
+ print(f"(thinking {'on' if think else 'off'})")
+ continue
+ history.append({"role": "user", "content": user})
+ msgs = ([{"role": "system", "content": system}] if system else []) + history
+ ids = torch.tensor([tok.prompt_ids(msgs, think)], dtype=torch.long, device=dev)
+ start = ids.shape[1]
+ print("\nAssistant: ", end="", flush=True)
+ printed, t0 = "", time.time()
+ try:
+ with torch.no_grad():
+ for _ in range(a.max_new):
+ nxt = sample(logits_of(model(ids))[0, -1], a.temperature, a.top_p)
+ if nxt in stop:
+ break
+ ids = torch.cat([ids, torch.tensor([[nxt]], device=ids.device)], dim=1)
+ text = tok.decode(ids[0, start:].tolist())
+ if not text.endswith("�"): # wait for a complete UTF-8 character
+ print(text[len(printed):], end="", flush=True)
+ printed = text
+ except KeyboardInterrupt:
+ print(" [stopped]", end="")
+ gen = ids.shape[1] - start
+ raw = tok.decode(ids[0, start:].tolist())
+ print(f"\n [{gen} tokens, {gen / max(time.time() - t0, 1e-6):.2f} tok/s]")
+ # Qwen: earlier turns go back into the history without the thinking part
+ reply = raw.split("")[-1]
+ for t in tok.special:
+ reply = reply.replace(t, "")
+ history.append({"role": "assistant", "content": reply.strip()})
+
+
+if __name__ == "__main__":
+ main()
diff --git a/sft_final/chat_template.jinja b/sft_final/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..2e0b08f0d19001d28b1b801b64a7a52c0b4cc088
--- /dev/null
+++ b/sft_final/chat_template.jinja
@@ -0,0 +1,171 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+ {%- if content is string %}
+ {{- content }}
+ {%- elif content is iterable and content is not mapping %}
+ {%- for item in content %}
+ {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+ {%- if is_system_content %}
+ {{- raise_exception('System message cannot contain images.') }}
+ {%- endif %}
+ {%- if do_vision_count %}
+ {%- set image_count.value = image_count.value + 1 %}
+ {%- endif %}
+ {%- if add_vision_id %}
+ {{- 'Picture ' ~ image_count.value ~ ': ' }}
+ {%- endif %}
+ {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+ {%- elif 'video' in item or item.type == 'video' %}
+ {%- if is_system_content %}
+ {{- raise_exception('System message cannot contain videos.') }}
+ {%- endif %}
+ {%- if do_vision_count %}
+ {%- set video_count.value = video_count.value + 1 %}
+ {%- endif %}
+ {%- if add_vision_id %}
+ {{- 'Video ' ~ video_count.value ~ ': ' }}
+ {%- endif %}
+ {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+ {%- elif 'text' in item %}
+ {{- item.text }}
+ {%- else %}
+ {{- raise_exception('Unexpected item type in content.') }}
+ {%- endif %}
+ {%- endfor %}
+ {%- elif content is none or content is undefined %}
+ {{- '' }}
+ {%- else %}
+ {{- raise_exception('Unexpected content type.') }}
+ {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+ {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- set reasoning_instructions = '' %}
+{%- if enable_thinking is defined and enable_thinking is true %}
+ {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}
+ {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}
+ {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}
+ {%- endif %}
+ {%- if resolved_reasoning_effort == 'xhigh' %}
+ {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}
+ {%- elif resolved_reasoning_effort == 'low' %}
+ {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}
+ {%- endif %}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+ {{- '<|im_start|>system\n' }}
+ {%- if reasoning_instructions %}
+ {{- reasoning_instructions + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou have access to the following functions:\n\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n" }}
+ {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }}
+ {%- if messages[0].role == 'system' %}
+ {%- set content = render_content(messages[0].content, false, true)|trim %}
+ {%- if content %}
+ {{- '\n\n' + content }}
+ {%- endif %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {%- set content = render_content(messages[0].content, false, true)|trim %}
+ {%- if content %}
+ {{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + content + '<|im_end|>\n' }}
+ {%- elif reasoning_instructions %}
+ {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
+ {%- endif %}
+ {%- elif reasoning_instructions %}
+ {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" %}
+ {%- set content = render_content(message.content, false)|trim %}
+ {%- if not(content.startswith('') and content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+ {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+ {%- set content = render_content(message.content, true)|trim %}
+ {%- if message.role == "system" %}
+ {%- if not loop.first %}
+ {{- raise_exception('System message must be at the beginning.') }}
+ {%- endif %}
+ {%- elif message.role == "user" %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- endif %}
+ {%- set reasoning_content = reasoning_content|trim %}
+ {%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {%- if loop.first %}
+ {%- if content|trim %}
+ {{- '\n\n\n\n' }}
+ {%- else %}
+ {{- '\n\n' }}
+ {%- endif %}
+ {%- else %}
+ {{- '\n\n\n' }}
+ {%- endif %}
+ {%- if tool_call.arguments is defined and tool_call.arguments != '' %}
+ {%- for args_name, args_value in tool_call.arguments|items %}
+ {{- '\n' }}
+ {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}
+ {{- args_value }}
+ {{- '\n\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.previtem and loop.previtem.role != "tool" %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if not loop.last and loop.nextitem.role != "tool" %}
+ {{- '<|im_end|>\n' }}
+ {%- elif loop.last %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- else %}
+ {{- raise_exception('Unexpected message role.') }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is true %}
+ {{- '\n' }}
+ {%- else %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
+
diff --git a/sft_final/config.json b/sft_final/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..0fc460a96e919e58d9146d8fbd530773df256728
--- /dev/null
+++ b/sft_final/config.json
@@ -0,0 +1,178 @@
+{
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "attn_output_gate": true,
+ "bos_token_id": 248044,
+ "dtype": "bfloat16",
+ "eos_token_id": 248044,
+ "full_attention_interval": 4,
+ "head_dim": 256,
+ "hidden_act": "silu",
+ "hidden_size": 5120,
+ "initializer_range": 0.02,
+ "intermediate_size": 26624,
+ "layer_types": [
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention"
+ ],
+ "linear_conv_kernel_dim": 4,
+ "linear_key_head_dim": 128,
+ "linear_num_key_heads": 16,
+ "linear_num_value_heads": 48,
+ "linear_value_head_dim": 128,
+ "mamba_ssm_dtype": "float32",
+ "max_position_embeddings": 262144,
+ "model_type": "qwen3_5_text",
+ "mtp_num_hidden_layers": 1,
+ "mtp_use_dedicated_embeddings": false,
+ "num_attention_heads": 24,
+ "num_hidden_layers": 128,
+ "num_key_value_heads": 4,
+ "output_gate_type": "swish",
+ "pad_token_id": null,
+ "partial_rotary_factor": 0.25,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "mrope_interleaved": true,
+ "mrope_section": [
+ 11,
+ 11,
+ 10
+ ],
+ "partial_rotary_factor": 0.25,
+ "rope_theta": 10000000,
+ "rope_type": "default"
+ },
+ "tie_word_embeddings": false,
+ "use_cache": true,
+ "vocab_size": 248320,
+ "architectures": [
+ "Qwen3_5ForCausalLM"
+ ]
+}
\ No newline at end of file
diff --git a/sft_final/expansion_depth.json b/sft_final/expansion_depth.json
new file mode 100644
index 0000000000000000000000000000000000000000..b7787cc90e337393983e2860798df1e45d7dc67c
--- /dev/null
+++ b/sft_final/expansion_depth.json
@@ -0,0 +1,17623 @@
+{
+ "kind": "depth",
+ "src": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded",
+ "layers_old": 64,
+ "layers_new": 128,
+ "hidden": 5120,
+ "intermediate": 26624,
+ "copies": [
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "layout": [
+ [
+ 0,
+ false
+ ],
+ [
+ 1,
+ false
+ ],
+ [
+ 2,
+ false
+ ],
+ [
+ 3,
+ false
+ ],
+ [
+ 0,
+ true
+ ],
+ [
+ 1,
+ true
+ ],
+ [
+ 2,
+ true
+ ],
+ [
+ 3,
+ true
+ ],
+ [
+ 4,
+ false
+ ],
+ [
+ 5,
+ false
+ ],
+ [
+ 6,
+ false
+ ],
+ [
+ 7,
+ false
+ ],
+ [
+ 4,
+ true
+ ],
+ [
+ 5,
+ true
+ ],
+ [
+ 6,
+ true
+ ],
+ [
+ 7,
+ true
+ ],
+ [
+ 8,
+ false
+ ],
+ [
+ 9,
+ false
+ ],
+ [
+ 10,
+ false
+ ],
+ [
+ 11,
+ false
+ ],
+ [
+ 8,
+ true
+ ],
+ [
+ 9,
+ true
+ ],
+ [
+ 10,
+ true
+ ],
+ [
+ 11,
+ true
+ ],
+ [
+ 12,
+ false
+ ],
+ [
+ 13,
+ false
+ ],
+ [
+ 14,
+ false
+ ],
+ [
+ 15,
+ false
+ ],
+ [
+ 12,
+ true
+ ],
+ [
+ 13,
+ true
+ ],
+ [
+ 14,
+ true
+ ],
+ [
+ 15,
+ true
+ ],
+ [
+ 16,
+ false
+ ],
+ [
+ 17,
+ false
+ ],
+ [
+ 18,
+ false
+ ],
+ [
+ 19,
+ false
+ ],
+ [
+ 16,
+ true
+ ],
+ [
+ 17,
+ true
+ ],
+ [
+ 18,
+ true
+ ],
+ [
+ 19,
+ true
+ ],
+ [
+ 20,
+ false
+ ],
+ [
+ 21,
+ false
+ ],
+ [
+ 22,
+ false
+ ],
+ [
+ 23,
+ false
+ ],
+ [
+ 20,
+ true
+ ],
+ [
+ 21,
+ true
+ ],
+ [
+ 22,
+ true
+ ],
+ [
+ 23,
+ true
+ ],
+ [
+ 24,
+ false
+ ],
+ [
+ 25,
+ false
+ ],
+ [
+ 26,
+ false
+ ],
+ [
+ 27,
+ false
+ ],
+ [
+ 24,
+ true
+ ],
+ [
+ 25,
+ true
+ ],
+ [
+ 26,
+ true
+ ],
+ [
+ 27,
+ true
+ ],
+ [
+ 28,
+ false
+ ],
+ [
+ 29,
+ false
+ ],
+ [
+ 30,
+ false
+ ],
+ [
+ 31,
+ false
+ ],
+ [
+ 28,
+ true
+ ],
+ [
+ 29,
+ true
+ ],
+ [
+ 30,
+ true
+ ],
+ [
+ 31,
+ true
+ ],
+ [
+ 32,
+ false
+ ],
+ [
+ 33,
+ false
+ ],
+ [
+ 34,
+ false
+ ],
+ [
+ 35,
+ false
+ ],
+ [
+ 32,
+ true
+ ],
+ [
+ 33,
+ true
+ ],
+ [
+ 34,
+ true
+ ],
+ [
+ 35,
+ true
+ ],
+ [
+ 36,
+ false
+ ],
+ [
+ 37,
+ false
+ ],
+ [
+ 38,
+ false
+ ],
+ [
+ 39,
+ false
+ ],
+ [
+ 36,
+ true
+ ],
+ [
+ 37,
+ true
+ ],
+ [
+ 38,
+ true
+ ],
+ [
+ 39,
+ true
+ ],
+ [
+ 40,
+ false
+ ],
+ [
+ 41,
+ false
+ ],
+ [
+ 42,
+ false
+ ],
+ [
+ 43,
+ false
+ ],
+ [
+ 40,
+ true
+ ],
+ [
+ 41,
+ true
+ ],
+ [
+ 42,
+ true
+ ],
+ [
+ 43,
+ true
+ ],
+ [
+ 44,
+ false
+ ],
+ [
+ 45,
+ false
+ ],
+ [
+ 46,
+ false
+ ],
+ [
+ 47,
+ false
+ ],
+ [
+ 44,
+ true
+ ],
+ [
+ 45,
+ true
+ ],
+ [
+ 46,
+ true
+ ],
+ [
+ 47,
+ true
+ ],
+ [
+ 48,
+ false
+ ],
+ [
+ 49,
+ false
+ ],
+ [
+ 50,
+ false
+ ],
+ [
+ 51,
+ false
+ ],
+ [
+ 48,
+ true
+ ],
+ [
+ 49,
+ true
+ ],
+ [
+ 50,
+ true
+ ],
+ [
+ 51,
+ true
+ ],
+ [
+ 52,
+ false
+ ],
+ [
+ 53,
+ false
+ ],
+ [
+ 54,
+ false
+ ],
+ [
+ 55,
+ false
+ ],
+ [
+ 52,
+ true
+ ],
+ [
+ 53,
+ true
+ ],
+ [
+ 54,
+ true
+ ],
+ [
+ 55,
+ true
+ ],
+ [
+ 56,
+ false
+ ],
+ [
+ 57,
+ false
+ ],
+ [
+ 58,
+ false
+ ],
+ [
+ 59,
+ false
+ ],
+ [
+ 56,
+ true
+ ],
+ [
+ 57,
+ true
+ ],
+ [
+ 58,
+ true
+ ],
+ [
+ 59,
+ true
+ ],
+ [
+ 60,
+ false
+ ],
+ [
+ 61,
+ false
+ ],
+ [
+ 62,
+ false
+ ],
+ [
+ 63,
+ false
+ ],
+ [
+ 60,
+ true
+ ],
+ [
+ 61,
+ true
+ ],
+ [
+ 62,
+ true
+ ],
+ [
+ 63,
+ true
+ ]
+ ],
+ "zeroed_in_copies": [
+ "linear_attn.out_proj.weight",
+ "self_attn.o_proj.weight",
+ "mlp.down_proj.weight"
+ ],
+ "params_before": 36521952256,
+ "params_after": 69934845440,
+ "shards": 27,
+ "bytes": 139869690880,
+ "side_files": [
+ "chat_template.jinja",
+ "config.json",
+ "gguf.txt",
+ "system_prompt.txt",
+ "tokenizer.json",
+ "tokenizer_config.json"
+ ],
+ "config_changes": [
+ "layer_types: 64 entries -> 128 (same 3 linear + 1 full pattern)",
+ "num_hidden_layers: 64 -> 128"
+ ],
+ "model_file_diff": [
+ "--- JiRackDeltaNet_27b.py",
+ "+++ JiRackDeltaNet_70b.py",
+ "@@ -52,2 +52,2 @@",
+ "-INTERMEDIATE_SIZE = 17408",
+ "-NUM_LAYERS = 64",
+ "+INTERMEDIATE_SIZE = 26624",
+ "+NUM_LAYERS = 128"
+ ],
+ "minutes": 8.5,
+ "recipe": [
+ {
+ "key": "model.embed_tokens.weight",
+ "src": "model.embed_tokens.weight",
+ "op": "copy"
+ },
+ {
+ "key": "model.layers.0.input_layernorm.weight",
+ "src": "model.layers.0.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.A_log",
+ "src": "model.layers.0.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.conv1d.weight",
+ "src": "model.layers.0.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.dt_bias",
+ "src": "model.layers.0.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.in_proj_a.weight",
+ "src": "model.layers.0.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.in_proj_b.weight",
+ "src": "model.layers.0.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.0.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.in_proj_z.weight",
+ "src": "model.layers.0.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.norm.weight",
+ "src": "model.layers.0.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.out_proj.weight",
+ "src": "model.layers.0.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.mlp.down_proj.weight",
+ "src": "model.layers.0.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.mlp.gate_proj.weight",
+ "src": "model.layers.0.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.mlp.up_proj.weight",
+ "src": "model.layers.0.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.post_attention_layernorm.weight",
+ "src": "model.layers.0.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.input_layernorm.weight",
+ "src": "model.layers.1.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.A_log",
+ "src": "model.layers.1.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.conv1d.weight",
+ "src": "model.layers.1.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.dt_bias",
+ "src": "model.layers.1.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.in_proj_a.weight",
+ "src": "model.layers.1.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.in_proj_b.weight",
+ "src": "model.layers.1.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.1.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.in_proj_z.weight",
+ "src": "model.layers.1.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.norm.weight",
+ "src": "model.layers.1.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.out_proj.weight",
+ "src": "model.layers.1.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.mlp.down_proj.weight",
+ "src": "model.layers.1.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.mlp.gate_proj.weight",
+ "src": "model.layers.1.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.mlp.up_proj.weight",
+ "src": "model.layers.1.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.post_attention_layernorm.weight",
+ "src": "model.layers.1.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.input_layernorm.weight",
+ "src": "model.layers.2.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.A_log",
+ "src": "model.layers.2.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.conv1d.weight",
+ "src": "model.layers.2.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.dt_bias",
+ "src": "model.layers.2.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.in_proj_a.weight",
+ "src": "model.layers.2.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.in_proj_b.weight",
+ "src": "model.layers.2.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.2.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.in_proj_z.weight",
+ "src": "model.layers.2.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.norm.weight",
+ "src": "model.layers.2.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.out_proj.weight",
+ "src": "model.layers.2.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.mlp.down_proj.weight",
+ "src": "model.layers.2.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.mlp.gate_proj.weight",
+ "src": "model.layers.2.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.mlp.up_proj.weight",
+ "src": "model.layers.2.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.post_attention_layernorm.weight",
+ "src": "model.layers.2.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.input_layernorm.weight",
+ "src": "model.layers.3.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.mlp.down_proj.weight",
+ "src": "model.layers.3.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.mlp.gate_proj.weight",
+ "src": "model.layers.3.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.mlp.up_proj.weight",
+ "src": "model.layers.3.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.post_attention_layernorm.weight",
+ "src": "model.layers.3.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.self_attn.k_norm.weight",
+ "src": "model.layers.3.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.self_attn.k_proj.weight",
+ "src": "model.layers.3.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.self_attn.o_proj.weight",
+ "src": "model.layers.3.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.self_attn.q_norm.weight",
+ "src": "model.layers.3.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.self_attn.q_proj.weight",
+ "src": "model.layers.3.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.self_attn.v_proj.weight",
+ "src": "model.layers.3.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.4.input_layernorm.weight",
+ "src": "model.layers.0.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.A_log",
+ "src": "model.layers.0.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.conv1d.weight",
+ "src": "model.layers.0.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.dt_bias",
+ "src": "model.layers.0.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.in_proj_a.weight",
+ "src": "model.layers.0.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.in_proj_b.weight",
+ "src": "model.layers.0.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.0.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.in_proj_z.weight",
+ "src": "model.layers.0.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.norm.weight",
+ "src": "model.layers.0.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.out_proj.weight",
+ "src": "model.layers.0.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.mlp.down_proj.weight",
+ "src": "model.layers.0.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.mlp.gate_proj.weight",
+ "src": "model.layers.0.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.mlp.up_proj.weight",
+ "src": "model.layers.0.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.post_attention_layernorm.weight",
+ "src": "model.layers.0.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.input_layernorm.weight",
+ "src": "model.layers.1.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.A_log",
+ "src": "model.layers.1.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.conv1d.weight",
+ "src": "model.layers.1.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.dt_bias",
+ "src": "model.layers.1.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.in_proj_a.weight",
+ "src": "model.layers.1.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.in_proj_b.weight",
+ "src": "model.layers.1.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.1.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.in_proj_z.weight",
+ "src": "model.layers.1.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.norm.weight",
+ "src": "model.layers.1.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.out_proj.weight",
+ "src": "model.layers.1.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.mlp.down_proj.weight",
+ "src": "model.layers.1.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.mlp.gate_proj.weight",
+ "src": "model.layers.1.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.mlp.up_proj.weight",
+ "src": "model.layers.1.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.post_attention_layernorm.weight",
+ "src": "model.layers.1.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.input_layernorm.weight",
+ "src": "model.layers.2.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.A_log",
+ "src": "model.layers.2.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.conv1d.weight",
+ "src": "model.layers.2.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.dt_bias",
+ "src": "model.layers.2.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.in_proj_a.weight",
+ "src": "model.layers.2.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.in_proj_b.weight",
+ "src": "model.layers.2.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.2.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.in_proj_z.weight",
+ "src": "model.layers.2.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.norm.weight",
+ "src": "model.layers.2.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.out_proj.weight",
+ "src": "model.layers.2.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.mlp.down_proj.weight",
+ "src": "model.layers.2.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.mlp.gate_proj.weight",
+ "src": "model.layers.2.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.mlp.up_proj.weight",
+ "src": "model.layers.2.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.post_attention_layernorm.weight",
+ "src": "model.layers.2.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.input_layernorm.weight",
+ "src": "model.layers.3.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.mlp.down_proj.weight",
+ "src": "model.layers.3.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.mlp.gate_proj.weight",
+ "src": "model.layers.3.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.mlp.up_proj.weight",
+ "src": "model.layers.3.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.post_attention_layernorm.weight",
+ "src": "model.layers.3.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.self_attn.k_norm.weight",
+ "src": "model.layers.3.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.self_attn.k_proj.weight",
+ "src": "model.layers.3.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.self_attn.o_proj.weight",
+ "src": "model.layers.3.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.self_attn.q_norm.weight",
+ "src": "model.layers.3.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.self_attn.q_proj.weight",
+ "src": "model.layers.3.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.self_attn.v_proj.weight",
+ "src": "model.layers.3.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.8.input_layernorm.weight",
+ "src": "model.layers.4.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.A_log",
+ "src": "model.layers.4.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.conv1d.weight",
+ "src": "model.layers.4.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.dt_bias",
+ "src": "model.layers.4.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.in_proj_a.weight",
+ "src": "model.layers.4.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.in_proj_b.weight",
+ "src": "model.layers.4.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.4.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.in_proj_z.weight",
+ "src": "model.layers.4.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.norm.weight",
+ "src": "model.layers.4.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.out_proj.weight",
+ "src": "model.layers.4.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.mlp.down_proj.weight",
+ "src": "model.layers.4.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.mlp.gate_proj.weight",
+ "src": "model.layers.4.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.mlp.up_proj.weight",
+ "src": "model.layers.4.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.post_attention_layernorm.weight",
+ "src": "model.layers.4.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.input_layernorm.weight",
+ "src": "model.layers.5.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.A_log",
+ "src": "model.layers.5.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.conv1d.weight",
+ "src": "model.layers.5.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.dt_bias",
+ "src": "model.layers.5.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.in_proj_a.weight",
+ "src": "model.layers.5.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.in_proj_b.weight",
+ "src": "model.layers.5.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.5.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.in_proj_z.weight",
+ "src": "model.layers.5.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.norm.weight",
+ "src": "model.layers.5.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.out_proj.weight",
+ "src": "model.layers.5.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.mlp.down_proj.weight",
+ "src": "model.layers.5.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.mlp.gate_proj.weight",
+ "src": "model.layers.5.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.mlp.up_proj.weight",
+ "src": "model.layers.5.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.post_attention_layernorm.weight",
+ "src": "model.layers.5.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.input_layernorm.weight",
+ "src": "model.layers.6.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.A_log",
+ "src": "model.layers.6.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.conv1d.weight",
+ "src": "model.layers.6.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.dt_bias",
+ "src": "model.layers.6.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.in_proj_a.weight",
+ "src": "model.layers.6.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.in_proj_b.weight",
+ "src": "model.layers.6.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.6.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.in_proj_z.weight",
+ "src": "model.layers.6.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.norm.weight",
+ "src": "model.layers.6.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.out_proj.weight",
+ "src": "model.layers.6.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.mlp.down_proj.weight",
+ "src": "model.layers.6.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.mlp.gate_proj.weight",
+ "src": "model.layers.6.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.mlp.up_proj.weight",
+ "src": "model.layers.6.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.post_attention_layernorm.weight",
+ "src": "model.layers.6.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.input_layernorm.weight",
+ "src": "model.layers.7.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.mlp.down_proj.weight",
+ "src": "model.layers.7.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.mlp.gate_proj.weight",
+ "src": "model.layers.7.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.mlp.up_proj.weight",
+ "src": "model.layers.7.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.post_attention_layernorm.weight",
+ "src": "model.layers.7.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.self_attn.k_norm.weight",
+ "src": "model.layers.7.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.self_attn.k_proj.weight",
+ "src": "model.layers.7.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.self_attn.o_proj.weight",
+ "src": "model.layers.7.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.self_attn.q_norm.weight",
+ "src": "model.layers.7.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.self_attn.q_proj.weight",
+ "src": "model.layers.7.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.self_attn.v_proj.weight",
+ "src": "model.layers.7.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.12.input_layernorm.weight",
+ "src": "model.layers.4.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.A_log",
+ "src": "model.layers.4.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.conv1d.weight",
+ "src": "model.layers.4.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.dt_bias",
+ "src": "model.layers.4.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.in_proj_a.weight",
+ "src": "model.layers.4.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.in_proj_b.weight",
+ "src": "model.layers.4.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.4.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.in_proj_z.weight",
+ "src": "model.layers.4.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.norm.weight",
+ "src": "model.layers.4.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.out_proj.weight",
+ "src": "model.layers.4.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.mlp.down_proj.weight",
+ "src": "model.layers.4.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.mlp.gate_proj.weight",
+ "src": "model.layers.4.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.mlp.up_proj.weight",
+ "src": "model.layers.4.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.post_attention_layernorm.weight",
+ "src": "model.layers.4.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.input_layernorm.weight",
+ "src": "model.layers.5.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.A_log",
+ "src": "model.layers.5.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.conv1d.weight",
+ "src": "model.layers.5.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.dt_bias",
+ "src": "model.layers.5.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.in_proj_a.weight",
+ "src": "model.layers.5.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.in_proj_b.weight",
+ "src": "model.layers.5.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.5.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.in_proj_z.weight",
+ "src": "model.layers.5.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.norm.weight",
+ "src": "model.layers.5.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.out_proj.weight",
+ "src": "model.layers.5.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.mlp.down_proj.weight",
+ "src": "model.layers.5.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.mlp.gate_proj.weight",
+ "src": "model.layers.5.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.mlp.up_proj.weight",
+ "src": "model.layers.5.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.post_attention_layernorm.weight",
+ "src": "model.layers.5.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.input_layernorm.weight",
+ "src": "model.layers.6.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.A_log",
+ "src": "model.layers.6.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.conv1d.weight",
+ "src": "model.layers.6.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.dt_bias",
+ "src": "model.layers.6.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.in_proj_a.weight",
+ "src": "model.layers.6.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.in_proj_b.weight",
+ "src": "model.layers.6.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.6.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.in_proj_z.weight",
+ "src": "model.layers.6.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.norm.weight",
+ "src": "model.layers.6.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.out_proj.weight",
+ "src": "model.layers.6.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.mlp.down_proj.weight",
+ "src": "model.layers.6.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.mlp.gate_proj.weight",
+ "src": "model.layers.6.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.mlp.up_proj.weight",
+ "src": "model.layers.6.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.post_attention_layernorm.weight",
+ "src": "model.layers.6.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.input_layernorm.weight",
+ "src": "model.layers.7.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.mlp.down_proj.weight",
+ "src": "model.layers.7.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.mlp.gate_proj.weight",
+ "src": "model.layers.7.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.mlp.up_proj.weight",
+ "src": "model.layers.7.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.post_attention_layernorm.weight",
+ "src": "model.layers.7.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.self_attn.k_norm.weight",
+ "src": "model.layers.7.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.self_attn.k_proj.weight",
+ "src": "model.layers.7.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.self_attn.o_proj.weight",
+ "src": "model.layers.7.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.self_attn.q_norm.weight",
+ "src": "model.layers.7.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.self_attn.q_proj.weight",
+ "src": "model.layers.7.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.self_attn.v_proj.weight",
+ "src": "model.layers.7.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.16.input_layernorm.weight",
+ "src": "model.layers.8.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.A_log",
+ "src": "model.layers.8.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.conv1d.weight",
+ "src": "model.layers.8.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.dt_bias",
+ "src": "model.layers.8.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.in_proj_a.weight",
+ "src": "model.layers.8.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.in_proj_b.weight",
+ "src": "model.layers.8.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.8.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.in_proj_z.weight",
+ "src": "model.layers.8.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.norm.weight",
+ "src": "model.layers.8.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.out_proj.weight",
+ "src": "model.layers.8.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.mlp.down_proj.weight",
+ "src": "model.layers.8.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.mlp.gate_proj.weight",
+ "src": "model.layers.8.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.mlp.up_proj.weight",
+ "src": "model.layers.8.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.post_attention_layernorm.weight",
+ "src": "model.layers.8.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.input_layernorm.weight",
+ "src": "model.layers.9.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.A_log",
+ "src": "model.layers.9.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.conv1d.weight",
+ "src": "model.layers.9.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.dt_bias",
+ "src": "model.layers.9.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.in_proj_a.weight",
+ "src": "model.layers.9.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.in_proj_b.weight",
+ "src": "model.layers.9.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.9.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.in_proj_z.weight",
+ "src": "model.layers.9.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.norm.weight",
+ "src": "model.layers.9.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.out_proj.weight",
+ "src": "model.layers.9.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.mlp.down_proj.weight",
+ "src": "model.layers.9.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.mlp.gate_proj.weight",
+ "src": "model.layers.9.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.mlp.up_proj.weight",
+ "src": "model.layers.9.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.post_attention_layernorm.weight",
+ "src": "model.layers.9.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.input_layernorm.weight",
+ "src": "model.layers.10.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.A_log",
+ "src": "model.layers.10.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.conv1d.weight",
+ "src": "model.layers.10.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.dt_bias",
+ "src": "model.layers.10.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.in_proj_a.weight",
+ "src": "model.layers.10.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.in_proj_b.weight",
+ "src": "model.layers.10.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.10.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.in_proj_z.weight",
+ "src": "model.layers.10.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.norm.weight",
+ "src": "model.layers.10.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.out_proj.weight",
+ "src": "model.layers.10.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.mlp.down_proj.weight",
+ "src": "model.layers.10.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.mlp.gate_proj.weight",
+ "src": "model.layers.10.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.mlp.up_proj.weight",
+ "src": "model.layers.10.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.post_attention_layernorm.weight",
+ "src": "model.layers.10.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.input_layernorm.weight",
+ "src": "model.layers.11.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.mlp.down_proj.weight",
+ "src": "model.layers.11.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.mlp.gate_proj.weight",
+ "src": "model.layers.11.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.mlp.up_proj.weight",
+ "src": "model.layers.11.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.post_attention_layernorm.weight",
+ "src": "model.layers.11.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.self_attn.k_norm.weight",
+ "src": "model.layers.11.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.self_attn.k_proj.weight",
+ "src": "model.layers.11.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.self_attn.o_proj.weight",
+ "src": "model.layers.11.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.self_attn.q_norm.weight",
+ "src": "model.layers.11.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.self_attn.q_proj.weight",
+ "src": "model.layers.11.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.self_attn.v_proj.weight",
+ "src": "model.layers.11.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.20.input_layernorm.weight",
+ "src": "model.layers.8.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.A_log",
+ "src": "model.layers.8.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.conv1d.weight",
+ "src": "model.layers.8.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.dt_bias",
+ "src": "model.layers.8.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.in_proj_a.weight",
+ "src": "model.layers.8.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.in_proj_b.weight",
+ "src": "model.layers.8.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.8.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.in_proj_z.weight",
+ "src": "model.layers.8.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.norm.weight",
+ "src": "model.layers.8.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.out_proj.weight",
+ "src": "model.layers.8.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.mlp.down_proj.weight",
+ "src": "model.layers.8.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.mlp.gate_proj.weight",
+ "src": "model.layers.8.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.mlp.up_proj.weight",
+ "src": "model.layers.8.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.post_attention_layernorm.weight",
+ "src": "model.layers.8.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.input_layernorm.weight",
+ "src": "model.layers.9.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.A_log",
+ "src": "model.layers.9.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.conv1d.weight",
+ "src": "model.layers.9.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.dt_bias",
+ "src": "model.layers.9.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.in_proj_a.weight",
+ "src": "model.layers.9.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.in_proj_b.weight",
+ "src": "model.layers.9.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.9.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.in_proj_z.weight",
+ "src": "model.layers.9.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.norm.weight",
+ "src": "model.layers.9.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.out_proj.weight",
+ "src": "model.layers.9.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.mlp.down_proj.weight",
+ "src": "model.layers.9.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.mlp.gate_proj.weight",
+ "src": "model.layers.9.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.mlp.up_proj.weight",
+ "src": "model.layers.9.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.post_attention_layernorm.weight",
+ "src": "model.layers.9.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.input_layernorm.weight",
+ "src": "model.layers.10.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.A_log",
+ "src": "model.layers.10.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.conv1d.weight",
+ "src": "model.layers.10.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.dt_bias",
+ "src": "model.layers.10.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.in_proj_a.weight",
+ "src": "model.layers.10.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.in_proj_b.weight",
+ "src": "model.layers.10.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.10.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.in_proj_z.weight",
+ "src": "model.layers.10.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.norm.weight",
+ "src": "model.layers.10.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.out_proj.weight",
+ "src": "model.layers.10.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.mlp.down_proj.weight",
+ "src": "model.layers.10.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.mlp.gate_proj.weight",
+ "src": "model.layers.10.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.mlp.up_proj.weight",
+ "src": "model.layers.10.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.post_attention_layernorm.weight",
+ "src": "model.layers.10.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.input_layernorm.weight",
+ "src": "model.layers.11.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.mlp.down_proj.weight",
+ "src": "model.layers.11.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.mlp.gate_proj.weight",
+ "src": "model.layers.11.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.mlp.up_proj.weight",
+ "src": "model.layers.11.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.post_attention_layernorm.weight",
+ "src": "model.layers.11.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.self_attn.k_norm.weight",
+ "src": "model.layers.11.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.self_attn.k_proj.weight",
+ "src": "model.layers.11.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.self_attn.o_proj.weight",
+ "src": "model.layers.11.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.self_attn.q_norm.weight",
+ "src": "model.layers.11.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.self_attn.q_proj.weight",
+ "src": "model.layers.11.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.self_attn.v_proj.weight",
+ "src": "model.layers.11.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.24.input_layernorm.weight",
+ "src": "model.layers.12.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.A_log",
+ "src": "model.layers.12.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.conv1d.weight",
+ "src": "model.layers.12.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.dt_bias",
+ "src": "model.layers.12.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.in_proj_a.weight",
+ "src": "model.layers.12.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.in_proj_b.weight",
+ "src": "model.layers.12.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.12.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.in_proj_z.weight",
+ "src": "model.layers.12.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.norm.weight",
+ "src": "model.layers.12.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.out_proj.weight",
+ "src": "model.layers.12.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.mlp.down_proj.weight",
+ "src": "model.layers.12.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.mlp.gate_proj.weight",
+ "src": "model.layers.12.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.mlp.up_proj.weight",
+ "src": "model.layers.12.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.post_attention_layernorm.weight",
+ "src": "model.layers.12.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.input_layernorm.weight",
+ "src": "model.layers.13.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.A_log",
+ "src": "model.layers.13.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.conv1d.weight",
+ "src": "model.layers.13.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.dt_bias",
+ "src": "model.layers.13.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.in_proj_a.weight",
+ "src": "model.layers.13.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.in_proj_b.weight",
+ "src": "model.layers.13.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.13.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.in_proj_z.weight",
+ "src": "model.layers.13.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.norm.weight",
+ "src": "model.layers.13.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.out_proj.weight",
+ "src": "model.layers.13.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.mlp.down_proj.weight",
+ "src": "model.layers.13.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.mlp.gate_proj.weight",
+ "src": "model.layers.13.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.mlp.up_proj.weight",
+ "src": "model.layers.13.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.post_attention_layernorm.weight",
+ "src": "model.layers.13.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.input_layernorm.weight",
+ "src": "model.layers.14.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.A_log",
+ "src": "model.layers.14.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.conv1d.weight",
+ "src": "model.layers.14.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.dt_bias",
+ "src": "model.layers.14.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.in_proj_a.weight",
+ "src": "model.layers.14.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.in_proj_b.weight",
+ "src": "model.layers.14.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.14.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.in_proj_z.weight",
+ "src": "model.layers.14.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.norm.weight",
+ "src": "model.layers.14.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.out_proj.weight",
+ "src": "model.layers.14.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.mlp.down_proj.weight",
+ "src": "model.layers.14.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.mlp.gate_proj.weight",
+ "src": "model.layers.14.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.mlp.up_proj.weight",
+ "src": "model.layers.14.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.post_attention_layernorm.weight",
+ "src": "model.layers.14.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.input_layernorm.weight",
+ "src": "model.layers.15.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.mlp.down_proj.weight",
+ "src": "model.layers.15.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.mlp.gate_proj.weight",
+ "src": "model.layers.15.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.mlp.up_proj.weight",
+ "src": "model.layers.15.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.post_attention_layernorm.weight",
+ "src": "model.layers.15.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.self_attn.k_norm.weight",
+ "src": "model.layers.15.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.self_attn.k_proj.weight",
+ "src": "model.layers.15.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.self_attn.o_proj.weight",
+ "src": "model.layers.15.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.self_attn.q_norm.weight",
+ "src": "model.layers.15.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.self_attn.q_proj.weight",
+ "src": "model.layers.15.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.self_attn.v_proj.weight",
+ "src": "model.layers.15.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.28.input_layernorm.weight",
+ "src": "model.layers.12.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.A_log",
+ "src": "model.layers.12.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.conv1d.weight",
+ "src": "model.layers.12.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.dt_bias",
+ "src": "model.layers.12.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.in_proj_a.weight",
+ "src": "model.layers.12.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.in_proj_b.weight",
+ "src": "model.layers.12.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.12.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.in_proj_z.weight",
+ "src": "model.layers.12.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.norm.weight",
+ "src": "model.layers.12.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.out_proj.weight",
+ "src": "model.layers.12.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.mlp.down_proj.weight",
+ "src": "model.layers.12.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.mlp.gate_proj.weight",
+ "src": "model.layers.12.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.mlp.up_proj.weight",
+ "src": "model.layers.12.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.post_attention_layernorm.weight",
+ "src": "model.layers.12.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.input_layernorm.weight",
+ "src": "model.layers.13.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.A_log",
+ "src": "model.layers.13.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.conv1d.weight",
+ "src": "model.layers.13.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.dt_bias",
+ "src": "model.layers.13.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.in_proj_a.weight",
+ "src": "model.layers.13.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.in_proj_b.weight",
+ "src": "model.layers.13.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.13.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.in_proj_z.weight",
+ "src": "model.layers.13.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.norm.weight",
+ "src": "model.layers.13.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.out_proj.weight",
+ "src": "model.layers.13.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.mlp.down_proj.weight",
+ "src": "model.layers.13.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.mlp.gate_proj.weight",
+ "src": "model.layers.13.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.mlp.up_proj.weight",
+ "src": "model.layers.13.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.post_attention_layernorm.weight",
+ "src": "model.layers.13.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.input_layernorm.weight",
+ "src": "model.layers.14.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.A_log",
+ "src": "model.layers.14.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.conv1d.weight",
+ "src": "model.layers.14.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.dt_bias",
+ "src": "model.layers.14.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.in_proj_a.weight",
+ "src": "model.layers.14.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.in_proj_b.weight",
+ "src": "model.layers.14.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.14.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.in_proj_z.weight",
+ "src": "model.layers.14.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.norm.weight",
+ "src": "model.layers.14.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.out_proj.weight",
+ "src": "model.layers.14.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.mlp.down_proj.weight",
+ "src": "model.layers.14.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.mlp.gate_proj.weight",
+ "src": "model.layers.14.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.mlp.up_proj.weight",
+ "src": "model.layers.14.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.post_attention_layernorm.weight",
+ "src": "model.layers.14.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.input_layernorm.weight",
+ "src": "model.layers.15.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.mlp.down_proj.weight",
+ "src": "model.layers.15.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.mlp.gate_proj.weight",
+ "src": "model.layers.15.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.mlp.up_proj.weight",
+ "src": "model.layers.15.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.post_attention_layernorm.weight",
+ "src": "model.layers.15.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.self_attn.k_norm.weight",
+ "src": "model.layers.15.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.self_attn.k_proj.weight",
+ "src": "model.layers.15.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.self_attn.o_proj.weight",
+ "src": "model.layers.15.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.self_attn.q_norm.weight",
+ "src": "model.layers.15.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.self_attn.q_proj.weight",
+ "src": "model.layers.15.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.self_attn.v_proj.weight",
+ "src": "model.layers.15.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.32.input_layernorm.weight",
+ "src": "model.layers.16.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.A_log",
+ "src": "model.layers.16.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.conv1d.weight",
+ "src": "model.layers.16.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.dt_bias",
+ "src": "model.layers.16.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.in_proj_a.weight",
+ "src": "model.layers.16.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.in_proj_b.weight",
+ "src": "model.layers.16.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.16.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.in_proj_z.weight",
+ "src": "model.layers.16.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.norm.weight",
+ "src": "model.layers.16.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.out_proj.weight",
+ "src": "model.layers.16.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.mlp.down_proj.weight",
+ "src": "model.layers.16.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.mlp.gate_proj.weight",
+ "src": "model.layers.16.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.mlp.up_proj.weight",
+ "src": "model.layers.16.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.post_attention_layernorm.weight",
+ "src": "model.layers.16.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.input_layernorm.weight",
+ "src": "model.layers.17.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.A_log",
+ "src": "model.layers.17.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.conv1d.weight",
+ "src": "model.layers.17.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.dt_bias",
+ "src": "model.layers.17.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.in_proj_a.weight",
+ "src": "model.layers.17.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.in_proj_b.weight",
+ "src": "model.layers.17.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.17.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.in_proj_z.weight",
+ "src": "model.layers.17.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.norm.weight",
+ "src": "model.layers.17.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.out_proj.weight",
+ "src": "model.layers.17.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.mlp.down_proj.weight",
+ "src": "model.layers.17.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.mlp.gate_proj.weight",
+ "src": "model.layers.17.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.mlp.up_proj.weight",
+ "src": "model.layers.17.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.post_attention_layernorm.weight",
+ "src": "model.layers.17.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.input_layernorm.weight",
+ "src": "model.layers.18.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.A_log",
+ "src": "model.layers.18.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.conv1d.weight",
+ "src": "model.layers.18.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.dt_bias",
+ "src": "model.layers.18.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.in_proj_a.weight",
+ "src": "model.layers.18.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.in_proj_b.weight",
+ "src": "model.layers.18.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.18.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.in_proj_z.weight",
+ "src": "model.layers.18.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.norm.weight",
+ "src": "model.layers.18.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.out_proj.weight",
+ "src": "model.layers.18.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.mlp.down_proj.weight",
+ "src": "model.layers.18.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.mlp.gate_proj.weight",
+ "src": "model.layers.18.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.mlp.up_proj.weight",
+ "src": "model.layers.18.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.post_attention_layernorm.weight",
+ "src": "model.layers.18.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.input_layernorm.weight",
+ "src": "model.layers.19.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.mlp.down_proj.weight",
+ "src": "model.layers.19.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.mlp.gate_proj.weight",
+ "src": "model.layers.19.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.mlp.up_proj.weight",
+ "src": "model.layers.19.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.post_attention_layernorm.weight",
+ "src": "model.layers.19.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.self_attn.k_norm.weight",
+ "src": "model.layers.19.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.self_attn.k_proj.weight",
+ "src": "model.layers.19.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.self_attn.o_proj.weight",
+ "src": "model.layers.19.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.self_attn.q_norm.weight",
+ "src": "model.layers.19.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.self_attn.q_proj.weight",
+ "src": "model.layers.19.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.self_attn.v_proj.weight",
+ "src": "model.layers.19.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.36.input_layernorm.weight",
+ "src": "model.layers.16.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.A_log",
+ "src": "model.layers.16.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.conv1d.weight",
+ "src": "model.layers.16.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.dt_bias",
+ "src": "model.layers.16.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.in_proj_a.weight",
+ "src": "model.layers.16.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.in_proj_b.weight",
+ "src": "model.layers.16.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.16.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.in_proj_z.weight",
+ "src": "model.layers.16.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.norm.weight",
+ "src": "model.layers.16.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.out_proj.weight",
+ "src": "model.layers.16.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.mlp.down_proj.weight",
+ "src": "model.layers.16.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.mlp.gate_proj.weight",
+ "src": "model.layers.16.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.mlp.up_proj.weight",
+ "src": "model.layers.16.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.post_attention_layernorm.weight",
+ "src": "model.layers.16.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.input_layernorm.weight",
+ "src": "model.layers.17.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.A_log",
+ "src": "model.layers.17.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.conv1d.weight",
+ "src": "model.layers.17.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.dt_bias",
+ "src": "model.layers.17.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.in_proj_a.weight",
+ "src": "model.layers.17.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.in_proj_b.weight",
+ "src": "model.layers.17.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.17.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.in_proj_z.weight",
+ "src": "model.layers.17.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.norm.weight",
+ "src": "model.layers.17.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.out_proj.weight",
+ "src": "model.layers.17.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.mlp.down_proj.weight",
+ "src": "model.layers.17.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.mlp.gate_proj.weight",
+ "src": "model.layers.17.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.mlp.up_proj.weight",
+ "src": "model.layers.17.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.post_attention_layernorm.weight",
+ "src": "model.layers.17.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.input_layernorm.weight",
+ "src": "model.layers.18.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.A_log",
+ "src": "model.layers.18.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.conv1d.weight",
+ "src": "model.layers.18.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.dt_bias",
+ "src": "model.layers.18.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.in_proj_a.weight",
+ "src": "model.layers.18.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.in_proj_b.weight",
+ "src": "model.layers.18.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.18.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.in_proj_z.weight",
+ "src": "model.layers.18.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.norm.weight",
+ "src": "model.layers.18.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.out_proj.weight",
+ "src": "model.layers.18.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.mlp.down_proj.weight",
+ "src": "model.layers.18.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.mlp.gate_proj.weight",
+ "src": "model.layers.18.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.mlp.up_proj.weight",
+ "src": "model.layers.18.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.post_attention_layernorm.weight",
+ "src": "model.layers.18.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.input_layernorm.weight",
+ "src": "model.layers.19.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.mlp.down_proj.weight",
+ "src": "model.layers.19.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.mlp.gate_proj.weight",
+ "src": "model.layers.19.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.mlp.up_proj.weight",
+ "src": "model.layers.19.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.post_attention_layernorm.weight",
+ "src": "model.layers.19.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.self_attn.k_norm.weight",
+ "src": "model.layers.19.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.self_attn.k_proj.weight",
+ "src": "model.layers.19.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.self_attn.o_proj.weight",
+ "src": "model.layers.19.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.self_attn.q_norm.weight",
+ "src": "model.layers.19.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.self_attn.q_proj.weight",
+ "src": "model.layers.19.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.self_attn.v_proj.weight",
+ "src": "model.layers.19.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.40.input_layernorm.weight",
+ "src": "model.layers.20.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.A_log",
+ "src": "model.layers.20.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.conv1d.weight",
+ "src": "model.layers.20.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.dt_bias",
+ "src": "model.layers.20.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.in_proj_a.weight",
+ "src": "model.layers.20.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.in_proj_b.weight",
+ "src": "model.layers.20.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.20.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.in_proj_z.weight",
+ "src": "model.layers.20.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.norm.weight",
+ "src": "model.layers.20.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.out_proj.weight",
+ "src": "model.layers.20.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.mlp.down_proj.weight",
+ "src": "model.layers.20.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.mlp.gate_proj.weight",
+ "src": "model.layers.20.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.mlp.up_proj.weight",
+ "src": "model.layers.20.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.post_attention_layernorm.weight",
+ "src": "model.layers.20.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.input_layernorm.weight",
+ "src": "model.layers.21.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.A_log",
+ "src": "model.layers.21.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.conv1d.weight",
+ "src": "model.layers.21.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.dt_bias",
+ "src": "model.layers.21.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.in_proj_a.weight",
+ "src": "model.layers.21.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.in_proj_b.weight",
+ "src": "model.layers.21.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.21.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.in_proj_z.weight",
+ "src": "model.layers.21.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.norm.weight",
+ "src": "model.layers.21.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.out_proj.weight",
+ "src": "model.layers.21.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.mlp.down_proj.weight",
+ "src": "model.layers.21.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.mlp.gate_proj.weight",
+ "src": "model.layers.21.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.mlp.up_proj.weight",
+ "src": "model.layers.21.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.post_attention_layernorm.weight",
+ "src": "model.layers.21.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.input_layernorm.weight",
+ "src": "model.layers.22.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.A_log",
+ "src": "model.layers.22.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.conv1d.weight",
+ "src": "model.layers.22.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.dt_bias",
+ "src": "model.layers.22.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.in_proj_a.weight",
+ "src": "model.layers.22.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.in_proj_b.weight",
+ "src": "model.layers.22.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.22.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.in_proj_z.weight",
+ "src": "model.layers.22.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.norm.weight",
+ "src": "model.layers.22.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.out_proj.weight",
+ "src": "model.layers.22.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.mlp.down_proj.weight",
+ "src": "model.layers.22.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.mlp.gate_proj.weight",
+ "src": "model.layers.22.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.mlp.up_proj.weight",
+ "src": "model.layers.22.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.post_attention_layernorm.weight",
+ "src": "model.layers.22.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.input_layernorm.weight",
+ "src": "model.layers.23.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.mlp.down_proj.weight",
+ "src": "model.layers.23.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.mlp.gate_proj.weight",
+ "src": "model.layers.23.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.mlp.up_proj.weight",
+ "src": "model.layers.23.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.post_attention_layernorm.weight",
+ "src": "model.layers.23.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.self_attn.k_norm.weight",
+ "src": "model.layers.23.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.self_attn.k_proj.weight",
+ "src": "model.layers.23.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.self_attn.o_proj.weight",
+ "src": "model.layers.23.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.self_attn.q_norm.weight",
+ "src": "model.layers.23.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.self_attn.q_proj.weight",
+ "src": "model.layers.23.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.self_attn.v_proj.weight",
+ "src": "model.layers.23.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.44.input_layernorm.weight",
+ "src": "model.layers.20.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.A_log",
+ "src": "model.layers.20.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.conv1d.weight",
+ "src": "model.layers.20.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.dt_bias",
+ "src": "model.layers.20.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.in_proj_a.weight",
+ "src": "model.layers.20.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.in_proj_b.weight",
+ "src": "model.layers.20.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.20.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.in_proj_z.weight",
+ "src": "model.layers.20.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.norm.weight",
+ "src": "model.layers.20.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.out_proj.weight",
+ "src": "model.layers.20.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.mlp.down_proj.weight",
+ "src": "model.layers.20.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.mlp.gate_proj.weight",
+ "src": "model.layers.20.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.mlp.up_proj.weight",
+ "src": "model.layers.20.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.post_attention_layernorm.weight",
+ "src": "model.layers.20.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.input_layernorm.weight",
+ "src": "model.layers.21.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.A_log",
+ "src": "model.layers.21.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.conv1d.weight",
+ "src": "model.layers.21.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.dt_bias",
+ "src": "model.layers.21.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.in_proj_a.weight",
+ "src": "model.layers.21.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.in_proj_b.weight",
+ "src": "model.layers.21.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.21.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.in_proj_z.weight",
+ "src": "model.layers.21.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.norm.weight",
+ "src": "model.layers.21.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.out_proj.weight",
+ "src": "model.layers.21.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.mlp.down_proj.weight",
+ "src": "model.layers.21.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.mlp.gate_proj.weight",
+ "src": "model.layers.21.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.mlp.up_proj.weight",
+ "src": "model.layers.21.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.post_attention_layernorm.weight",
+ "src": "model.layers.21.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.input_layernorm.weight",
+ "src": "model.layers.22.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.A_log",
+ "src": "model.layers.22.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.conv1d.weight",
+ "src": "model.layers.22.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.dt_bias",
+ "src": "model.layers.22.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.in_proj_a.weight",
+ "src": "model.layers.22.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.in_proj_b.weight",
+ "src": "model.layers.22.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.22.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.in_proj_z.weight",
+ "src": "model.layers.22.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.norm.weight",
+ "src": "model.layers.22.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.out_proj.weight",
+ "src": "model.layers.22.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.mlp.down_proj.weight",
+ "src": "model.layers.22.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.mlp.gate_proj.weight",
+ "src": "model.layers.22.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.mlp.up_proj.weight",
+ "src": "model.layers.22.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.post_attention_layernorm.weight",
+ "src": "model.layers.22.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.input_layernorm.weight",
+ "src": "model.layers.23.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.mlp.down_proj.weight",
+ "src": "model.layers.23.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.mlp.gate_proj.weight",
+ "src": "model.layers.23.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.mlp.up_proj.weight",
+ "src": "model.layers.23.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.post_attention_layernorm.weight",
+ "src": "model.layers.23.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.self_attn.k_norm.weight",
+ "src": "model.layers.23.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.self_attn.k_proj.weight",
+ "src": "model.layers.23.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.self_attn.o_proj.weight",
+ "src": "model.layers.23.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.self_attn.q_norm.weight",
+ "src": "model.layers.23.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.self_attn.q_proj.weight",
+ "src": "model.layers.23.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.self_attn.v_proj.weight",
+ "src": "model.layers.23.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.48.input_layernorm.weight",
+ "src": "model.layers.24.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.A_log",
+ "src": "model.layers.24.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.conv1d.weight",
+ "src": "model.layers.24.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.dt_bias",
+ "src": "model.layers.24.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.in_proj_a.weight",
+ "src": "model.layers.24.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.in_proj_b.weight",
+ "src": "model.layers.24.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.24.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.in_proj_z.weight",
+ "src": "model.layers.24.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.norm.weight",
+ "src": "model.layers.24.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.out_proj.weight",
+ "src": "model.layers.24.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.mlp.down_proj.weight",
+ "src": "model.layers.24.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.mlp.gate_proj.weight",
+ "src": "model.layers.24.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.mlp.up_proj.weight",
+ "src": "model.layers.24.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.post_attention_layernorm.weight",
+ "src": "model.layers.24.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.input_layernorm.weight",
+ "src": "model.layers.25.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.A_log",
+ "src": "model.layers.25.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.conv1d.weight",
+ "src": "model.layers.25.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.dt_bias",
+ "src": "model.layers.25.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.in_proj_a.weight",
+ "src": "model.layers.25.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.in_proj_b.weight",
+ "src": "model.layers.25.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.25.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.in_proj_z.weight",
+ "src": "model.layers.25.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.norm.weight",
+ "src": "model.layers.25.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.out_proj.weight",
+ "src": "model.layers.25.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.mlp.down_proj.weight",
+ "src": "model.layers.25.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.mlp.gate_proj.weight",
+ "src": "model.layers.25.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.mlp.up_proj.weight",
+ "src": "model.layers.25.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.post_attention_layernorm.weight",
+ "src": "model.layers.25.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.input_layernorm.weight",
+ "src": "model.layers.26.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.A_log",
+ "src": "model.layers.26.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.conv1d.weight",
+ "src": "model.layers.26.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.dt_bias",
+ "src": "model.layers.26.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.in_proj_a.weight",
+ "src": "model.layers.26.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.in_proj_b.weight",
+ "src": "model.layers.26.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.26.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.in_proj_z.weight",
+ "src": "model.layers.26.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.norm.weight",
+ "src": "model.layers.26.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.out_proj.weight",
+ "src": "model.layers.26.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.mlp.down_proj.weight",
+ "src": "model.layers.26.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.mlp.gate_proj.weight",
+ "src": "model.layers.26.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.mlp.up_proj.weight",
+ "src": "model.layers.26.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.post_attention_layernorm.weight",
+ "src": "model.layers.26.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.input_layernorm.weight",
+ "src": "model.layers.27.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.mlp.down_proj.weight",
+ "src": "model.layers.27.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.mlp.gate_proj.weight",
+ "src": "model.layers.27.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.mlp.up_proj.weight",
+ "src": "model.layers.27.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.post_attention_layernorm.weight",
+ "src": "model.layers.27.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.self_attn.k_norm.weight",
+ "src": "model.layers.27.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.self_attn.k_proj.weight",
+ "src": "model.layers.27.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.self_attn.o_proj.weight",
+ "src": "model.layers.27.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.self_attn.q_norm.weight",
+ "src": "model.layers.27.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.self_attn.q_proj.weight",
+ "src": "model.layers.27.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.self_attn.v_proj.weight",
+ "src": "model.layers.27.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.52.input_layernorm.weight",
+ "src": "model.layers.24.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.A_log",
+ "src": "model.layers.24.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.conv1d.weight",
+ "src": "model.layers.24.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.dt_bias",
+ "src": "model.layers.24.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.in_proj_a.weight",
+ "src": "model.layers.24.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.in_proj_b.weight",
+ "src": "model.layers.24.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.24.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.in_proj_z.weight",
+ "src": "model.layers.24.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.norm.weight",
+ "src": "model.layers.24.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.out_proj.weight",
+ "src": "model.layers.24.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.mlp.down_proj.weight",
+ "src": "model.layers.24.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.mlp.gate_proj.weight",
+ "src": "model.layers.24.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.mlp.up_proj.weight",
+ "src": "model.layers.24.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.post_attention_layernorm.weight",
+ "src": "model.layers.24.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.input_layernorm.weight",
+ "src": "model.layers.25.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.A_log",
+ "src": "model.layers.25.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.conv1d.weight",
+ "src": "model.layers.25.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.dt_bias",
+ "src": "model.layers.25.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.in_proj_a.weight",
+ "src": "model.layers.25.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.in_proj_b.weight",
+ "src": "model.layers.25.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.25.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.in_proj_z.weight",
+ "src": "model.layers.25.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.norm.weight",
+ "src": "model.layers.25.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.out_proj.weight",
+ "src": "model.layers.25.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.mlp.down_proj.weight",
+ "src": "model.layers.25.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.mlp.gate_proj.weight",
+ "src": "model.layers.25.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.mlp.up_proj.weight",
+ "src": "model.layers.25.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.post_attention_layernorm.weight",
+ "src": "model.layers.25.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.input_layernorm.weight",
+ "src": "model.layers.26.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.A_log",
+ "src": "model.layers.26.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.conv1d.weight",
+ "src": "model.layers.26.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.dt_bias",
+ "src": "model.layers.26.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.in_proj_a.weight",
+ "src": "model.layers.26.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.in_proj_b.weight",
+ "src": "model.layers.26.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.26.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.in_proj_z.weight",
+ "src": "model.layers.26.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.norm.weight",
+ "src": "model.layers.26.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.out_proj.weight",
+ "src": "model.layers.26.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.mlp.down_proj.weight",
+ "src": "model.layers.26.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.mlp.gate_proj.weight",
+ "src": "model.layers.26.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.mlp.up_proj.weight",
+ "src": "model.layers.26.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.post_attention_layernorm.weight",
+ "src": "model.layers.26.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.input_layernorm.weight",
+ "src": "model.layers.27.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.mlp.down_proj.weight",
+ "src": "model.layers.27.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.mlp.gate_proj.weight",
+ "src": "model.layers.27.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.mlp.up_proj.weight",
+ "src": "model.layers.27.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.post_attention_layernorm.weight",
+ "src": "model.layers.27.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.self_attn.k_norm.weight",
+ "src": "model.layers.27.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.self_attn.k_proj.weight",
+ "src": "model.layers.27.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.self_attn.o_proj.weight",
+ "src": "model.layers.27.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.self_attn.q_norm.weight",
+ "src": "model.layers.27.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.self_attn.q_proj.weight",
+ "src": "model.layers.27.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.self_attn.v_proj.weight",
+ "src": "model.layers.27.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.56.input_layernorm.weight",
+ "src": "model.layers.28.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.A_log",
+ "src": "model.layers.28.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.conv1d.weight",
+ "src": "model.layers.28.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.dt_bias",
+ "src": "model.layers.28.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.in_proj_a.weight",
+ "src": "model.layers.28.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.in_proj_b.weight",
+ "src": "model.layers.28.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.28.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.in_proj_z.weight",
+ "src": "model.layers.28.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.norm.weight",
+ "src": "model.layers.28.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.out_proj.weight",
+ "src": "model.layers.28.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.mlp.down_proj.weight",
+ "src": "model.layers.28.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.mlp.gate_proj.weight",
+ "src": "model.layers.28.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.mlp.up_proj.weight",
+ "src": "model.layers.28.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.post_attention_layernorm.weight",
+ "src": "model.layers.28.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.input_layernorm.weight",
+ "src": "model.layers.29.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.A_log",
+ "src": "model.layers.29.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.conv1d.weight",
+ "src": "model.layers.29.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.dt_bias",
+ "src": "model.layers.29.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.in_proj_a.weight",
+ "src": "model.layers.29.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.in_proj_b.weight",
+ "src": "model.layers.29.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.29.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.in_proj_z.weight",
+ "src": "model.layers.29.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.norm.weight",
+ "src": "model.layers.29.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.out_proj.weight",
+ "src": "model.layers.29.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.mlp.down_proj.weight",
+ "src": "model.layers.29.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.mlp.gate_proj.weight",
+ "src": "model.layers.29.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.mlp.up_proj.weight",
+ "src": "model.layers.29.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.post_attention_layernorm.weight",
+ "src": "model.layers.29.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.input_layernorm.weight",
+ "src": "model.layers.30.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.A_log",
+ "src": "model.layers.30.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.conv1d.weight",
+ "src": "model.layers.30.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.dt_bias",
+ "src": "model.layers.30.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.in_proj_a.weight",
+ "src": "model.layers.30.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.in_proj_b.weight",
+ "src": "model.layers.30.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.30.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.in_proj_z.weight",
+ "src": "model.layers.30.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.norm.weight",
+ "src": "model.layers.30.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.out_proj.weight",
+ "src": "model.layers.30.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.mlp.down_proj.weight",
+ "src": "model.layers.30.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.mlp.gate_proj.weight",
+ "src": "model.layers.30.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.mlp.up_proj.weight",
+ "src": "model.layers.30.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.post_attention_layernorm.weight",
+ "src": "model.layers.30.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.input_layernorm.weight",
+ "src": "model.layers.31.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.mlp.down_proj.weight",
+ "src": "model.layers.31.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.mlp.gate_proj.weight",
+ "src": "model.layers.31.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.mlp.up_proj.weight",
+ "src": "model.layers.31.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.post_attention_layernorm.weight",
+ "src": "model.layers.31.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.self_attn.k_norm.weight",
+ "src": "model.layers.31.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.self_attn.k_proj.weight",
+ "src": "model.layers.31.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.self_attn.o_proj.weight",
+ "src": "model.layers.31.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.self_attn.q_norm.weight",
+ "src": "model.layers.31.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.self_attn.q_proj.weight",
+ "src": "model.layers.31.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.self_attn.v_proj.weight",
+ "src": "model.layers.31.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.60.input_layernorm.weight",
+ "src": "model.layers.28.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.A_log",
+ "src": "model.layers.28.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.conv1d.weight",
+ "src": "model.layers.28.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.dt_bias",
+ "src": "model.layers.28.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.in_proj_a.weight",
+ "src": "model.layers.28.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.in_proj_b.weight",
+ "src": "model.layers.28.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.28.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.in_proj_z.weight",
+ "src": "model.layers.28.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.norm.weight",
+ "src": "model.layers.28.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.out_proj.weight",
+ "src": "model.layers.28.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.mlp.down_proj.weight",
+ "src": "model.layers.28.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.mlp.gate_proj.weight",
+ "src": "model.layers.28.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.mlp.up_proj.weight",
+ "src": "model.layers.28.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.post_attention_layernorm.weight",
+ "src": "model.layers.28.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.input_layernorm.weight",
+ "src": "model.layers.29.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.A_log",
+ "src": "model.layers.29.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.conv1d.weight",
+ "src": "model.layers.29.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.dt_bias",
+ "src": "model.layers.29.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.in_proj_a.weight",
+ "src": "model.layers.29.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.in_proj_b.weight",
+ "src": "model.layers.29.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.29.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.in_proj_z.weight",
+ "src": "model.layers.29.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.norm.weight",
+ "src": "model.layers.29.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.out_proj.weight",
+ "src": "model.layers.29.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.mlp.down_proj.weight",
+ "src": "model.layers.29.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.mlp.gate_proj.weight",
+ "src": "model.layers.29.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.mlp.up_proj.weight",
+ "src": "model.layers.29.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.post_attention_layernorm.weight",
+ "src": "model.layers.29.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.input_layernorm.weight",
+ "src": "model.layers.30.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.A_log",
+ "src": "model.layers.30.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.conv1d.weight",
+ "src": "model.layers.30.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.dt_bias",
+ "src": "model.layers.30.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.in_proj_a.weight",
+ "src": "model.layers.30.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.in_proj_b.weight",
+ "src": "model.layers.30.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.30.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.in_proj_z.weight",
+ "src": "model.layers.30.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.norm.weight",
+ "src": "model.layers.30.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.out_proj.weight",
+ "src": "model.layers.30.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.mlp.down_proj.weight",
+ "src": "model.layers.30.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.mlp.gate_proj.weight",
+ "src": "model.layers.30.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.mlp.up_proj.weight",
+ "src": "model.layers.30.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.post_attention_layernorm.weight",
+ "src": "model.layers.30.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.input_layernorm.weight",
+ "src": "model.layers.31.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.mlp.down_proj.weight",
+ "src": "model.layers.31.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.mlp.gate_proj.weight",
+ "src": "model.layers.31.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.mlp.up_proj.weight",
+ "src": "model.layers.31.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.post_attention_layernorm.weight",
+ "src": "model.layers.31.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.self_attn.k_norm.weight",
+ "src": "model.layers.31.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.self_attn.k_proj.weight",
+ "src": "model.layers.31.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.self_attn.o_proj.weight",
+ "src": "model.layers.31.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.self_attn.q_norm.weight",
+ "src": "model.layers.31.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.self_attn.q_proj.weight",
+ "src": "model.layers.31.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.self_attn.v_proj.weight",
+ "src": "model.layers.31.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.64.input_layernorm.weight",
+ "src": "model.layers.32.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.A_log",
+ "src": "model.layers.32.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.conv1d.weight",
+ "src": "model.layers.32.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.dt_bias",
+ "src": "model.layers.32.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.in_proj_a.weight",
+ "src": "model.layers.32.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.in_proj_b.weight",
+ "src": "model.layers.32.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.32.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.in_proj_z.weight",
+ "src": "model.layers.32.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.norm.weight",
+ "src": "model.layers.32.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.out_proj.weight",
+ "src": "model.layers.32.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.mlp.down_proj.weight",
+ "src": "model.layers.32.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.mlp.gate_proj.weight",
+ "src": "model.layers.32.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.mlp.up_proj.weight",
+ "src": "model.layers.32.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.post_attention_layernorm.weight",
+ "src": "model.layers.32.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.input_layernorm.weight",
+ "src": "model.layers.33.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.A_log",
+ "src": "model.layers.33.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.conv1d.weight",
+ "src": "model.layers.33.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.dt_bias",
+ "src": "model.layers.33.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.in_proj_a.weight",
+ "src": "model.layers.33.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.in_proj_b.weight",
+ "src": "model.layers.33.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.33.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.in_proj_z.weight",
+ "src": "model.layers.33.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.norm.weight",
+ "src": "model.layers.33.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.out_proj.weight",
+ "src": "model.layers.33.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.mlp.down_proj.weight",
+ "src": "model.layers.33.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.mlp.gate_proj.weight",
+ "src": "model.layers.33.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.mlp.up_proj.weight",
+ "src": "model.layers.33.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.post_attention_layernorm.weight",
+ "src": "model.layers.33.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.input_layernorm.weight",
+ "src": "model.layers.34.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.A_log",
+ "src": "model.layers.34.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.conv1d.weight",
+ "src": "model.layers.34.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.dt_bias",
+ "src": "model.layers.34.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.in_proj_a.weight",
+ "src": "model.layers.34.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.in_proj_b.weight",
+ "src": "model.layers.34.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.34.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.in_proj_z.weight",
+ "src": "model.layers.34.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.norm.weight",
+ "src": "model.layers.34.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.out_proj.weight",
+ "src": "model.layers.34.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.mlp.down_proj.weight",
+ "src": "model.layers.34.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.mlp.gate_proj.weight",
+ "src": "model.layers.34.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.mlp.up_proj.weight",
+ "src": "model.layers.34.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.post_attention_layernorm.weight",
+ "src": "model.layers.34.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.input_layernorm.weight",
+ "src": "model.layers.35.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.mlp.down_proj.weight",
+ "src": "model.layers.35.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.mlp.gate_proj.weight",
+ "src": "model.layers.35.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.mlp.up_proj.weight",
+ "src": "model.layers.35.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.post_attention_layernorm.weight",
+ "src": "model.layers.35.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.self_attn.k_norm.weight",
+ "src": "model.layers.35.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.self_attn.k_proj.weight",
+ "src": "model.layers.35.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.self_attn.o_proj.weight",
+ "src": "model.layers.35.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.self_attn.q_norm.weight",
+ "src": "model.layers.35.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.self_attn.q_proj.weight",
+ "src": "model.layers.35.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.self_attn.v_proj.weight",
+ "src": "model.layers.35.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.68.input_layernorm.weight",
+ "src": "model.layers.32.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.A_log",
+ "src": "model.layers.32.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.conv1d.weight",
+ "src": "model.layers.32.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.dt_bias",
+ "src": "model.layers.32.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.in_proj_a.weight",
+ "src": "model.layers.32.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.in_proj_b.weight",
+ "src": "model.layers.32.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.32.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.in_proj_z.weight",
+ "src": "model.layers.32.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.norm.weight",
+ "src": "model.layers.32.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.out_proj.weight",
+ "src": "model.layers.32.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.mlp.down_proj.weight",
+ "src": "model.layers.32.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.mlp.gate_proj.weight",
+ "src": "model.layers.32.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.mlp.up_proj.weight",
+ "src": "model.layers.32.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.post_attention_layernorm.weight",
+ "src": "model.layers.32.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.input_layernorm.weight",
+ "src": "model.layers.33.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.A_log",
+ "src": "model.layers.33.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.conv1d.weight",
+ "src": "model.layers.33.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.dt_bias",
+ "src": "model.layers.33.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.in_proj_a.weight",
+ "src": "model.layers.33.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.in_proj_b.weight",
+ "src": "model.layers.33.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.33.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.in_proj_z.weight",
+ "src": "model.layers.33.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.norm.weight",
+ "src": "model.layers.33.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.out_proj.weight",
+ "src": "model.layers.33.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.mlp.down_proj.weight",
+ "src": "model.layers.33.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.mlp.gate_proj.weight",
+ "src": "model.layers.33.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.mlp.up_proj.weight",
+ "src": "model.layers.33.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.post_attention_layernorm.weight",
+ "src": "model.layers.33.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.input_layernorm.weight",
+ "src": "model.layers.34.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.A_log",
+ "src": "model.layers.34.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.conv1d.weight",
+ "src": "model.layers.34.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.dt_bias",
+ "src": "model.layers.34.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.in_proj_a.weight",
+ "src": "model.layers.34.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.in_proj_b.weight",
+ "src": "model.layers.34.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.34.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.in_proj_z.weight",
+ "src": "model.layers.34.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.norm.weight",
+ "src": "model.layers.34.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.out_proj.weight",
+ "src": "model.layers.34.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.mlp.down_proj.weight",
+ "src": "model.layers.34.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.mlp.gate_proj.weight",
+ "src": "model.layers.34.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.mlp.up_proj.weight",
+ "src": "model.layers.34.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.post_attention_layernorm.weight",
+ "src": "model.layers.34.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.input_layernorm.weight",
+ "src": "model.layers.35.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.mlp.down_proj.weight",
+ "src": "model.layers.35.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.mlp.gate_proj.weight",
+ "src": "model.layers.35.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.mlp.up_proj.weight",
+ "src": "model.layers.35.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.post_attention_layernorm.weight",
+ "src": "model.layers.35.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.self_attn.k_norm.weight",
+ "src": "model.layers.35.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.self_attn.k_proj.weight",
+ "src": "model.layers.35.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.self_attn.o_proj.weight",
+ "src": "model.layers.35.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.self_attn.q_norm.weight",
+ "src": "model.layers.35.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.self_attn.q_proj.weight",
+ "src": "model.layers.35.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.self_attn.v_proj.weight",
+ "src": "model.layers.35.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.72.input_layernorm.weight",
+ "src": "model.layers.36.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.A_log",
+ "src": "model.layers.36.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.conv1d.weight",
+ "src": "model.layers.36.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.dt_bias",
+ "src": "model.layers.36.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.in_proj_a.weight",
+ "src": "model.layers.36.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.in_proj_b.weight",
+ "src": "model.layers.36.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.36.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.in_proj_z.weight",
+ "src": "model.layers.36.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.norm.weight",
+ "src": "model.layers.36.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.out_proj.weight",
+ "src": "model.layers.36.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.mlp.down_proj.weight",
+ "src": "model.layers.36.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.mlp.gate_proj.weight",
+ "src": "model.layers.36.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.mlp.up_proj.weight",
+ "src": "model.layers.36.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.post_attention_layernorm.weight",
+ "src": "model.layers.36.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.input_layernorm.weight",
+ "src": "model.layers.37.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.A_log",
+ "src": "model.layers.37.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.conv1d.weight",
+ "src": "model.layers.37.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.dt_bias",
+ "src": "model.layers.37.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.in_proj_a.weight",
+ "src": "model.layers.37.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.in_proj_b.weight",
+ "src": "model.layers.37.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.37.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.in_proj_z.weight",
+ "src": "model.layers.37.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.norm.weight",
+ "src": "model.layers.37.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.out_proj.weight",
+ "src": "model.layers.37.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.mlp.down_proj.weight",
+ "src": "model.layers.37.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.mlp.gate_proj.weight",
+ "src": "model.layers.37.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.mlp.up_proj.weight",
+ "src": "model.layers.37.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.post_attention_layernorm.weight",
+ "src": "model.layers.37.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.input_layernorm.weight",
+ "src": "model.layers.38.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.A_log",
+ "src": "model.layers.38.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.conv1d.weight",
+ "src": "model.layers.38.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.dt_bias",
+ "src": "model.layers.38.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.in_proj_a.weight",
+ "src": "model.layers.38.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.in_proj_b.weight",
+ "src": "model.layers.38.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.38.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.in_proj_z.weight",
+ "src": "model.layers.38.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.norm.weight",
+ "src": "model.layers.38.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.out_proj.weight",
+ "src": "model.layers.38.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.mlp.down_proj.weight",
+ "src": "model.layers.38.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.mlp.gate_proj.weight",
+ "src": "model.layers.38.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.mlp.up_proj.weight",
+ "src": "model.layers.38.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.post_attention_layernorm.weight",
+ "src": "model.layers.38.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.input_layernorm.weight",
+ "src": "model.layers.39.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.mlp.down_proj.weight",
+ "src": "model.layers.39.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.mlp.gate_proj.weight",
+ "src": "model.layers.39.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.mlp.up_proj.weight",
+ "src": "model.layers.39.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.post_attention_layernorm.weight",
+ "src": "model.layers.39.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.self_attn.k_norm.weight",
+ "src": "model.layers.39.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.self_attn.k_proj.weight",
+ "src": "model.layers.39.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.self_attn.o_proj.weight",
+ "src": "model.layers.39.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.self_attn.q_norm.weight",
+ "src": "model.layers.39.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.self_attn.q_proj.weight",
+ "src": "model.layers.39.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.self_attn.v_proj.weight",
+ "src": "model.layers.39.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.76.input_layernorm.weight",
+ "src": "model.layers.36.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.A_log",
+ "src": "model.layers.36.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.conv1d.weight",
+ "src": "model.layers.36.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.dt_bias",
+ "src": "model.layers.36.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.in_proj_a.weight",
+ "src": "model.layers.36.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.in_proj_b.weight",
+ "src": "model.layers.36.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.36.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.in_proj_z.weight",
+ "src": "model.layers.36.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.norm.weight",
+ "src": "model.layers.36.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.out_proj.weight",
+ "src": "model.layers.36.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.mlp.down_proj.weight",
+ "src": "model.layers.36.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.mlp.gate_proj.weight",
+ "src": "model.layers.36.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.mlp.up_proj.weight",
+ "src": "model.layers.36.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.post_attention_layernorm.weight",
+ "src": "model.layers.36.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.input_layernorm.weight",
+ "src": "model.layers.37.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.A_log",
+ "src": "model.layers.37.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.conv1d.weight",
+ "src": "model.layers.37.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.dt_bias",
+ "src": "model.layers.37.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.in_proj_a.weight",
+ "src": "model.layers.37.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.in_proj_b.weight",
+ "src": "model.layers.37.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.37.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.in_proj_z.weight",
+ "src": "model.layers.37.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.norm.weight",
+ "src": "model.layers.37.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.out_proj.weight",
+ "src": "model.layers.37.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.mlp.down_proj.weight",
+ "src": "model.layers.37.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.mlp.gate_proj.weight",
+ "src": "model.layers.37.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.mlp.up_proj.weight",
+ "src": "model.layers.37.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.post_attention_layernorm.weight",
+ "src": "model.layers.37.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.input_layernorm.weight",
+ "src": "model.layers.38.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.A_log",
+ "src": "model.layers.38.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.conv1d.weight",
+ "src": "model.layers.38.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.dt_bias",
+ "src": "model.layers.38.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.in_proj_a.weight",
+ "src": "model.layers.38.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.in_proj_b.weight",
+ "src": "model.layers.38.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.38.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.in_proj_z.weight",
+ "src": "model.layers.38.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.norm.weight",
+ "src": "model.layers.38.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.out_proj.weight",
+ "src": "model.layers.38.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.mlp.down_proj.weight",
+ "src": "model.layers.38.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.mlp.gate_proj.weight",
+ "src": "model.layers.38.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.mlp.up_proj.weight",
+ "src": "model.layers.38.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.post_attention_layernorm.weight",
+ "src": "model.layers.38.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.input_layernorm.weight",
+ "src": "model.layers.39.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.mlp.down_proj.weight",
+ "src": "model.layers.39.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.mlp.gate_proj.weight",
+ "src": "model.layers.39.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.mlp.up_proj.weight",
+ "src": "model.layers.39.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.post_attention_layernorm.weight",
+ "src": "model.layers.39.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.self_attn.k_norm.weight",
+ "src": "model.layers.39.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.self_attn.k_proj.weight",
+ "src": "model.layers.39.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.self_attn.o_proj.weight",
+ "src": "model.layers.39.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.self_attn.q_norm.weight",
+ "src": "model.layers.39.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.self_attn.q_proj.weight",
+ "src": "model.layers.39.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.self_attn.v_proj.weight",
+ "src": "model.layers.39.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.80.input_layernorm.weight",
+ "src": "model.layers.40.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.A_log",
+ "src": "model.layers.40.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.conv1d.weight",
+ "src": "model.layers.40.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.dt_bias",
+ "src": "model.layers.40.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.in_proj_a.weight",
+ "src": "model.layers.40.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.in_proj_b.weight",
+ "src": "model.layers.40.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.40.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.in_proj_z.weight",
+ "src": "model.layers.40.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.norm.weight",
+ "src": "model.layers.40.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.out_proj.weight",
+ "src": "model.layers.40.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.mlp.down_proj.weight",
+ "src": "model.layers.40.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.mlp.gate_proj.weight",
+ "src": "model.layers.40.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.mlp.up_proj.weight",
+ "src": "model.layers.40.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.post_attention_layernorm.weight",
+ "src": "model.layers.40.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.input_layernorm.weight",
+ "src": "model.layers.41.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.A_log",
+ "src": "model.layers.41.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.conv1d.weight",
+ "src": "model.layers.41.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.dt_bias",
+ "src": "model.layers.41.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.in_proj_a.weight",
+ "src": "model.layers.41.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.in_proj_b.weight",
+ "src": "model.layers.41.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.41.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.in_proj_z.weight",
+ "src": "model.layers.41.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.norm.weight",
+ "src": "model.layers.41.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.out_proj.weight",
+ "src": "model.layers.41.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.mlp.down_proj.weight",
+ "src": "model.layers.41.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.mlp.gate_proj.weight",
+ "src": "model.layers.41.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.mlp.up_proj.weight",
+ "src": "model.layers.41.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.post_attention_layernorm.weight",
+ "src": "model.layers.41.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.input_layernorm.weight",
+ "src": "model.layers.42.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.A_log",
+ "src": "model.layers.42.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.conv1d.weight",
+ "src": "model.layers.42.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.dt_bias",
+ "src": "model.layers.42.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.in_proj_a.weight",
+ "src": "model.layers.42.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.in_proj_b.weight",
+ "src": "model.layers.42.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.42.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.in_proj_z.weight",
+ "src": "model.layers.42.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.norm.weight",
+ "src": "model.layers.42.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.out_proj.weight",
+ "src": "model.layers.42.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.mlp.down_proj.weight",
+ "src": "model.layers.42.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.mlp.gate_proj.weight",
+ "src": "model.layers.42.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.mlp.up_proj.weight",
+ "src": "model.layers.42.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.post_attention_layernorm.weight",
+ "src": "model.layers.42.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.input_layernorm.weight",
+ "src": "model.layers.43.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.mlp.down_proj.weight",
+ "src": "model.layers.43.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.mlp.gate_proj.weight",
+ "src": "model.layers.43.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.mlp.up_proj.weight",
+ "src": "model.layers.43.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.post_attention_layernorm.weight",
+ "src": "model.layers.43.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.self_attn.k_norm.weight",
+ "src": "model.layers.43.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.self_attn.k_proj.weight",
+ "src": "model.layers.43.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.self_attn.o_proj.weight",
+ "src": "model.layers.43.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.self_attn.q_norm.weight",
+ "src": "model.layers.43.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.self_attn.q_proj.weight",
+ "src": "model.layers.43.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.self_attn.v_proj.weight",
+ "src": "model.layers.43.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.84.input_layernorm.weight",
+ "src": "model.layers.40.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.A_log",
+ "src": "model.layers.40.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.conv1d.weight",
+ "src": "model.layers.40.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.dt_bias",
+ "src": "model.layers.40.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.in_proj_a.weight",
+ "src": "model.layers.40.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.in_proj_b.weight",
+ "src": "model.layers.40.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.40.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.in_proj_z.weight",
+ "src": "model.layers.40.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.norm.weight",
+ "src": "model.layers.40.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.out_proj.weight",
+ "src": "model.layers.40.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.mlp.down_proj.weight",
+ "src": "model.layers.40.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.mlp.gate_proj.weight",
+ "src": "model.layers.40.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.mlp.up_proj.weight",
+ "src": "model.layers.40.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.post_attention_layernorm.weight",
+ "src": "model.layers.40.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.input_layernorm.weight",
+ "src": "model.layers.41.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.A_log",
+ "src": "model.layers.41.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.conv1d.weight",
+ "src": "model.layers.41.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.dt_bias",
+ "src": "model.layers.41.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.in_proj_a.weight",
+ "src": "model.layers.41.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.in_proj_b.weight",
+ "src": "model.layers.41.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.41.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.in_proj_z.weight",
+ "src": "model.layers.41.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.norm.weight",
+ "src": "model.layers.41.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.out_proj.weight",
+ "src": "model.layers.41.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.mlp.down_proj.weight",
+ "src": "model.layers.41.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.mlp.gate_proj.weight",
+ "src": "model.layers.41.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.mlp.up_proj.weight",
+ "src": "model.layers.41.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.post_attention_layernorm.weight",
+ "src": "model.layers.41.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.input_layernorm.weight",
+ "src": "model.layers.42.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.A_log",
+ "src": "model.layers.42.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.conv1d.weight",
+ "src": "model.layers.42.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.dt_bias",
+ "src": "model.layers.42.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.in_proj_a.weight",
+ "src": "model.layers.42.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.in_proj_b.weight",
+ "src": "model.layers.42.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.42.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.in_proj_z.weight",
+ "src": "model.layers.42.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.norm.weight",
+ "src": "model.layers.42.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.out_proj.weight",
+ "src": "model.layers.42.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.mlp.down_proj.weight",
+ "src": "model.layers.42.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.mlp.gate_proj.weight",
+ "src": "model.layers.42.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.mlp.up_proj.weight",
+ "src": "model.layers.42.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.post_attention_layernorm.weight",
+ "src": "model.layers.42.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.input_layernorm.weight",
+ "src": "model.layers.43.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.mlp.down_proj.weight",
+ "src": "model.layers.43.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.mlp.gate_proj.weight",
+ "src": "model.layers.43.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.mlp.up_proj.weight",
+ "src": "model.layers.43.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.post_attention_layernorm.weight",
+ "src": "model.layers.43.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.self_attn.k_norm.weight",
+ "src": "model.layers.43.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.self_attn.k_proj.weight",
+ "src": "model.layers.43.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.self_attn.o_proj.weight",
+ "src": "model.layers.43.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.self_attn.q_norm.weight",
+ "src": "model.layers.43.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.self_attn.q_proj.weight",
+ "src": "model.layers.43.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.self_attn.v_proj.weight",
+ "src": "model.layers.43.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.88.input_layernorm.weight",
+ "src": "model.layers.44.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.A_log",
+ "src": "model.layers.44.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.conv1d.weight",
+ "src": "model.layers.44.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.dt_bias",
+ "src": "model.layers.44.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.in_proj_a.weight",
+ "src": "model.layers.44.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.in_proj_b.weight",
+ "src": "model.layers.44.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.44.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.in_proj_z.weight",
+ "src": "model.layers.44.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.norm.weight",
+ "src": "model.layers.44.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.out_proj.weight",
+ "src": "model.layers.44.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.mlp.down_proj.weight",
+ "src": "model.layers.44.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.mlp.gate_proj.weight",
+ "src": "model.layers.44.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.mlp.up_proj.weight",
+ "src": "model.layers.44.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.post_attention_layernorm.weight",
+ "src": "model.layers.44.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.input_layernorm.weight",
+ "src": "model.layers.45.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.A_log",
+ "src": "model.layers.45.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.conv1d.weight",
+ "src": "model.layers.45.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.dt_bias",
+ "src": "model.layers.45.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.in_proj_a.weight",
+ "src": "model.layers.45.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.in_proj_b.weight",
+ "src": "model.layers.45.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.45.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.in_proj_z.weight",
+ "src": "model.layers.45.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.norm.weight",
+ "src": "model.layers.45.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.out_proj.weight",
+ "src": "model.layers.45.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.mlp.down_proj.weight",
+ "src": "model.layers.45.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.mlp.gate_proj.weight",
+ "src": "model.layers.45.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.mlp.up_proj.weight",
+ "src": "model.layers.45.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.post_attention_layernorm.weight",
+ "src": "model.layers.45.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.input_layernorm.weight",
+ "src": "model.layers.46.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.A_log",
+ "src": "model.layers.46.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.conv1d.weight",
+ "src": "model.layers.46.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.dt_bias",
+ "src": "model.layers.46.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.in_proj_a.weight",
+ "src": "model.layers.46.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.in_proj_b.weight",
+ "src": "model.layers.46.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.46.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.in_proj_z.weight",
+ "src": "model.layers.46.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.norm.weight",
+ "src": "model.layers.46.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.out_proj.weight",
+ "src": "model.layers.46.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.mlp.down_proj.weight",
+ "src": "model.layers.46.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.mlp.gate_proj.weight",
+ "src": "model.layers.46.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.mlp.up_proj.weight",
+ "src": "model.layers.46.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.post_attention_layernorm.weight",
+ "src": "model.layers.46.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.input_layernorm.weight",
+ "src": "model.layers.47.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.mlp.down_proj.weight",
+ "src": "model.layers.47.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.mlp.gate_proj.weight",
+ "src": "model.layers.47.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.mlp.up_proj.weight",
+ "src": "model.layers.47.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.post_attention_layernorm.weight",
+ "src": "model.layers.47.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.self_attn.k_norm.weight",
+ "src": "model.layers.47.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.self_attn.k_proj.weight",
+ "src": "model.layers.47.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.self_attn.o_proj.weight",
+ "src": "model.layers.47.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.self_attn.q_norm.weight",
+ "src": "model.layers.47.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.self_attn.q_proj.weight",
+ "src": "model.layers.47.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.self_attn.v_proj.weight",
+ "src": "model.layers.47.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.92.input_layernorm.weight",
+ "src": "model.layers.44.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.A_log",
+ "src": "model.layers.44.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.conv1d.weight",
+ "src": "model.layers.44.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.dt_bias",
+ "src": "model.layers.44.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.in_proj_a.weight",
+ "src": "model.layers.44.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.in_proj_b.weight",
+ "src": "model.layers.44.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.44.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.in_proj_z.weight",
+ "src": "model.layers.44.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.norm.weight",
+ "src": "model.layers.44.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.out_proj.weight",
+ "src": "model.layers.44.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.mlp.down_proj.weight",
+ "src": "model.layers.44.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.mlp.gate_proj.weight",
+ "src": "model.layers.44.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.mlp.up_proj.weight",
+ "src": "model.layers.44.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.post_attention_layernorm.weight",
+ "src": "model.layers.44.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.input_layernorm.weight",
+ "src": "model.layers.45.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.A_log",
+ "src": "model.layers.45.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.conv1d.weight",
+ "src": "model.layers.45.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.dt_bias",
+ "src": "model.layers.45.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.in_proj_a.weight",
+ "src": "model.layers.45.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.in_proj_b.weight",
+ "src": "model.layers.45.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.45.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.in_proj_z.weight",
+ "src": "model.layers.45.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.norm.weight",
+ "src": "model.layers.45.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.out_proj.weight",
+ "src": "model.layers.45.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.mlp.down_proj.weight",
+ "src": "model.layers.45.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.mlp.gate_proj.weight",
+ "src": "model.layers.45.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.mlp.up_proj.weight",
+ "src": "model.layers.45.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.post_attention_layernorm.weight",
+ "src": "model.layers.45.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.input_layernorm.weight",
+ "src": "model.layers.46.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.A_log",
+ "src": "model.layers.46.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.conv1d.weight",
+ "src": "model.layers.46.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.dt_bias",
+ "src": "model.layers.46.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.in_proj_a.weight",
+ "src": "model.layers.46.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.in_proj_b.weight",
+ "src": "model.layers.46.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.46.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.in_proj_z.weight",
+ "src": "model.layers.46.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.norm.weight",
+ "src": "model.layers.46.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.out_proj.weight",
+ "src": "model.layers.46.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.mlp.down_proj.weight",
+ "src": "model.layers.46.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.mlp.gate_proj.weight",
+ "src": "model.layers.46.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.mlp.up_proj.weight",
+ "src": "model.layers.46.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.post_attention_layernorm.weight",
+ "src": "model.layers.46.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.input_layernorm.weight",
+ "src": "model.layers.47.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.mlp.down_proj.weight",
+ "src": "model.layers.47.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.mlp.gate_proj.weight",
+ "src": "model.layers.47.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.mlp.up_proj.weight",
+ "src": "model.layers.47.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.post_attention_layernorm.weight",
+ "src": "model.layers.47.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.self_attn.k_norm.weight",
+ "src": "model.layers.47.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.self_attn.k_proj.weight",
+ "src": "model.layers.47.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.self_attn.o_proj.weight",
+ "src": "model.layers.47.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.self_attn.q_norm.weight",
+ "src": "model.layers.47.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.self_attn.q_proj.weight",
+ "src": "model.layers.47.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.self_attn.v_proj.weight",
+ "src": "model.layers.47.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.96.input_layernorm.weight",
+ "src": "model.layers.48.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.A_log",
+ "src": "model.layers.48.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.conv1d.weight",
+ "src": "model.layers.48.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.dt_bias",
+ "src": "model.layers.48.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.in_proj_a.weight",
+ "src": "model.layers.48.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.in_proj_b.weight",
+ "src": "model.layers.48.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.48.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.in_proj_z.weight",
+ "src": "model.layers.48.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.norm.weight",
+ "src": "model.layers.48.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.out_proj.weight",
+ "src": "model.layers.48.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.mlp.down_proj.weight",
+ "src": "model.layers.48.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.mlp.gate_proj.weight",
+ "src": "model.layers.48.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.mlp.up_proj.weight",
+ "src": "model.layers.48.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.post_attention_layernorm.weight",
+ "src": "model.layers.48.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.input_layernorm.weight",
+ "src": "model.layers.49.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.A_log",
+ "src": "model.layers.49.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.conv1d.weight",
+ "src": "model.layers.49.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.dt_bias",
+ "src": "model.layers.49.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.in_proj_a.weight",
+ "src": "model.layers.49.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.in_proj_b.weight",
+ "src": "model.layers.49.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.49.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.in_proj_z.weight",
+ "src": "model.layers.49.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.norm.weight",
+ "src": "model.layers.49.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.out_proj.weight",
+ "src": "model.layers.49.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.mlp.down_proj.weight",
+ "src": "model.layers.49.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.mlp.gate_proj.weight",
+ "src": "model.layers.49.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.mlp.up_proj.weight",
+ "src": "model.layers.49.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.post_attention_layernorm.weight",
+ "src": "model.layers.49.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.input_layernorm.weight",
+ "src": "model.layers.50.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.A_log",
+ "src": "model.layers.50.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.conv1d.weight",
+ "src": "model.layers.50.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.dt_bias",
+ "src": "model.layers.50.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.in_proj_a.weight",
+ "src": "model.layers.50.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.in_proj_b.weight",
+ "src": "model.layers.50.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.50.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.in_proj_z.weight",
+ "src": "model.layers.50.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.norm.weight",
+ "src": "model.layers.50.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.out_proj.weight",
+ "src": "model.layers.50.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.mlp.down_proj.weight",
+ "src": "model.layers.50.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.mlp.gate_proj.weight",
+ "src": "model.layers.50.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.mlp.up_proj.weight",
+ "src": "model.layers.50.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.post_attention_layernorm.weight",
+ "src": "model.layers.50.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.input_layernorm.weight",
+ "src": "model.layers.51.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.mlp.down_proj.weight",
+ "src": "model.layers.51.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.mlp.gate_proj.weight",
+ "src": "model.layers.51.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.mlp.up_proj.weight",
+ "src": "model.layers.51.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.post_attention_layernorm.weight",
+ "src": "model.layers.51.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.self_attn.k_norm.weight",
+ "src": "model.layers.51.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.self_attn.k_proj.weight",
+ "src": "model.layers.51.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.self_attn.o_proj.weight",
+ "src": "model.layers.51.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.self_attn.q_norm.weight",
+ "src": "model.layers.51.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.self_attn.q_proj.weight",
+ "src": "model.layers.51.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.self_attn.v_proj.weight",
+ "src": "model.layers.51.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.100.input_layernorm.weight",
+ "src": "model.layers.48.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.A_log",
+ "src": "model.layers.48.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.conv1d.weight",
+ "src": "model.layers.48.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.dt_bias",
+ "src": "model.layers.48.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.in_proj_a.weight",
+ "src": "model.layers.48.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.in_proj_b.weight",
+ "src": "model.layers.48.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.48.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.in_proj_z.weight",
+ "src": "model.layers.48.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.norm.weight",
+ "src": "model.layers.48.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.out_proj.weight",
+ "src": "model.layers.48.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.mlp.down_proj.weight",
+ "src": "model.layers.48.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.mlp.gate_proj.weight",
+ "src": "model.layers.48.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.mlp.up_proj.weight",
+ "src": "model.layers.48.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.post_attention_layernorm.weight",
+ "src": "model.layers.48.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.input_layernorm.weight",
+ "src": "model.layers.49.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.A_log",
+ "src": "model.layers.49.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.conv1d.weight",
+ "src": "model.layers.49.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.dt_bias",
+ "src": "model.layers.49.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.in_proj_a.weight",
+ "src": "model.layers.49.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.in_proj_b.weight",
+ "src": "model.layers.49.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.49.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.in_proj_z.weight",
+ "src": "model.layers.49.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.norm.weight",
+ "src": "model.layers.49.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.out_proj.weight",
+ "src": "model.layers.49.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.mlp.down_proj.weight",
+ "src": "model.layers.49.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.mlp.gate_proj.weight",
+ "src": "model.layers.49.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.mlp.up_proj.weight",
+ "src": "model.layers.49.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.post_attention_layernorm.weight",
+ "src": "model.layers.49.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.input_layernorm.weight",
+ "src": "model.layers.50.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.A_log",
+ "src": "model.layers.50.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.conv1d.weight",
+ "src": "model.layers.50.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.dt_bias",
+ "src": "model.layers.50.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.in_proj_a.weight",
+ "src": "model.layers.50.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.in_proj_b.weight",
+ "src": "model.layers.50.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.50.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.in_proj_z.weight",
+ "src": "model.layers.50.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.norm.weight",
+ "src": "model.layers.50.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.out_proj.weight",
+ "src": "model.layers.50.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.mlp.down_proj.weight",
+ "src": "model.layers.50.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.mlp.gate_proj.weight",
+ "src": "model.layers.50.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.mlp.up_proj.weight",
+ "src": "model.layers.50.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.post_attention_layernorm.weight",
+ "src": "model.layers.50.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.input_layernorm.weight",
+ "src": "model.layers.51.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.mlp.down_proj.weight",
+ "src": "model.layers.51.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.mlp.gate_proj.weight",
+ "src": "model.layers.51.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.mlp.up_proj.weight",
+ "src": "model.layers.51.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.post_attention_layernorm.weight",
+ "src": "model.layers.51.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.self_attn.k_norm.weight",
+ "src": "model.layers.51.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.self_attn.k_proj.weight",
+ "src": "model.layers.51.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.self_attn.o_proj.weight",
+ "src": "model.layers.51.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.self_attn.q_norm.weight",
+ "src": "model.layers.51.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.self_attn.q_proj.weight",
+ "src": "model.layers.51.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.self_attn.v_proj.weight",
+ "src": "model.layers.51.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.104.input_layernorm.weight",
+ "src": "model.layers.52.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.A_log",
+ "src": "model.layers.52.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.conv1d.weight",
+ "src": "model.layers.52.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.dt_bias",
+ "src": "model.layers.52.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.in_proj_a.weight",
+ "src": "model.layers.52.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.in_proj_b.weight",
+ "src": "model.layers.52.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.52.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.in_proj_z.weight",
+ "src": "model.layers.52.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.norm.weight",
+ "src": "model.layers.52.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.out_proj.weight",
+ "src": "model.layers.52.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.mlp.down_proj.weight",
+ "src": "model.layers.52.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.mlp.gate_proj.weight",
+ "src": "model.layers.52.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.mlp.up_proj.weight",
+ "src": "model.layers.52.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.post_attention_layernorm.weight",
+ "src": "model.layers.52.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.input_layernorm.weight",
+ "src": "model.layers.53.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.A_log",
+ "src": "model.layers.53.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.conv1d.weight",
+ "src": "model.layers.53.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.dt_bias",
+ "src": "model.layers.53.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.in_proj_a.weight",
+ "src": "model.layers.53.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.in_proj_b.weight",
+ "src": "model.layers.53.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.53.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.in_proj_z.weight",
+ "src": "model.layers.53.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.norm.weight",
+ "src": "model.layers.53.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.out_proj.weight",
+ "src": "model.layers.53.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.mlp.down_proj.weight",
+ "src": "model.layers.53.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.mlp.gate_proj.weight",
+ "src": "model.layers.53.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.mlp.up_proj.weight",
+ "src": "model.layers.53.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.post_attention_layernorm.weight",
+ "src": "model.layers.53.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.input_layernorm.weight",
+ "src": "model.layers.54.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.A_log",
+ "src": "model.layers.54.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.conv1d.weight",
+ "src": "model.layers.54.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.dt_bias",
+ "src": "model.layers.54.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.in_proj_a.weight",
+ "src": "model.layers.54.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.in_proj_b.weight",
+ "src": "model.layers.54.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.54.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.in_proj_z.weight",
+ "src": "model.layers.54.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.norm.weight",
+ "src": "model.layers.54.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.out_proj.weight",
+ "src": "model.layers.54.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.mlp.down_proj.weight",
+ "src": "model.layers.54.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.mlp.gate_proj.weight",
+ "src": "model.layers.54.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.mlp.up_proj.weight",
+ "src": "model.layers.54.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.post_attention_layernorm.weight",
+ "src": "model.layers.54.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.input_layernorm.weight",
+ "src": "model.layers.55.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.mlp.down_proj.weight",
+ "src": "model.layers.55.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.mlp.gate_proj.weight",
+ "src": "model.layers.55.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.mlp.up_proj.weight",
+ "src": "model.layers.55.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.post_attention_layernorm.weight",
+ "src": "model.layers.55.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.self_attn.k_norm.weight",
+ "src": "model.layers.55.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.self_attn.k_proj.weight",
+ "src": "model.layers.55.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.self_attn.o_proj.weight",
+ "src": "model.layers.55.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.self_attn.q_norm.weight",
+ "src": "model.layers.55.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.self_attn.q_proj.weight",
+ "src": "model.layers.55.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.self_attn.v_proj.weight",
+ "src": "model.layers.55.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.108.input_layernorm.weight",
+ "src": "model.layers.52.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.A_log",
+ "src": "model.layers.52.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.conv1d.weight",
+ "src": "model.layers.52.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.dt_bias",
+ "src": "model.layers.52.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.in_proj_a.weight",
+ "src": "model.layers.52.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.in_proj_b.weight",
+ "src": "model.layers.52.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.52.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.in_proj_z.weight",
+ "src": "model.layers.52.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.norm.weight",
+ "src": "model.layers.52.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.out_proj.weight",
+ "src": "model.layers.52.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.mlp.down_proj.weight",
+ "src": "model.layers.52.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.mlp.gate_proj.weight",
+ "src": "model.layers.52.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.mlp.up_proj.weight",
+ "src": "model.layers.52.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.post_attention_layernorm.weight",
+ "src": "model.layers.52.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.input_layernorm.weight",
+ "src": "model.layers.53.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.A_log",
+ "src": "model.layers.53.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.conv1d.weight",
+ "src": "model.layers.53.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.dt_bias",
+ "src": "model.layers.53.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.in_proj_a.weight",
+ "src": "model.layers.53.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.in_proj_b.weight",
+ "src": "model.layers.53.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.53.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.in_proj_z.weight",
+ "src": "model.layers.53.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.norm.weight",
+ "src": "model.layers.53.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.out_proj.weight",
+ "src": "model.layers.53.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.mlp.down_proj.weight",
+ "src": "model.layers.53.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.mlp.gate_proj.weight",
+ "src": "model.layers.53.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.mlp.up_proj.weight",
+ "src": "model.layers.53.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.post_attention_layernorm.weight",
+ "src": "model.layers.53.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.input_layernorm.weight",
+ "src": "model.layers.54.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.A_log",
+ "src": "model.layers.54.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.conv1d.weight",
+ "src": "model.layers.54.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.dt_bias",
+ "src": "model.layers.54.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.in_proj_a.weight",
+ "src": "model.layers.54.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.in_proj_b.weight",
+ "src": "model.layers.54.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.54.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.in_proj_z.weight",
+ "src": "model.layers.54.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.norm.weight",
+ "src": "model.layers.54.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.out_proj.weight",
+ "src": "model.layers.54.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.mlp.down_proj.weight",
+ "src": "model.layers.54.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.mlp.gate_proj.weight",
+ "src": "model.layers.54.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.mlp.up_proj.weight",
+ "src": "model.layers.54.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.post_attention_layernorm.weight",
+ "src": "model.layers.54.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.input_layernorm.weight",
+ "src": "model.layers.55.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.mlp.down_proj.weight",
+ "src": "model.layers.55.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.mlp.gate_proj.weight",
+ "src": "model.layers.55.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.mlp.up_proj.weight",
+ "src": "model.layers.55.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.post_attention_layernorm.weight",
+ "src": "model.layers.55.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.self_attn.k_norm.weight",
+ "src": "model.layers.55.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.self_attn.k_proj.weight",
+ "src": "model.layers.55.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.self_attn.o_proj.weight",
+ "src": "model.layers.55.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.self_attn.q_norm.weight",
+ "src": "model.layers.55.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.self_attn.q_proj.weight",
+ "src": "model.layers.55.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.self_attn.v_proj.weight",
+ "src": "model.layers.55.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.112.input_layernorm.weight",
+ "src": "model.layers.56.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.A_log",
+ "src": "model.layers.56.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.conv1d.weight",
+ "src": "model.layers.56.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.dt_bias",
+ "src": "model.layers.56.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.in_proj_a.weight",
+ "src": "model.layers.56.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.in_proj_b.weight",
+ "src": "model.layers.56.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.56.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.in_proj_z.weight",
+ "src": "model.layers.56.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.norm.weight",
+ "src": "model.layers.56.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.out_proj.weight",
+ "src": "model.layers.56.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.mlp.down_proj.weight",
+ "src": "model.layers.56.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.mlp.gate_proj.weight",
+ "src": "model.layers.56.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.mlp.up_proj.weight",
+ "src": "model.layers.56.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.post_attention_layernorm.weight",
+ "src": "model.layers.56.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.input_layernorm.weight",
+ "src": "model.layers.57.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.A_log",
+ "src": "model.layers.57.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.conv1d.weight",
+ "src": "model.layers.57.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.dt_bias",
+ "src": "model.layers.57.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.in_proj_a.weight",
+ "src": "model.layers.57.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.in_proj_b.weight",
+ "src": "model.layers.57.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.57.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.in_proj_z.weight",
+ "src": "model.layers.57.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.norm.weight",
+ "src": "model.layers.57.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.out_proj.weight",
+ "src": "model.layers.57.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.mlp.down_proj.weight",
+ "src": "model.layers.57.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.mlp.gate_proj.weight",
+ "src": "model.layers.57.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.mlp.up_proj.weight",
+ "src": "model.layers.57.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.post_attention_layernorm.weight",
+ "src": "model.layers.57.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.input_layernorm.weight",
+ "src": "model.layers.58.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.A_log",
+ "src": "model.layers.58.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.conv1d.weight",
+ "src": "model.layers.58.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.dt_bias",
+ "src": "model.layers.58.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.in_proj_a.weight",
+ "src": "model.layers.58.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.in_proj_b.weight",
+ "src": "model.layers.58.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.58.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.in_proj_z.weight",
+ "src": "model.layers.58.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.norm.weight",
+ "src": "model.layers.58.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.out_proj.weight",
+ "src": "model.layers.58.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.mlp.down_proj.weight",
+ "src": "model.layers.58.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.mlp.gate_proj.weight",
+ "src": "model.layers.58.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.mlp.up_proj.weight",
+ "src": "model.layers.58.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.post_attention_layernorm.weight",
+ "src": "model.layers.58.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.input_layernorm.weight",
+ "src": "model.layers.59.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.mlp.down_proj.weight",
+ "src": "model.layers.59.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.mlp.gate_proj.weight",
+ "src": "model.layers.59.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.mlp.up_proj.weight",
+ "src": "model.layers.59.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.post_attention_layernorm.weight",
+ "src": "model.layers.59.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.self_attn.k_norm.weight",
+ "src": "model.layers.59.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.self_attn.k_proj.weight",
+ "src": "model.layers.59.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.self_attn.o_proj.weight",
+ "src": "model.layers.59.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.self_attn.q_norm.weight",
+ "src": "model.layers.59.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.self_attn.q_proj.weight",
+ "src": "model.layers.59.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.self_attn.v_proj.weight",
+ "src": "model.layers.59.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.116.input_layernorm.weight",
+ "src": "model.layers.56.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.A_log",
+ "src": "model.layers.56.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.conv1d.weight",
+ "src": "model.layers.56.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.dt_bias",
+ "src": "model.layers.56.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.in_proj_a.weight",
+ "src": "model.layers.56.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.in_proj_b.weight",
+ "src": "model.layers.56.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.56.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.in_proj_z.weight",
+ "src": "model.layers.56.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.norm.weight",
+ "src": "model.layers.56.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.out_proj.weight",
+ "src": "model.layers.56.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.mlp.down_proj.weight",
+ "src": "model.layers.56.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.mlp.gate_proj.weight",
+ "src": "model.layers.56.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.mlp.up_proj.weight",
+ "src": "model.layers.56.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.post_attention_layernorm.weight",
+ "src": "model.layers.56.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.input_layernorm.weight",
+ "src": "model.layers.57.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.A_log",
+ "src": "model.layers.57.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.conv1d.weight",
+ "src": "model.layers.57.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.dt_bias",
+ "src": "model.layers.57.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.in_proj_a.weight",
+ "src": "model.layers.57.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.in_proj_b.weight",
+ "src": "model.layers.57.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.57.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.in_proj_z.weight",
+ "src": "model.layers.57.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.norm.weight",
+ "src": "model.layers.57.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.out_proj.weight",
+ "src": "model.layers.57.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.mlp.down_proj.weight",
+ "src": "model.layers.57.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.mlp.gate_proj.weight",
+ "src": "model.layers.57.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.mlp.up_proj.weight",
+ "src": "model.layers.57.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.post_attention_layernorm.weight",
+ "src": "model.layers.57.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.input_layernorm.weight",
+ "src": "model.layers.58.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.A_log",
+ "src": "model.layers.58.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.conv1d.weight",
+ "src": "model.layers.58.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.dt_bias",
+ "src": "model.layers.58.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.in_proj_a.weight",
+ "src": "model.layers.58.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.in_proj_b.weight",
+ "src": "model.layers.58.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.58.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.in_proj_z.weight",
+ "src": "model.layers.58.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.norm.weight",
+ "src": "model.layers.58.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.out_proj.weight",
+ "src": "model.layers.58.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.mlp.down_proj.weight",
+ "src": "model.layers.58.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.mlp.gate_proj.weight",
+ "src": "model.layers.58.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.mlp.up_proj.weight",
+ "src": "model.layers.58.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.post_attention_layernorm.weight",
+ "src": "model.layers.58.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.input_layernorm.weight",
+ "src": "model.layers.59.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.mlp.down_proj.weight",
+ "src": "model.layers.59.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.mlp.gate_proj.weight",
+ "src": "model.layers.59.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.mlp.up_proj.weight",
+ "src": "model.layers.59.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.post_attention_layernorm.weight",
+ "src": "model.layers.59.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.self_attn.k_norm.weight",
+ "src": "model.layers.59.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.self_attn.k_proj.weight",
+ "src": "model.layers.59.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.self_attn.o_proj.weight",
+ "src": "model.layers.59.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.self_attn.q_norm.weight",
+ "src": "model.layers.59.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.self_attn.q_proj.weight",
+ "src": "model.layers.59.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.self_attn.v_proj.weight",
+ "src": "model.layers.59.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.120.input_layernorm.weight",
+ "src": "model.layers.60.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.A_log",
+ "src": "model.layers.60.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.conv1d.weight",
+ "src": "model.layers.60.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.dt_bias",
+ "src": "model.layers.60.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.in_proj_a.weight",
+ "src": "model.layers.60.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.in_proj_b.weight",
+ "src": "model.layers.60.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.60.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.in_proj_z.weight",
+ "src": "model.layers.60.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.norm.weight",
+ "src": "model.layers.60.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.out_proj.weight",
+ "src": "model.layers.60.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.mlp.down_proj.weight",
+ "src": "model.layers.60.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.mlp.gate_proj.weight",
+ "src": "model.layers.60.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.mlp.up_proj.weight",
+ "src": "model.layers.60.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.post_attention_layernorm.weight",
+ "src": "model.layers.60.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.input_layernorm.weight",
+ "src": "model.layers.61.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.A_log",
+ "src": "model.layers.61.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.conv1d.weight",
+ "src": "model.layers.61.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.dt_bias",
+ "src": "model.layers.61.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.in_proj_a.weight",
+ "src": "model.layers.61.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.in_proj_b.weight",
+ "src": "model.layers.61.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.61.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.in_proj_z.weight",
+ "src": "model.layers.61.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.norm.weight",
+ "src": "model.layers.61.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.out_proj.weight",
+ "src": "model.layers.61.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.mlp.down_proj.weight",
+ "src": "model.layers.61.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.mlp.gate_proj.weight",
+ "src": "model.layers.61.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.mlp.up_proj.weight",
+ "src": "model.layers.61.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.post_attention_layernorm.weight",
+ "src": "model.layers.61.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.input_layernorm.weight",
+ "src": "model.layers.62.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.A_log",
+ "src": "model.layers.62.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.conv1d.weight",
+ "src": "model.layers.62.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.dt_bias",
+ "src": "model.layers.62.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.in_proj_a.weight",
+ "src": "model.layers.62.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.in_proj_b.weight",
+ "src": "model.layers.62.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.62.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.in_proj_z.weight",
+ "src": "model.layers.62.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.norm.weight",
+ "src": "model.layers.62.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.out_proj.weight",
+ "src": "model.layers.62.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.mlp.down_proj.weight",
+ "src": "model.layers.62.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.mlp.gate_proj.weight",
+ "src": "model.layers.62.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.mlp.up_proj.weight",
+ "src": "model.layers.62.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.post_attention_layernorm.weight",
+ "src": "model.layers.62.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.input_layernorm.weight",
+ "src": "model.layers.63.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.mlp.down_proj.weight",
+ "src": "model.layers.63.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.mlp.gate_proj.weight",
+ "src": "model.layers.63.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.mlp.up_proj.weight",
+ "src": "model.layers.63.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.post_attention_layernorm.weight",
+ "src": "model.layers.63.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.self_attn.k_norm.weight",
+ "src": "model.layers.63.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.self_attn.k_proj.weight",
+ "src": "model.layers.63.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.self_attn.o_proj.weight",
+ "src": "model.layers.63.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.self_attn.q_norm.weight",
+ "src": "model.layers.63.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.self_attn.q_proj.weight",
+ "src": "model.layers.63.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.self_attn.v_proj.weight",
+ "src": "model.layers.63.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.124.input_layernorm.weight",
+ "src": "model.layers.60.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.A_log",
+ "src": "model.layers.60.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.conv1d.weight",
+ "src": "model.layers.60.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.dt_bias",
+ "src": "model.layers.60.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.in_proj_a.weight",
+ "src": "model.layers.60.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.in_proj_b.weight",
+ "src": "model.layers.60.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.60.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.in_proj_z.weight",
+ "src": "model.layers.60.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.norm.weight",
+ "src": "model.layers.60.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.out_proj.weight",
+ "src": "model.layers.60.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.mlp.down_proj.weight",
+ "src": "model.layers.60.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.mlp.gate_proj.weight",
+ "src": "model.layers.60.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.mlp.up_proj.weight",
+ "src": "model.layers.60.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.post_attention_layernorm.weight",
+ "src": "model.layers.60.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.input_layernorm.weight",
+ "src": "model.layers.61.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.A_log",
+ "src": "model.layers.61.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.conv1d.weight",
+ "src": "model.layers.61.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.dt_bias",
+ "src": "model.layers.61.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.in_proj_a.weight",
+ "src": "model.layers.61.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.in_proj_b.weight",
+ "src": "model.layers.61.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.61.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.in_proj_z.weight",
+ "src": "model.layers.61.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.norm.weight",
+ "src": "model.layers.61.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.out_proj.weight",
+ "src": "model.layers.61.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.mlp.down_proj.weight",
+ "src": "model.layers.61.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.mlp.gate_proj.weight",
+ "src": "model.layers.61.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.mlp.up_proj.weight",
+ "src": "model.layers.61.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.post_attention_layernorm.weight",
+ "src": "model.layers.61.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.input_layernorm.weight",
+ "src": "model.layers.62.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.A_log",
+ "src": "model.layers.62.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.conv1d.weight",
+ "src": "model.layers.62.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.dt_bias",
+ "src": "model.layers.62.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.in_proj_a.weight",
+ "src": "model.layers.62.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.in_proj_b.weight",
+ "src": "model.layers.62.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.62.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.in_proj_z.weight",
+ "src": "model.layers.62.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.norm.weight",
+ "src": "model.layers.62.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.out_proj.weight",
+ "src": "model.layers.62.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.mlp.down_proj.weight",
+ "src": "model.layers.62.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.mlp.gate_proj.weight",
+ "src": "model.layers.62.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.mlp.up_proj.weight",
+ "src": "model.layers.62.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.post_attention_layernorm.weight",
+ "src": "model.layers.62.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.input_layernorm.weight",
+ "src": "model.layers.63.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.mlp.down_proj.weight",
+ "src": "model.layers.63.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.mlp.gate_proj.weight",
+ "src": "model.layers.63.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.mlp.up_proj.weight",
+ "src": "model.layers.63.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.post_attention_layernorm.weight",
+ "src": "model.layers.63.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.self_attn.k_norm.weight",
+ "src": "model.layers.63.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.self_attn.k_proj.weight",
+ "src": "model.layers.63.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.self_attn.o_proj.weight",
+ "src": "model.layers.63.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.self_attn.q_norm.weight",
+ "src": "model.layers.63.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.self_attn.q_proj.weight",
+ "src": "model.layers.63.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.self_attn.v_proj.weight",
+ "src": "model.layers.63.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.norm.weight",
+ "src": "model.norm.weight",
+ "op": "copy"
+ },
+ {
+ "key": "lm_head.weight",
+ "src": "lm_head.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.fc.weight",
+ "src": "mtp.fc.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.input_layernorm.weight",
+ "src": "mtp.layers.0.input_layernorm.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.mlp.down_proj.weight",
+ "src": "mtp.layers.0.mlp.down_proj.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.mlp.gate_proj.weight",
+ "src": "mtp.layers.0.mlp.gate_proj.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.mlp.up_proj.weight",
+ "src": "mtp.layers.0.mlp.up_proj.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.post_attention_layernorm.weight",
+ "src": "mtp.layers.0.post_attention_layernorm.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.self_attn.k_norm.weight",
+ "src": "mtp.layers.0.self_attn.k_norm.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.self_attn.k_proj.weight",
+ "src": "mtp.layers.0.self_attn.k_proj.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.self_attn.o_proj.weight",
+ "src": "mtp.layers.0.self_attn.o_proj.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.self_attn.q_norm.weight",
+ "src": "mtp.layers.0.self_attn.q_norm.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.self_attn.q_proj.weight",
+ "src": "mtp.layers.0.self_attn.q_proj.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.self_attn.v_proj.weight",
+ "src": "mtp.layers.0.self_attn.v_proj.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.norm.weight",
+ "src": "mtp.norm.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.pre_fc_norm_embedding.weight",
+ "src": "mtp.pre_fc_norm_embedding.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.pre_fc_norm_hidden.weight",
+ "src": "mtp.pre_fc_norm_hidden.weight",
+ "op": "copy"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/sft_final/gate_full.json b/sft_final/gate_full.json
new file mode 100644
index 0000000000000000000000000000000000000000..fd994473314956d3469f37eadfafa60e03341fbf
--- /dev/null
+++ b/sft_final/gate_full.json
@@ -0,0 +1,81 @@
+{
+ "verdict": "PASS",
+ "expected": "exact",
+ "mini": false,
+ "compute": "fp32blocks",
+ "a": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded",
+ "b": "/mnt/nfs_share/JiRackDeltaNet_70b/expanded",
+ "layers_run": [
+ 64,
+ 128
+ ],
+ "intermediate": [
+ 26624,
+ 26624
+ ],
+ "prompts": [
+ "The capital of France is",
+ "def fibonacci(n):\n ",
+ "Столица России —"
+ ],
+ "ids": [
+ [
+ 760,
+ 6511,
+ 314,
+ 9338,
+ 369
+ ],
+ [
+ 727,
+ 73111,
+ 1393,
+ 1590,
+ 198,
+ 257
+ ],
+ [
+ 212396,
+ 168314,
+ 170312,
+ 1892
+ ]
+ ],
+ "rows": [
+ {
+ "T": 5,
+ "identical": true,
+ "max_abs": 0.0,
+ "mean_abs": 0.0,
+ "max_kl": 0.0,
+ "top1_agree": 5
+ },
+ {
+ "T": 6,
+ "identical": true,
+ "max_abs": 0.0,
+ "mean_abs": 0.0,
+ "max_kl": 0.0,
+ "top1_agree": 6
+ },
+ {
+ "T": 4,
+ "identical": true,
+ "max_abs": 0.0,
+ "mean_abs": 0.0,
+ "max_kl": 0.0,
+ "top1_agree": 4
+ }
+ ],
+ "a_info": {
+ "cached": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded/gate_logits_fp32blocks_L64.npz"
+ },
+ "b_info": {
+ "loaded": 1714,
+ "skipped": 0,
+ "unexpected": [],
+ "missing_buffers": [],
+ "seconds": 2016
+ },
+ "torch": "2.13.0+cu130"
+}
\ No newline at end of file
diff --git a/sft_final/gate_logits_fp32blocks_L128.npz b/sft_final/gate_logits_fp32blocks_L128.npz
new file mode 100644
index 0000000000000000000000000000000000000000..f27c114c7eb269402bd0dd3e1ddb0caca7507239
--- /dev/null
+++ b/sft_final/gate_logits_fp32blocks_L128.npz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a589ae8136832e11c0c408e43a560cf110f011e3ecef3b60852db63eb256d025
+size 14900556
diff --git a/sft_final/gate_logits_fp32blocks_L8.npz b/sft_final/gate_logits_fp32blocks_L8.npz
new file mode 100644
index 0000000000000000000000000000000000000000..b368f7b678cbc848a11eb979478a0eab0d1e5143
--- /dev/null
+++ b/sft_final/gate_logits_fp32blocks_L8.npz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8975717cc37cabfaffcea30896d5aaefceb526cb2d0b1a0b993830d57358e29e
+size 14900556
diff --git a/sft_final/gate_mini.json b/sft_final/gate_mini.json
new file mode 100644
index 0000000000000000000000000000000000000000..1dee6cad5b932734ff025c277058bef355c5eea5
--- /dev/null
+++ b/sft_final/gate_mini.json
@@ -0,0 +1,81 @@
+{
+ "verdict": "PASS",
+ "expected": "exact",
+ "mini": true,
+ "compute": "fp32blocks",
+ "a": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded",
+ "b": "/mnt/nfs_share/JiRackDeltaNet_70b/expanded",
+ "layers_run": [
+ 4,
+ 8
+ ],
+ "intermediate": [
+ 26624,
+ 26624
+ ],
+ "prompts": [
+ "The capital of France is",
+ "def fibonacci(n):\n ",
+ "Столица России —"
+ ],
+ "ids": [
+ [
+ 760,
+ 6511,
+ 314,
+ 9338,
+ 369
+ ],
+ [
+ 727,
+ 73111,
+ 1393,
+ 1590,
+ 198,
+ 257
+ ],
+ [
+ 212396,
+ 168314,
+ 170312,
+ 1892
+ ]
+ ],
+ "rows": [
+ {
+ "T": 5,
+ "identical": true,
+ "max_abs": 0.0,
+ "mean_abs": 0.0,
+ "max_kl": 0.0,
+ "top1_agree": 5
+ },
+ {
+ "T": 6,
+ "identical": true,
+ "max_abs": 0.0,
+ "mean_abs": 0.0,
+ "max_kl": 0.0,
+ "top1_agree": 6
+ },
+ {
+ "T": 4,
+ "identical": true,
+ "max_abs": 0.0,
+ "mean_abs": 0.0,
+ "max_kl": 0.0,
+ "top1_agree": 4
+ }
+ ],
+ "a_info": {
+ "cached": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded/gate_logits_fp32blocks_L4.npz"
+ },
+ "b_info": {
+ "loaded": 124,
+ "skipped": 1590,
+ "unexpected": [],
+ "missing_buffers": [],
+ "seconds": 138
+ },
+ "torch": "2.13.0+cu130"
+}
\ No newline at end of file
diff --git a/sft_final/gguf.txt b/sft_final/gguf.txt
new file mode 100644
index 0000000000000000000000000000000000000000..79bf8f0314f527cf5c6c5f85426a44eb306fe28d
--- /dev/null
+++ b/sft_final/gguf.txt
@@ -0,0 +1,35 @@
+# 1. Конвертация в bf16 (как договорились)
+python convert_hf_to_gguf.py /mnt/nfs_share/Qwen3_8 \
+ --outfile /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.gguf --outtype bf16
+
+# 2. Сборка llama-quantize (один раз)
+cd /mnt/nfs_share/llama.cpp
+cmake -B build
+cmake --build build -j
+
+# 3. Квантование в Q4_K_M
+./build/bin/llama-quantize \
+ /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.gguf \
+ /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.Q4_K_M.gguf \
+ Q4_K_M
+
+
+
+cd /mnt/nfs_share/llama.cpp
+cmake -B build -DGGML_CUDA=ON # для NVIDIA
+make -C build
+
+
+/mnt/nfs_share/llama.cpp/build/bin/llama-cli \
+ -m /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.Q4_K_M.gguf \
+ -n 512 \
+ -c 4096 \
+ -p "Explain quantum computing in simple terms:"
+
+
+
+
+
+/mnt/nfs_share/llama.cpp/build/bin/llama-server \
+ -m /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.Q4_K_M.gguf \
+ --port 8000
diff --git a/sft_final/jirack_sft_state.json b/sft_final/jirack_sft_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ccb679b502005f458f3dea7722e9fc187a51eee
--- /dev/null
+++ b/sft_final/jirack_sft_state.json
@@ -0,0 +1,9 @@
+{
+ "global_step": 374,
+ "lr": 1e-05,
+ "max_steps": 374,
+ "shards": "0",
+ "tok_micro": 8192,
+ "accum": 2,
+ "init_dir": "/home/ubuntu/deltanet70b"
+}
\ No newline at end of file
diff --git a/sft_final/model-00001-of-00027.safetensors b/sft_final/model-00001-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..10d0d41d95c4f8846529fbf5c76fe46c5ce1e3c6
--- /dev/null
+++ b/sft_final/model-00001-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1360d4af0ce009aaa30a0159a667ddc59298e927a8cdf820f015db8d17eba0a
+size 5146507008
diff --git a/sft_final/model-00002-of-00027.safetensors b/sft_final/model-00002-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0fc276c64d492bafbca66f1e6d9cdd655a62ec88
--- /dev/null
+++ b/sft_final/model-00002-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b88380f4538c1d5a023e89b795fb9e3aa7d1887ea8e14edd20ee451bc55a075
+size 5204245040
diff --git a/sft_final/model-00003-of-00027.safetensors b/sft_final/model-00003-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d09f4435c94ecb3aef293cb835bc32dc55b43320
--- /dev/null
+++ b/sft_final/model-00003-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07ee192c49b4fa41f8236a3b92153820c42cc4bc4ee8cfdb985eeded7cd48127
+size 5226281296
diff --git a/sft_final/model-00004-of-00027.safetensors b/sft_final/model-00004-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..484d609eb022f6835c9a3f5af41bbc746dd98911
--- /dev/null
+++ b/sft_final/model-00004-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55e52de05145920a220e89bb564adf50e43ea99597e6a566dcc5744b9066807c
+size 5226281352
diff --git a/sft_final/model-00005-of-00027.safetensors b/sft_final/model-00005-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..bac6117f11a4f944d22eedb82053d4437f3a0bda
--- /dev/null
+++ b/sft_final/model-00005-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebb1ecb964ee82f89f796796689e767c876440672c90f6a1c0793405c42cd2f0
+size 5226281352
diff --git a/sft_final/model-00006-of-00027.safetensors b/sft_final/model-00006-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..a5b4f60d364cba6762209a1fa9b4b07e34344659
--- /dev/null
+++ b/sft_final/model-00006-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57ca557f3b558c080e8bde31c87e4214099995e408df8d9d207ef5260d05f70c
+size 5204245104
diff --git a/sft_final/model-00007-of-00027.safetensors b/sft_final/model-00007-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e1a429c15bfdc2d1a22109cd50dbb3f5cb721b47
--- /dev/null
+++ b/sft_final/model-00007-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4bd5929c8cb98f744f8d0d6a82adc3c21960dcfc357c8db9ba11d10cbcdd2ba
+size 5226281352
diff --git a/sft_final/model-00008-of-00027.safetensors b/sft_final/model-00008-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..2bc7c10a8c57faf42f1c254e786446826e79bdde
--- /dev/null
+++ b/sft_final/model-00008-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de56d111c6e08a02a5d7c863adf0c05980c4efa23e742113caacda1d248eb304
+size 5226281352
diff --git a/sft_final/model-00009-of-00027.safetensors b/sft_final/model-00009-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..404fa3a6ba83bb2f36d57817c64577af527349b6
--- /dev/null
+++ b/sft_final/model-00009-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cb7e2b4989b3be6bfd0ba94bd84c14b06356cb76aa199e9f788af3ce12ac73c
+size 5226281352
diff --git a/sft_final/model-00010-of-00027.safetensors b/sft_final/model-00010-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b30a72c36b4ff71a5af1ae985dba9fd039f93ee1
--- /dev/null
+++ b/sft_final/model-00010-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd4c4509fdf30a4404d16d7d3c734eb28bc8bafe1864a1759b9c0dc33e66da62
+size 5204245104
diff --git a/sft_final/model-00011-of-00027.safetensors b/sft_final/model-00011-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..84efb0c7fe6b78b98588d218ae2877b3d3e0c277
--- /dev/null
+++ b/sft_final/model-00011-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce3bc4cc3bbf0f02ae4ee3f703df6fc824ffc1a55086421f8c7ca7f8d00bdd3c
+size 5226281352
diff --git a/sft_final/model-00012-of-00027.safetensors b/sft_final/model-00012-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..96844fbc7da10f9709791848222006f24d0a9cc3
--- /dev/null
+++ b/sft_final/model-00012-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fecbfb15b8e92fd41b22d3509a178572764b6ffd39b4142dc190e466e925dfc
+size 5226281352
diff --git a/sft_final/model-00013-of-00027.safetensors b/sft_final/model-00013-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..28a49e98ddd18c49ebd8fd0094dd34733293e15e
--- /dev/null
+++ b/sft_final/model-00013-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27236071639417aaa92d756239fc403f2b6ab72d3fef247a614a2f681f107796
+size 5226281352
diff --git a/sft_final/model-00014-of-00027.safetensors b/sft_final/model-00014-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..118fb5d0144a2152713e15bf1e50c690e77ef2ca
--- /dev/null
+++ b/sft_final/model-00014-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47c992e281a5387c19de30b86c344b3d3b0649ab2d11977b9c294d49cee53ed1
+size 5204245104
diff --git a/sft_final/model-00015-of-00027.safetensors b/sft_final/model-00015-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..bcc415be0ec97165a00f5a044bb7957594fa4ed3
--- /dev/null
+++ b/sft_final/model-00015-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d7e1f039d216da37c40e655775534c52210ca7bb08b90d902459be52b39959d
+size 5226281352
diff --git a/sft_final/model-00016-of-00027.safetensors b/sft_final/model-00016-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..eb54ab81b0b792d4e9e99e8057ab275de1f8552f
--- /dev/null
+++ b/sft_final/model-00016-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcb1983d57449ce9af7f702cf5db51cedae41df04c418acdc2c7e78852a8af9b
+size 5226281352
diff --git a/sft_final/model-00017-of-00027.safetensors b/sft_final/model-00017-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ae315c9cf164fd3b1e4a09a13944c2866bd1a9bd
--- /dev/null
+++ b/sft_final/model-00017-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0fb91fdf474b9f2f0b7e404b304a126047fb9da957934a70ff658a107d9ecfd
+size 5226281352
diff --git a/sft_final/model-00018-of-00027.safetensors b/sft_final/model-00018-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..068474f9f12984481523f014d334fcfd39c95612
--- /dev/null
+++ b/sft_final/model-00018-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd5eb70f5171fb1dceba0d0277bfc5af75fa895952bd646717bfd6a25958f017
+size 5204245104
diff --git a/sft_final/model-00019-of-00027.safetensors b/sft_final/model-00019-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1b2f8c2e9e45e640d6b0da2a12044dad469bcfa9
--- /dev/null
+++ b/sft_final/model-00019-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49e77b4426051b3fc8c7c18da83d48288f3ca4b694fc687a92d903fdd5a07063
+size 5226281352
diff --git a/sft_final/model-00020-of-00027.safetensors b/sft_final/model-00020-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e4485ec4824de47b0537fee42a1c52e748ce90dd
--- /dev/null
+++ b/sft_final/model-00020-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4fe115d9440d1732381ee44bbc07e142d5871c1e288bcdf3598273261daace87
+size 5226281352
diff --git a/sft_final/model-00021-of-00027.safetensors b/sft_final/model-00021-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b6ff93cd7828a3754405ede8bff6fa10b2d7c477
--- /dev/null
+++ b/sft_final/model-00021-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22652bf288a10fa233fcf7adcc9d02d4309643badbd0c8cce1274c0cf9ec715b
+size 5226281360
diff --git a/sft_final/model-00022-of-00027.safetensors b/sft_final/model-00022-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ed62d18705dedf92211cd6f3c8cf7ef8a9104b62
--- /dev/null
+++ b/sft_final/model-00022-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6db8017e7414d546907bd8ebb90458e9932b4b8d5dfcbe071ecf4f5d982aa289
+size 5204245168
diff --git a/sft_final/model-00023-of-00027.safetensors b/sft_final/model-00023-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..e24aad7ac61a41e00ffda4fef41e8687c1208e4e
--- /dev/null
+++ b/sft_final/model-00023-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50cc1a1b337270c327cb90566b1fd99c9c3e42f08e96f148fca6c1f8c432c7c1
+size 5226281424
diff --git a/sft_final/model-00024-of-00027.safetensors b/sft_final/model-00024-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fd45b044be77d70218289a6474c020f2b33f4242
--- /dev/null
+++ b/sft_final/model-00024-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24b94858756a2c0ad41faf3f33fc6a15e3406844288896336cd12112e79755ac
+size 5226281424
diff --git a/sft_final/model-00025-of-00027.safetensors b/sft_final/model-00025-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..460ae65ebf088e77fa7cae26ab87b1baf0f1c51c
--- /dev/null
+++ b/sft_final/model-00025-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca3ab652f04c9b441244c00fe1a50304309c6a7486c2f37cafd0ceb22ae86208
+size 5226281424
diff --git a/sft_final/model-00026-of-00027.safetensors b/sft_final/model-00026-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..174298932a19680f6eb7a15ed358a9a40de12179
--- /dev/null
+++ b/sft_final/model-00026-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:360f47926cc9428ee23e4fb75ea1911ed4f0970a19679ce5ed80338f15409328
+size 5204245168
diff --git a/sft_final/model-00027-of-00027.safetensors b/sft_final/model-00027-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..6534ac5ab8b014d3898ee290b7a71234b9966538
--- /dev/null
+++ b/sft_final/model-00027-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16fe0a0184cdf5468f0e406cba58990cc930932519bbd5bd5ac3163261ffb41b
+size 4220603664
diff --git a/sft_final/model.safetensors.index.json b/sft_final/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..c491b9236a690a00f19e1dca19eb27b8ff5221f1
--- /dev/null
+++ b/sft_final/model.safetensors.index.json
@@ -0,0 +1,1721 @@
+{
+ "metadata": {
+ "total_size": 139869690880
+ },
+ "weight_map": {
+ "model.embed_tokens.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.dt_bias": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.A_log": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.conv1d.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.norm.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.out_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.in_proj_qkv.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.in_proj_z.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.in_proj_b.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.in_proj_a.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.mlp.down_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.dt_bias": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.A_log": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.conv1d.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.norm.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.out_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.in_proj_qkv.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.in_proj_z.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.in_proj_b.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.in_proj_a.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.mlp.up_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.mlp.down_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.input_layernorm.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.dt_bias": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.A_log": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.conv1d.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.norm.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.out_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.in_proj_qkv.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.in_proj_z.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.in_proj_b.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.in_proj_a.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.mlp.up_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.2.mlp.down_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.2.input_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.q_norm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.k_norm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.mlp.up_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.mlp.down_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.input_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.dt_bias": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.A_log": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.conv1d.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.norm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.out_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.in_proj_qkv.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.in_proj_z.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.in_proj_b.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.in_proj_a.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.mlp.up_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.mlp.down_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.input_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.dt_bias": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.A_log": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.conv1d.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.norm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.out_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.in_proj_qkv.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.in_proj_z.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.in_proj_b.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.in_proj_a.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.mlp.up_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.mlp.down_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.input_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.dt_bias": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.A_log": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.conv1d.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.norm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.out_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.in_proj_qkv.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.in_proj_z.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.in_proj_b.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.in_proj_a.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.mlp.up_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.mlp.down_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.input_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.mlp.up_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.7.mlp.down_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.7.input_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.7.post_attention_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.dt_bias": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.A_log": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.conv1d.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.norm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.out_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.in_proj_qkv.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.in_proj_z.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.in_proj_b.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.in_proj_a.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.mlp.gate_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.mlp.up_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.mlp.down_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.input_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.dt_bias": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.A_log": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.conv1d.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.norm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.out_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.in_proj_qkv.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.in_proj_z.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.in_proj_b.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.in_proj_a.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.mlp.up_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.mlp.down_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.input_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.dt_bias": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.A_log": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.conv1d.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.norm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.out_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.in_proj_qkv.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.in_proj_z.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.in_proj_b.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.in_proj_a.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.mlp.up_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.mlp.down_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.input_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.q_norm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.k_norm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.mlp.up_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.mlp.down_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.input_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.dt_bias": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.A_log": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.conv1d.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.norm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.out_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.in_proj_qkv.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.in_proj_z.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.in_proj_b.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.in_proj_a.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.mlp.up_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.12.mlp.down_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.12.input_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.12.post_attention_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.dt_bias": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.A_log": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.conv1d.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.norm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.out_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.in_proj_qkv.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.in_proj_z.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.in_proj_b.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.in_proj_a.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.mlp.gate_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.mlp.up_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.mlp.down_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.input_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.post_attention_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.dt_bias": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.A_log": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.conv1d.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.norm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.out_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.in_proj_qkv.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.in_proj_z.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.in_proj_b.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.in_proj_a.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.mlp.gate_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.mlp.up_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.mlp.down_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.input_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.q_norm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.k_norm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.mlp.up_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.mlp.down_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.input_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.dt_bias": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.A_log": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.conv1d.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.norm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.out_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.in_proj_qkv.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.in_proj_z.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.in_proj_b.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.in_proj_a.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.mlp.up_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.mlp.down_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.input_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.dt_bias": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.A_log": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.conv1d.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.norm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.out_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.in_proj_qkv.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.in_proj_z.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.in_proj_b.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.in_proj_a.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.mlp.up_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.17.mlp.down_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.17.input_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.17.post_attention_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.dt_bias": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.A_log": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.conv1d.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.norm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.out_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.in_proj_qkv.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.in_proj_z.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.in_proj_b.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.in_proj_a.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.mlp.gate_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.mlp.up_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.mlp.down_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.input_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.post_attention_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.q_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.k_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.v_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.o_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.q_norm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.k_norm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.mlp.gate_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.mlp.up_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.mlp.down_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.input_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.post_attention_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.dt_bias": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.A_log": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.conv1d.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.norm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.out_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.in_proj_qkv.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.in_proj_z.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.in_proj_b.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.in_proj_a.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.mlp.up_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.mlp.down_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.input_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.dt_bias": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.A_log": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.conv1d.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.norm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.out_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.in_proj_qkv.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.in_proj_z.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.in_proj_b.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.in_proj_a.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.mlp.up_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.mlp.down_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.input_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.dt_bias": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.A_log": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.conv1d.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.norm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.out_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.in_proj_qkv.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.in_proj_z.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.in_proj_b.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.in_proj_a.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.mlp.up_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.22.mlp.down_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.22.input_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.22.post_attention_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.q_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.k_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.v_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.o_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.q_norm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.k_norm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.mlp.gate_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.mlp.up_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.mlp.down_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.input_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.post_attention_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.dt_bias": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.A_log": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.conv1d.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.norm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.out_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.in_proj_qkv.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.in_proj_z.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.in_proj_b.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.in_proj_a.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.mlp.gate_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.mlp.up_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.mlp.down_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.input_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.post_attention_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.dt_bias": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.A_log": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.conv1d.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.norm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.out_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.in_proj_qkv.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.in_proj_z.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.in_proj_b.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.in_proj_a.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.mlp.gate_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.mlp.up_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.mlp.down_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.input_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.dt_bias": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.A_log": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.conv1d.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.norm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.out_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.in_proj_qkv.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.in_proj_z.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.in_proj_b.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.in_proj_a.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.mlp.up_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.mlp.down_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.input_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.q_norm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.k_norm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.mlp.up_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.27.mlp.down_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.27.input_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.27.post_attention_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.dt_bias": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.A_log": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.conv1d.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.norm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.out_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.in_proj_qkv.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.in_proj_z.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.in_proj_b.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.in_proj_a.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.mlp.gate_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.mlp.up_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.mlp.down_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.input_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.post_attention_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.dt_bias": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.A_log": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.conv1d.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.norm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.out_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.in_proj_qkv.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.in_proj_z.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.in_proj_b.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.in_proj_a.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.mlp.gate_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.mlp.up_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.mlp.down_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.input_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.post_attention_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.dt_bias": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.A_log": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.conv1d.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.norm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.out_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.in_proj_qkv.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.in_proj_z.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.in_proj_b.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.in_proj_a.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.mlp.gate_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.mlp.up_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.mlp.down_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.input_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.post_attention_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.q_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.k_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.v_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.o_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.q_norm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.k_norm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.mlp.gate_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.mlp.up_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.mlp.down_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.input_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.dt_bias": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.A_log": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.conv1d.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.norm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.out_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.in_proj_qkv.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.in_proj_z.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.in_proj_b.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.in_proj_a.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.mlp.gate_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.mlp.up_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.32.mlp.down_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.32.input_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.32.post_attention_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.dt_bias": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.A_log": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.conv1d.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.norm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.out_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.in_proj_qkv.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.in_proj_z.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.in_proj_b.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.in_proj_a.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.mlp.gate_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.mlp.up_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.mlp.down_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.input_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.post_attention_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.dt_bias": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.A_log": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.conv1d.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.norm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.out_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.in_proj_qkv.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.in_proj_z.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.in_proj_b.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.in_proj_a.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.mlp.gate_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.mlp.up_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.mlp.down_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.input_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.post_attention_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.q_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.k_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.v_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.o_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.q_norm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.k_norm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.mlp.gate_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.mlp.up_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.mlp.down_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.input_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.post_attention_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.dt_bias": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.A_log": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.conv1d.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.norm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.out_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.in_proj_qkv.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.in_proj_z.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.in_proj_b.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.in_proj_a.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.mlp.gate_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.mlp.up_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.mlp.down_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.input_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.post_attention_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.dt_bias": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.A_log": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.conv1d.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.norm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.out_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.in_proj_qkv.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.in_proj_z.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.in_proj_b.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.in_proj_a.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.mlp.gate_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.mlp.up_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.37.mlp.down_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.37.input_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.37.post_attention_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.dt_bias": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.A_log": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.conv1d.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.norm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.out_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.in_proj_qkv.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.in_proj_z.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.in_proj_b.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.in_proj_a.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.mlp.gate_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.mlp.up_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.mlp.down_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.input_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.post_attention_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.q_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.k_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.v_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.o_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.q_norm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.k_norm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.mlp.gate_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.mlp.up_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.mlp.down_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.input_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.post_attention_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.dt_bias": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.A_log": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.conv1d.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.norm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.out_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.in_proj_qkv.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.in_proj_z.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.in_proj_b.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.in_proj_a.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.mlp.gate_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.mlp.up_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.mlp.down_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.input_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.post_attention_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.dt_bias": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.A_log": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.conv1d.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.norm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.out_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.in_proj_qkv.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.in_proj_z.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.in_proj_b.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.in_proj_a.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.mlp.gate_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.mlp.up_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.mlp.down_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.input_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.post_attention_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.dt_bias": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.A_log": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.conv1d.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.norm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.out_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.in_proj_qkv.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.in_proj_z.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.in_proj_b.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.in_proj_a.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.mlp.gate_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.mlp.up_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.42.mlp.down_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.42.input_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.42.post_attention_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.q_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.k_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.v_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.o_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.q_norm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.k_norm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.mlp.gate_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.mlp.up_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.mlp.down_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.input_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.post_attention_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.dt_bias": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.A_log": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.conv1d.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.norm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.out_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.in_proj_qkv.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.in_proj_z.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.in_proj_b.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.in_proj_a.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.mlp.gate_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.mlp.up_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.mlp.down_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.input_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.post_attention_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.dt_bias": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.A_log": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.conv1d.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.norm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.out_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.in_proj_qkv.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.in_proj_z.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.in_proj_b.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.in_proj_a.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.mlp.gate_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.mlp.up_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.mlp.down_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.input_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.post_attention_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.dt_bias": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.A_log": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.conv1d.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.norm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.out_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.in_proj_qkv.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.in_proj_z.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.in_proj_b.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.in_proj_a.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.mlp.gate_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.mlp.up_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.mlp.down_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.input_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.post_attention_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.q_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.k_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.v_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.o_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.q_norm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.k_norm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.mlp.gate_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.mlp.up_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.47.mlp.down_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.47.input_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.47.post_attention_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.dt_bias": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.A_log": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.conv1d.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.norm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.out_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.in_proj_qkv.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.in_proj_z.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.in_proj_b.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.in_proj_a.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.mlp.gate_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.mlp.up_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.mlp.down_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.input_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.post_attention_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.dt_bias": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.A_log": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.conv1d.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.norm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.out_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.in_proj_qkv.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.in_proj_z.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.in_proj_b.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.in_proj_a.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.mlp.gate_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.mlp.up_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.mlp.down_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.input_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.post_attention_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.dt_bias": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.A_log": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.conv1d.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.norm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.out_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.in_proj_qkv.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.in_proj_z.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.in_proj_b.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.in_proj_a.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.mlp.gate_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.mlp.up_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.mlp.down_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.input_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.post_attention_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.q_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.k_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.v_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.o_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.q_norm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.k_norm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.mlp.gate_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.mlp.up_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.mlp.down_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.input_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.post_attention_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.dt_bias": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.A_log": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.conv1d.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.norm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.out_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.in_proj_qkv.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.in_proj_z.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.in_proj_b.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.in_proj_a.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.mlp.gate_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.mlp.up_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.52.mlp.down_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.52.input_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.52.post_attention_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.dt_bias": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.A_log": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.conv1d.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.norm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.out_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.in_proj_qkv.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.in_proj_z.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.in_proj_b.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.in_proj_a.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.mlp.gate_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.mlp.up_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.mlp.down_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.input_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.post_attention_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.dt_bias": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.A_log": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.conv1d.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.norm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.out_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.in_proj_qkv.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.in_proj_z.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.in_proj_b.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.in_proj_a.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.mlp.gate_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.mlp.up_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.mlp.down_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.input_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.post_attention_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.q_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.k_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.v_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.o_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.q_norm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.k_norm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.mlp.gate_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.mlp.up_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.mlp.down_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.input_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.post_attention_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.dt_bias": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.A_log": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.conv1d.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.norm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.out_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.in_proj_qkv.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.in_proj_z.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.in_proj_b.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.in_proj_a.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.mlp.gate_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.mlp.up_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.mlp.down_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.input_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.post_attention_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.dt_bias": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.A_log": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.conv1d.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.norm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.out_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.in_proj_qkv.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.in_proj_z.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.in_proj_b.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.in_proj_a.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.mlp.gate_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.mlp.up_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.57.mlp.down_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.57.input_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.57.post_attention_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.dt_bias": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.A_log": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.conv1d.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.norm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.out_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.in_proj_qkv.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.in_proj_z.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.in_proj_b.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.in_proj_a.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.mlp.gate_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.mlp.up_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.mlp.down_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.input_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.post_attention_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.q_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.k_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.v_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.o_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.q_norm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.k_norm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.mlp.gate_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.mlp.up_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.mlp.down_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.input_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.post_attention_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.dt_bias": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.A_log": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.conv1d.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.norm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.out_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.in_proj_qkv.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.in_proj_z.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.in_proj_b.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.in_proj_a.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.mlp.gate_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.mlp.up_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.mlp.down_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.input_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.post_attention_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.dt_bias": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.A_log": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.conv1d.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.norm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.out_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.in_proj_qkv.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.in_proj_z.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.in_proj_b.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.in_proj_a.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.mlp.gate_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.mlp.up_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.mlp.down_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.input_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.post_attention_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.dt_bias": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.A_log": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.conv1d.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.norm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.out_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.in_proj_qkv.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.in_proj_z.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.in_proj_b.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.in_proj_a.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.mlp.gate_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.mlp.up_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.62.mlp.down_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.62.input_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.62.post_attention_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.q_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.k_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.v_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.o_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.q_norm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.k_norm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.mlp.gate_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.mlp.up_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.mlp.down_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.input_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.post_attention_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.dt_bias": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.A_log": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.conv1d.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.norm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.out_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.in_proj_qkv.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.in_proj_z.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.in_proj_b.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.in_proj_a.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.mlp.gate_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.mlp.up_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.mlp.down_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.input_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.post_attention_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.dt_bias": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.A_log": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.conv1d.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.norm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.out_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.in_proj_qkv.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.in_proj_z.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.in_proj_b.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.in_proj_a.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.mlp.gate_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.mlp.up_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.mlp.down_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.input_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.post_attention_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.dt_bias": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.A_log": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.conv1d.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.norm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.out_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.in_proj_qkv.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.in_proj_z.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.in_proj_b.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.in_proj_a.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.mlp.gate_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.mlp.up_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.mlp.down_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.input_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.post_attention_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.q_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.k_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.v_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.o_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.q_norm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.k_norm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.mlp.gate_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.mlp.up_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.67.mlp.down_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.67.input_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.67.post_attention_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.dt_bias": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.A_log": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.conv1d.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.norm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.out_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.in_proj_qkv.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.in_proj_z.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.in_proj_b.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.in_proj_a.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.mlp.gate_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.mlp.up_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.mlp.down_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.input_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.post_attention_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.dt_bias": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.A_log": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.conv1d.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.norm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.out_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.in_proj_qkv.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.in_proj_z.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.in_proj_b.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.in_proj_a.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.mlp.gate_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.mlp.up_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.mlp.down_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.input_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.post_attention_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.dt_bias": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.A_log": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.conv1d.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.norm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.out_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.in_proj_qkv.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.in_proj_z.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.in_proj_b.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.in_proj_a.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.mlp.gate_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.mlp.up_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.mlp.down_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.input_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.post_attention_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.q_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.k_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.v_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.o_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.q_norm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.k_norm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.mlp.gate_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.mlp.up_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.mlp.down_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.input_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.post_attention_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.dt_bias": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.A_log": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.conv1d.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.norm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.out_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.in_proj_qkv.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.in_proj_z.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.in_proj_b.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.in_proj_a.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.mlp.gate_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.mlp.up_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.72.mlp.down_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.72.input_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.72.post_attention_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.dt_bias": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.A_log": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.conv1d.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.norm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.out_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.in_proj_qkv.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.in_proj_z.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.in_proj_b.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.in_proj_a.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.mlp.gate_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.mlp.up_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.mlp.down_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.input_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.post_attention_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.dt_bias": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.A_log": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.conv1d.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.norm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.out_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.in_proj_qkv.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.in_proj_z.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.in_proj_b.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.in_proj_a.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.mlp.gate_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.mlp.up_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.mlp.down_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.input_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.post_attention_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.q_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.k_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.v_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.o_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.q_norm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.k_norm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.mlp.gate_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.mlp.up_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.mlp.down_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.input_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.post_attention_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.dt_bias": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.A_log": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.conv1d.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.norm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.out_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.in_proj_qkv.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.in_proj_z.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.in_proj_b.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.in_proj_a.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.mlp.gate_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.mlp.up_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.mlp.down_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.input_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.post_attention_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.dt_bias": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.A_log": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.conv1d.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.norm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.out_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.in_proj_qkv.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.in_proj_z.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.in_proj_b.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.in_proj_a.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.mlp.gate_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.mlp.up_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.77.mlp.down_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.77.input_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.77.post_attention_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.dt_bias": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.A_log": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.conv1d.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.norm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.out_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.in_proj_qkv.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.in_proj_z.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.in_proj_b.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.in_proj_a.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.mlp.gate_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.mlp.up_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.mlp.down_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.input_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.post_attention_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.q_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.k_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.v_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.o_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.q_norm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.k_norm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.mlp.gate_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.mlp.up_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.mlp.down_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.input_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.post_attention_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.dt_bias": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.A_log": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.conv1d.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.norm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.out_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.in_proj_qkv.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.in_proj_z.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.in_proj_b.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.in_proj_a.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.mlp.gate_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.mlp.up_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.mlp.down_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.input_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.post_attention_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.dt_bias": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.A_log": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.conv1d.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.norm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.out_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.in_proj_qkv.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.in_proj_z.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.in_proj_b.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.in_proj_a.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.mlp.gate_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.mlp.up_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.mlp.down_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.input_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.post_attention_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.dt_bias": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.A_log": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.conv1d.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.norm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.out_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.in_proj_qkv.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.in_proj_z.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.in_proj_b.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.in_proj_a.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.mlp.gate_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.mlp.up_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.82.mlp.down_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.82.input_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.82.post_attention_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.q_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.k_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.v_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.o_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.q_norm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.k_norm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.mlp.gate_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.mlp.up_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.mlp.down_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.input_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.post_attention_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.dt_bias": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.A_log": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.conv1d.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.norm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.out_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.in_proj_qkv.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.in_proj_z.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.in_proj_b.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.in_proj_a.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.mlp.gate_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.mlp.up_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.mlp.down_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.input_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.post_attention_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.dt_bias": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.A_log": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.conv1d.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.norm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.out_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.in_proj_qkv.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.in_proj_z.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.in_proj_b.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.in_proj_a.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.mlp.gate_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.mlp.up_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.mlp.down_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.input_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.post_attention_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.dt_bias": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.A_log": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.conv1d.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.norm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.out_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.in_proj_qkv.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.in_proj_z.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.in_proj_b.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.in_proj_a.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.mlp.gate_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.mlp.up_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.mlp.down_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.input_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.post_attention_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.q_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.k_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.v_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.o_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.q_norm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.k_norm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.mlp.gate_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.mlp.up_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.87.mlp.down_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.87.input_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.87.post_attention_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.dt_bias": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.A_log": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.conv1d.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.norm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.out_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.in_proj_qkv.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.in_proj_z.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.in_proj_b.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.in_proj_a.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.mlp.gate_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.mlp.up_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.mlp.down_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.input_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.post_attention_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.dt_bias": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.A_log": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.conv1d.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.norm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.out_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.in_proj_qkv.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.in_proj_z.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.in_proj_b.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.in_proj_a.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.mlp.gate_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.mlp.up_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.mlp.down_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.input_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.post_attention_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.dt_bias": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.A_log": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.conv1d.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.norm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.out_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.in_proj_qkv.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.in_proj_z.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.in_proj_b.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.in_proj_a.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.mlp.gate_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.mlp.up_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.mlp.down_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.input_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.post_attention_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.q_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.k_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.v_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.o_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.q_norm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.k_norm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.mlp.gate_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.mlp.up_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.mlp.down_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.input_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.post_attention_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.dt_bias": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.A_log": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.conv1d.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.norm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.out_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.in_proj_qkv.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.in_proj_z.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.in_proj_b.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.in_proj_a.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.mlp.gate_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.mlp.up_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.92.mlp.down_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.92.input_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.92.post_attention_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.dt_bias": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.A_log": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.conv1d.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.norm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.out_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.in_proj_qkv.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.in_proj_z.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.in_proj_b.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.in_proj_a.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.mlp.gate_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.mlp.up_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.mlp.down_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.input_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.post_attention_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.dt_bias": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.A_log": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.conv1d.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.norm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.out_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.in_proj_qkv.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.in_proj_z.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.in_proj_b.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.in_proj_a.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.mlp.gate_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.mlp.up_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.mlp.down_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.input_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.post_attention_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.q_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.k_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.v_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.o_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.q_norm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.k_norm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.mlp.gate_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.mlp.up_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.mlp.down_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.input_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.post_attention_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.dt_bias": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.A_log": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.conv1d.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.norm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.out_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.in_proj_qkv.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.in_proj_z.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.in_proj_b.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.in_proj_a.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.mlp.gate_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.mlp.up_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.mlp.down_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.input_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.post_attention_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.dt_bias": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.A_log": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.conv1d.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.norm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.out_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.in_proj_qkv.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.in_proj_z.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.in_proj_b.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.in_proj_a.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.mlp.gate_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.mlp.up_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.97.mlp.down_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.97.input_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.97.post_attention_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.dt_bias": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.A_log": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.conv1d.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.norm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.out_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.in_proj_qkv.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.in_proj_z.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.in_proj_b.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.in_proj_a.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.mlp.gate_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.mlp.up_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.mlp.down_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.input_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.post_attention_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.q_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.k_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.v_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.o_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.q_norm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.k_norm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.mlp.gate_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.mlp.up_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.mlp.down_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.input_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.post_attention_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.dt_bias": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.A_log": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.conv1d.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.norm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.out_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.in_proj_qkv.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.in_proj_z.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.in_proj_b.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.in_proj_a.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.mlp.gate_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.mlp.up_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.mlp.down_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.input_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.post_attention_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.dt_bias": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.A_log": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.conv1d.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.norm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.out_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.in_proj_qkv.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.in_proj_z.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.in_proj_b.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.in_proj_a.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.mlp.gate_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.mlp.up_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.mlp.down_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.input_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.post_attention_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.dt_bias": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.A_log": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.conv1d.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.norm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.out_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.in_proj_qkv.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.in_proj_z.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.in_proj_b.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.in_proj_a.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.mlp.gate_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.mlp.up_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.102.mlp.down_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.102.input_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.102.post_attention_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.q_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.k_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.v_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.o_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.q_norm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.k_norm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.mlp.gate_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.mlp.up_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.mlp.down_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.input_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.post_attention_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.dt_bias": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.A_log": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.conv1d.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.norm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.out_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.in_proj_qkv.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.in_proj_z.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.in_proj_b.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.in_proj_a.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.mlp.gate_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.mlp.up_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.mlp.down_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.input_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.post_attention_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.dt_bias": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.A_log": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.conv1d.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.norm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.out_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.in_proj_qkv.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.in_proj_z.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.in_proj_b.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.in_proj_a.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.mlp.gate_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.mlp.up_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.mlp.down_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.input_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.post_attention_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.dt_bias": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.A_log": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.conv1d.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.norm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.out_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.in_proj_qkv.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.in_proj_z.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.in_proj_b.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.in_proj_a.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.mlp.gate_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.mlp.up_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.mlp.down_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.input_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.post_attention_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.q_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.k_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.v_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.o_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.q_norm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.k_norm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.mlp.gate_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.mlp.up_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.107.mlp.down_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.107.input_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.107.post_attention_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.dt_bias": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.A_log": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.conv1d.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.norm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.out_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.in_proj_qkv.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.in_proj_z.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.in_proj_b.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.in_proj_a.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.mlp.gate_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.mlp.up_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.mlp.down_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.input_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.post_attention_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.dt_bias": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.A_log": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.conv1d.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.norm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.out_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.in_proj_qkv.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.in_proj_z.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.in_proj_b.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.in_proj_a.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.mlp.gate_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.mlp.up_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.mlp.down_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.input_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.post_attention_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.dt_bias": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.A_log": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.conv1d.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.norm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.out_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.in_proj_qkv.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.in_proj_z.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.in_proj_b.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.in_proj_a.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.mlp.gate_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.mlp.up_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.mlp.down_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.input_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.post_attention_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.q_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.k_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.v_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.o_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.q_norm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.k_norm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.mlp.gate_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.mlp.up_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.mlp.down_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.input_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.post_attention_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.dt_bias": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.A_log": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.conv1d.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.norm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.out_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.in_proj_qkv.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.in_proj_z.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.in_proj_b.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.in_proj_a.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.mlp.gate_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.mlp.up_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.112.mlp.down_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.112.input_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.112.post_attention_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.dt_bias": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.A_log": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.conv1d.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.norm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.out_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.in_proj_qkv.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.in_proj_z.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.in_proj_b.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.in_proj_a.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.mlp.gate_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.mlp.up_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.mlp.down_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.input_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.post_attention_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.dt_bias": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.A_log": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.conv1d.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.norm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.out_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.in_proj_qkv.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.in_proj_z.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.in_proj_b.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.in_proj_a.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.mlp.gate_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.mlp.up_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.mlp.down_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.input_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.post_attention_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.q_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.k_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.v_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.o_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.q_norm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.k_norm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.mlp.gate_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.mlp.up_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.mlp.down_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.input_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.post_attention_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.dt_bias": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.A_log": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.conv1d.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.norm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.out_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.in_proj_qkv.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.in_proj_z.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.in_proj_b.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.in_proj_a.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.mlp.gate_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.mlp.up_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.mlp.down_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.input_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.post_attention_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.dt_bias": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.A_log": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.conv1d.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.norm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.out_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.in_proj_qkv.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.in_proj_z.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.in_proj_b.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.in_proj_a.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.mlp.gate_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.mlp.up_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.117.mlp.down_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.117.input_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.117.post_attention_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.dt_bias": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.A_log": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.conv1d.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.norm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.out_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.in_proj_qkv.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.in_proj_z.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.in_proj_b.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.in_proj_a.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.mlp.gate_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.mlp.up_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.mlp.down_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.input_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.post_attention_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.q_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.k_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.v_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.o_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.q_norm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.k_norm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.mlp.gate_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.mlp.up_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.mlp.down_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.input_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.post_attention_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.dt_bias": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.A_log": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.conv1d.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.norm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.out_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.in_proj_qkv.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.in_proj_z.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.in_proj_b.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.in_proj_a.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.mlp.gate_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.mlp.up_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.mlp.down_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.input_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.post_attention_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.dt_bias": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.A_log": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.conv1d.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.norm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.out_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.in_proj_qkv.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.in_proj_z.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.in_proj_b.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.in_proj_a.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.mlp.gate_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.mlp.up_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.mlp.down_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.input_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.post_attention_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.dt_bias": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.A_log": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.conv1d.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.norm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.out_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.in_proj_qkv.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.in_proj_z.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.in_proj_b.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.in_proj_a.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.mlp.gate_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.mlp.up_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.122.mlp.down_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.122.input_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.122.post_attention_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.q_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.k_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.v_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.o_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.q_norm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.k_norm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.mlp.gate_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.mlp.up_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.mlp.down_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.input_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.post_attention_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.dt_bias": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.A_log": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.conv1d.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.norm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.out_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.in_proj_qkv.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.in_proj_z.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.in_proj_b.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.in_proj_a.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.mlp.gate_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.mlp.up_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.mlp.down_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.input_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.post_attention_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.dt_bias": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.A_log": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.conv1d.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.norm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.out_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.in_proj_qkv.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.in_proj_z.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.in_proj_b.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.in_proj_a.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.mlp.gate_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.mlp.up_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.mlp.down_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.input_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.post_attention_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.dt_bias": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.A_log": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.conv1d.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.norm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.out_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.in_proj_qkv.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.in_proj_z.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.in_proj_b.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.in_proj_a.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.mlp.gate_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.mlp.up_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.mlp.down_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.input_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.post_attention_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.q_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.k_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.v_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.o_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.q_norm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.k_norm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.mlp.gate_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.mlp.up_proj.weight": "model-00027-of-00027.safetensors",
+ "model.layers.127.mlp.down_proj.weight": "model-00027-of-00027.safetensors",
+ "model.layers.127.input_layernorm.weight": "model-00027-of-00027.safetensors",
+ "model.layers.127.post_attention_layernorm.weight": "model-00027-of-00027.safetensors",
+ "model.norm.weight": "model-00027-of-00027.safetensors",
+ "lm_head.weight": "model-00027-of-00027.safetensors",
+ "mtp.pre_fc_norm_embedding.weight": "model-00027-of-00027.safetensors",
+ "mtp.pre_fc_norm_hidden.weight": "model-00027-of-00027.safetensors",
+ "mtp.fc.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.self_attn.q_proj.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.self_attn.k_proj.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.self_attn.v_proj.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.self_attn.o_proj.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.self_attn.q_norm.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.self_attn.k_norm.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.mlp.gate_proj.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.mlp.up_proj.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.mlp.down_proj.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.input_layernorm.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.post_attention_layernorm.weight": "model-00027-of-00027.safetensors",
+ "mtp.norm.weight": "model-00027-of-00027.safetensors"
+ }
+}
\ No newline at end of file
diff --git a/sft_final/system_prompt.txt b/sft_final/system_prompt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d8a820acc8ba81380252f0545448846034adcdd2
--- /dev/null
+++ b/sft_final/system_prompt.txt
@@ -0,0 +1,6 @@
+You are JiRack, an assistant that answers immediately and to the point.
+Never use a reasoning block wrapped in think tags.
+Do not show internal reasoning, drafts, or step-by-step thought process.
+Give the final, ready answer right away, with no preliminary explanation of your reasoning.
+If you feel the need to think through the question, do it silently, internally,
+and output only the final answer to the user.
diff --git a/sft_final/tokenizer.json b/sft_final/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..adaf9dba01362c086cfcbbf51444ba709f7e0a2c
--- /dev/null
+++ b/sft_final/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7befaf450030f2fc58d7ea6b3941216ec6cccd85115bf7c2bd5d127f2632cff1
+size 20010936
diff --git a/sft_final/tokenizer_config.json b/sft_final/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..a453e3588da8daccea8f570c35872c97642c2a07
--- /dev/null
+++ b/sft_final/tokenizer_config.json
@@ -0,0 +1,148 @@
+{
+ "add_prefix_space": false,
+ "audio_bos_token": "<|audio_start|>",
+ "audio_eos_token": "<|audio_end|>",
+ "audio_token": "<|audio_pad|>",
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|image|>",
+ "<|video|>",
+ "<|sound|>",
+ "<|voice|>",
+ "<|listening|>",
+ "<|vision|>",
+ "<|mood_happy|>",
+ "<|mood_sad|>",
+ "<|mood_angry|>",
+ "<|mood_neutral|>",
+ "",
+ "",
+ "",
+ "<|action_start|>",
+ "<|action_end|>",
+ "<|trajectory_start|>",
+ "<|trajectory_end|>",
+ "<|joint_start|>",
+ "<|joint_end|>",
+ "<|sensor_start|>",
+ "<|sensor_end|>",
+ "<|command_start|>",
+ "<|command_end|>",
+ "<|state_start|>",
+ "<|state_end|>",
+ "<|pose|>",
+ "<|velocity|>",
+ "<|force|>",
+ "<|torque|>",
+ "<|gripper|>",
+ "<|navigation|>",
+ "<|obstacle|>",
+ "<|task_start|>",
+ "<|task_end|>",
+ "<|plan_start|>",
+ "<|plan_end|>",
+ "<|behavior_start|>",
+ "<|behavior_end|>",
+ "<|skill_start|>",
+ "<|skill_end|>",
+ "<|motor|>",
+ "<|servo|>",
+ "<|imu|>",
+ "<|lidar|>",
+ "<|camera|>",
+ "<|depth|>",
+ "<|waypoint|>",
+ "<|path|>",
+ "<|collision|>",
+ "<|grasp|>",
+ "<|release|>",
+ "<|homing|>",
+ "<|emergency_stop|>",
+ "<|calibration|>",
+ "<|manipulation|>",
+ "<|locomotion|>",
+ "<|feedback|>",
+ "<|control_loop|>",
+ "<|language|>",
+ "<|tool_call_start|>",
+ "<|tool_call_end|>",
+ "<|tool_result_start|>",
+ "<|tool_result_end|>",
+ "__SCIENCE__",
+ "__CODING__",
+ "__STOCK_EXCHANGE__",
+ "__MEDICINE__",
+ "__GOVERNMENT__",
+ "__NEWS__",
+ "__GENERAL__",
+ "__MATERIAL_SCIENCE__",
+ "__ELECTRONICS__",
+ "__MICROELECTRONICS__",
+ "__ENGINEERING__",
+ "__ROBOTICS__",
+ "__ENERGY__",
+ "__AUTOMOTIVE__",
+ "__AVIATION__",
+ "__MATH__",
+ "__PYTHON__",
+ "__C__",
+ "__CPP__",
+ "__C_SHARP__",
+ "__JAVA__",
+ "__JAVASCRIPT__",
+ "__TYPESCRIPT__",
+ "__RUST__",
+ "__GO__",
+ "__RUBY__",
+ "__PHP__",
+ "__SWIFT__",
+ "__KOTLIN__",
+ "__BASH__",
+ "__SQL__",
+ "__ASSEMBLY__",
+ "__PHILOSOPHY__",
+ "__LITERATURE__",
+ "__SOCIOLOGY__",
+ "__PSYCHOLOGY__",
+ "__POLITICAL_SCIENCE__",
+ "__CULTURAL_STUDIES__",
+ "__ETHNOGRAPHY__",
+ "__HUMAN_RIGHTS__",
+ "__COMPLIANCE__",
+ "__MILITARY__",
+ "__BANKING__",
+ "__OIL_INDUSTRY__",
+ "__LIGHT_INDUSTRY__",
+ "__NATURE__",
+ "__OCEAN__",
+ "__SPORT__",
+ "__CULINARY__",
+ "__TRAVEL__",
+ "__HOBBY__"
+ ],
+ "image_token": "<|image_pad|>",
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 262144,
+ "model_specific_special_tokens": {
+ "audio_bos_token": "<|audio_start|>",
+ "audio_eos_token": "<|audio_end|>",
+ "audio_token": "<|audio_pad|>",
+ "image_token": "<|image_pad|>",
+ "video_token": "<|video_pad|>",
+ "vision_bos_token": "<|vision_start|>",
+ "vision_eos_token": "<|vision_end|>"
+ },
+ "pad_token": "<|endoftext|>",
+ "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null,
+ "video_token": "<|video_pad|>",
+ "vision_bos_token": "<|vision_start|>",
+ "vision_eos_token": "<|vision_end|>"
+}
diff --git a/sft_final_fp8/.gitattributes b/sft_final_fp8/.gitattributes
new file mode 100644
index 0000000000000000000000000000000000000000..52373fe24473b1aa44333d318f578ae6bf04b49b
--- /dev/null
+++ b/sft_final_fp8/.gitattributes
@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/sft_final_fp8/JiRackDeltaNet_70b.py b/sft_final_fp8/JiRackDeltaNet_70b.py
new file mode 100644
index 0000000000000000000000000000000000000000..c099cd18bd19038eef7c5dd2e59d0897bcd7d962
--- /dev/null
+++ b/sft_final_fp8/JiRackDeltaNet_70b.py
@@ -0,0 +1,766 @@
+"""
+JiRackDeltaNet_27b.py
+=============================
+
+Собственная (hand-rolled) архитектура "JiRack Qwen3.8-27B" -- в стиле
+прежнего DS32B-проекта: один файл, свой TransformerBlock, никакого
+AutoConfig/AutoModelForCausalLM внутри класса архитектуры, никакого
+model_id в конструкторе. Все размерности зашиты константами ниже --
+подтверждены живым инспектом реального checkpoint'а (inspect_qwen38_keys.py,
+851 тензор, сверено).
+
+Математика DeltaNet-рекуррентности, свёртки, гейтинга и partial-rotary
+mRoPE-attention ПОРТИРОВАНА из реального modeling_qwen3_5.py (Apache 2.0,
+HuggingFace/Qwen), а не переизобретена -- чтобы не рисковать тихой
+числовой ошибкой в тонкой рекуррентной формуле. Портирован смысл, но
+код переписан под наши имена/стиль и урезан: без vision-башни, без
+KV-cache классов transformers (для тренировки полный форвард по
+последовательности не нужен; для генерации -- полный recompute на
+каждом шаге, без инкрементального кэша; это сознательное упрощение,
+при желании кэш можно добавить отдельно).
+
+Имена подмодулей (`linear_attn` / `self_attn`, `in_proj_qkv`, `out_proj`,
+`mlp.gate_proj` и т.д.) намеренно СОВПАДАЮТ с именами в реальном
+state_dict Qwen3_5ForCausalLM -- поэтому веса из HF-чекпоинта грузятся
+напрямую через load_state_dict(), без ручного key-mapping.
+
+Файл умеет:
+ - собрать модель (JiRackQwen38ForCausalLM) с нуля, случайные веса;
+ - принять state_dict в формате HF Qwen3_5ForCausalLM (после
+ AutoModelForCausalLM.from_pretrained(...).state_dict() в конвертере)
+ и загрузить его напрямую;
+ - пропатчить нужные nn.Linear -> BitLinear (TERNARIZE_PATTERNS);
+ - сохранить/загрузить свой JiRack-чекпоинт (save_checkpoint/load_checkpoint).
+
+Конвертер (convert_qwen38_to_jirack.py) по-прежнему знает про model_id
+и HF Hub -- но это знание живёт ТОЛЬКО в конвертере, не в архитектуре.
+"""
+
+import os
+import re
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+# =====================================================================
+# 1. CONFIRMED АРХИТЕКТУРА -- жёстко зашитые константы (не AutoConfig)
+# =====================================================================
+
+HIDDEN_SIZE = 5120
+VOCAB_SIZE = 248320
+INTERMEDIATE_SIZE = 26624
+NUM_LAYERS = 128
+RMS_NORM_EPS = 1e-6
+
+# Чередование слоёв: full_attention_interval = 4 ->
+# 3x linear_attention, 1x full_attention, повторяется на все 64 слоя.
+LAYER_TYPES = ["linear_attention"] * 3 + ["full_attention"]
+LAYER_TYPES = (LAYER_TYPES * (NUM_LAYERS // len(LAYER_TYPES) + 1))[:NUM_LAYERS]
+
+# --- Gated DeltaNet (linear_attention) ---
+LINEAR_NUM_KEY_HEADS = 16
+LINEAR_KEY_HEAD_DIM = 128
+LINEAR_NUM_VALUE_HEADS = 48
+LINEAR_VALUE_HEAD_DIM = 128
+LINEAR_CONV_KERNEL_DIM = 4
+DELTA_KEY_DIM = LINEAR_NUM_KEY_HEADS * LINEAR_KEY_HEAD_DIM # 2048
+DELTA_VALUE_DIM = LINEAR_NUM_VALUE_HEADS * LINEAR_VALUE_HEAD_DIM # 6144
+DELTA_CONV_DIM = DELTA_KEY_DIM * 2 + DELTA_VALUE_DIM # 10240
+DELTA_CHUNK_SIZE = 64
+
+# --- Gated Attention (full_attention) ---
+NUM_ATTENTION_HEADS = 24
+NUM_KEY_VALUE_HEADS = 4
+HEAD_DIM = 256
+PARTIAL_ROTARY_FACTOR = 0.25
+ROPE_THETA = 1e7
+MROPE_SECTION = [11, 11, 10]
+ATTENTION_BIAS = False
+
+ARCH_SUMMARY = f"""
+Qwen3.8-27B (qwen3_5), hand-rolled JiRack TransformerBlock, text-only.
+hidden_size={HIDDEN_SIZE}, vocab_size={VOCAB_SIZE},
+intermediate_size={INTERMEDIATE_SIZE}, num_hidden_layers={NUM_LAYERS}
+Чередование (interval=4): 3x linear_attention (DeltaNet), 1x full_attention
+"""
+
+KEPT_FULL_PRECISION = """
+Всегда остаются full precision (не GEMM, либо слишком чувствительны):
+ - dt_bias, A_log (сырые Parameters, decay DeltaNet)
+ - linear_attn.in_proj_b/in_proj_a (по умолчанию, см. GATE_PROJ_PATTERNS)
+ - conv1d (depthwise grouped conv, не GEMM)
+ - все нормы: RMSNormGated, q_norm/k_norm, model.norm
+ - embed_tokens, lm_head
+ - mtp.* (вся MTP/NextN-голова) (draft-слой спекулятивного декода;
+ тернаризация убивает acceptance rate,
+ комьюнити-GGUF держат её в Q8_0)
+"""
+
+
+# =====================================================================
+# 2. BitLinear -- наш тернарный слой
+# =====================================================================
+
+class BitLinear(nn.Linear):
+ """b1.58-style per-tensor absmean ternary weights, per-token int8
+ activations, непрерывный lambda-warmup (STE)."""
+
+ def __init__(self, in_features, out_features, bias=False):
+ super().__init__(in_features, out_features, bias=bias)
+ self.eps = 1e-5
+ self.register_buffer("lambda_", torch.zeros(()), persistent=True)
+
+ @classmethod
+ def from_linear(cls, lin: nn.Linear) -> "BitLinear":
+ bl = cls(lin.in_features, lin.out_features, bias=lin.bias is not None)
+ bl = bl.to(dtype=lin.weight.dtype, device=lin.weight.device)
+ with torch.no_grad():
+ bl.weight.copy_(lin.weight)
+ if lin.bias is not None:
+ bl.bias.copy_(lin.bias)
+ return bl
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ if not self.training and float(self.lambda_) < 1e-6:
+ return F.linear(x, self.weight, self.bias)
+
+ lam = self.lambda_.to(x.dtype)
+ w = self.weight
+ gamma = w.float().abs().mean().clamp(min=self.eps).to(w.dtype)
+ w_quant = torch.clamp(torch.round(w / gamma), -1.0, 1.0) * gamma
+ w_effective = w + lam * (w_quant - w).detach()
+
+ x_scale = 127.0 / x.abs().max(dim=-1, keepdim=True).values.clamp(min=self.eps)
+ x_quant = torch.clamp(torch.round(x * x_scale), -128.0, 127.0) / x_scale
+ x_effective = x + lam * (x_quant - x).detach()
+
+ return F.linear(x_effective, w_effective, self.bias)
+
+ def set_lambda(self, v: float):
+ self.lambda_.fill_(v)
+
+
+# =====================================================================
+# 3. Карта: что тернаризуем, что оставляем full precision
+# =====================================================================
+
+TERNARIZE_PATTERNS = [
+ r"linear_attn\.in_proj_qkv$",
+ r"linear_attn\.in_proj_z$",
+ r"linear_attn\.out_proj$",
+ r"self_attn\.q_proj$",
+ r"self_attn\.k_proj$",
+ r"self_attn\.v_proj$",
+ r"self_attn\.o_proj$",
+ r"mlp\.gate_proj$",
+ r"mlp\.up_proj$",
+ r"mlp\.down_proj$",
+]
+
+GATE_PROJ_PATTERNS = [
+ r"linear_attn\.in_proj_b$",
+ r"linear_attn\.in_proj_a$",
+]
+
+
+def apply_bitlinear_patch(model: nn.Module, include_gate_proj: bool = False):
+ """Точечная замена nn.Linear -> BitLinear по TERNARIZE_PATTERNS."""
+ patterns = list(TERNARIZE_PATTERNS)
+ if include_gate_proj:
+ patterns += GATE_PROJ_PATTERNS
+ compiled = [re.compile(p) for p in patterns]
+
+ patched = []
+ for name, module in list(model.named_modules()):
+ if not isinstance(module, nn.Linear):
+ continue
+ # MTP/NextN-голова НЕ тернаризуется -- всегда full precision.
+ # Суффиксные паттерны (self_attn.q_proj$ и т.п.) иначе зацепили бы
+ # mtp.layers.0.self_attn.q_proj.
+ if name.startswith("mtp"):
+ continue
+ if any(p.search(name) for p in compiled):
+ parent_name, _, child_name = name.rpartition(".")
+ parent = model.get_submodule(parent_name) if parent_name else model
+ setattr(parent, child_name, BitLinear.from_linear(module))
+ patched.append(name)
+ return patched
+
+
+# =====================================================================
+# 4. Общие блоки (порт математики из реального modeling_qwen3_5.py)
+# =====================================================================
+
+class RMSNorm(nn.Module):
+ """(x * w).to(dtype), w хранится как (weight) с offset +1 -- как в
+ оригинале Qwen3_5RMSNorm (инициализация нулями, эффективный вес 1+w)."""
+
+ def __init__(self, dim: int, eps: float = RMS_NORM_EPS):
+ super().__init__()
+ self.eps = eps
+ self.weight = nn.Parameter(torch.zeros(dim))
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ dtype = x.dtype
+ x = x.float()
+ x = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+ x = x * (1.0 + self.weight.float())
+ return x.to(dtype)
+
+
+class RMSNormGated(nn.Module):
+ """RMSNorm с гейтингом через SiLU(gate), применяется ПЕРЕД гейтингом
+ (норма считается по hidden_states, гейт применяется после)."""
+
+ def __init__(self, dim: int, eps: float = RMS_NORM_EPS):
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(dim))
+ self.eps = eps
+
+ def forward(self, x: torch.Tensor, gate: torch.Tensor) -> torch.Tensor:
+ dtype = x.dtype
+ x = x.float()
+ var = x.pow(2).mean(-1, keepdim=True)
+ x = x * torch.rsqrt(var + self.eps)
+ x = self.weight * x.to(dtype)
+ x = x * F.silu(gate.float())
+ return x.to(dtype)
+
+
+def rotate_half(x: torch.Tensor) -> torch.Tensor:
+ x1 = x[..., : x.shape[-1] // 2]
+ x2 = x[..., x.shape[-1] // 2:]
+ return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
+ """Partial rotary: только первые cos.shape[-1] каналов вращаются,
+ остальное (q_pass/k_pass) проходит без изменений."""
+ cos = cos.unsqueeze(unsqueeze_dim)
+ sin = sin.unsqueeze(unsqueeze_dim)
+ rotary_dim = cos.shape[-1]
+ q_rot, q_pass = q[..., :rotary_dim], q[..., rotary_dim:]
+ k_rot, k_pass = k[..., :rotary_dim], k[..., rotary_dim:]
+ q_embed = torch.cat([(q_rot * cos) + (rotate_half(q_rot) * sin), q_pass], dim=-1)
+ k_embed = torch.cat([(k_rot * cos) + (rotate_half(k_rot) * sin), k_pass], dim=-1)
+ return q_embed, k_embed
+
+
+class TextRotaryEmbedding(nn.Module):
+ """mRoPE с partial_rotary_factor. Для чистого текста (без vision)
+ 3 позиционные "оси" (T,H,W) совпадают -> interleaved mRoPE
+ вырождается в обычный RoPE, но мы всё равно считаем честно тем же
+ путём, что и оригинал, для полной числовой идентичности."""
+
+ def __init__(self, head_dim=HEAD_DIM, partial_rotary_factor=PARTIAL_ROTARY_FACTOR,
+ theta=ROPE_THETA, mrope_section=MROPE_SECTION):
+ super().__init__()
+ self.rotary_dim = int(head_dim * partial_rotary_factor)
+ inv_freq = 1.0 / (theta ** (torch.arange(0, self.rotary_dim, 2, dtype=torch.float32) / self.rotary_dim))
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
+ self.mrope_section = mrope_section
+
+ def _apply_interleaved_mrope(self, freqs: torch.Tensor) -> torch.Tensor:
+ # freqs: (3, bs, seq, rotary_dim // 2)
+ freqs_t = freqs[0].clone()
+ for dim, offset in enumerate((1, 2), start=1):
+ length = self.mrope_section[dim] * 3
+ idx = slice(offset, length, 3)
+ freqs_t[..., idx] = freqs[dim, ..., idx]
+ return freqs_t
+
+ @torch.no_grad()
+ def forward(self, x: torch.Tensor, position_ids: torch.LongTensor):
+ # position_ids: (bs, seq) -> реплицируем в 3 идентичные оси (text-only)
+ if position_ids.ndim == 2:
+ position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)
+ inv_freq_exp = self.inv_freq[None, None, :, None].float().expand(3, position_ids.shape[1], -1, 1)
+ pos_exp = position_ids[:, :, None, :].float()
+ freqs = (inv_freq_exp @ pos_exp).transpose(2, 3) # (3, bs, seq, rotary_dim//2)
+ freqs = self._apply_interleaved_mrope(freqs) # (bs, seq, rotary_dim//2)
+ emb = torch.cat((freqs, freqs), dim=-1) # (bs, seq, rotary_dim)
+ return emb.cos().to(x.dtype), emb.sin().to(x.dtype)
+
+
+def l2norm(x: torch.Tensor, dim: int = -1, eps: float = 1e-6):
+ return x * torch.rsqrt((x * x).sum(dim=dim, keepdim=True) + eps)
+
+
+def causal_conv1d(hidden_states: torch.Tensor, weight: torch.Tensor, activation: str = "silu"):
+ """hidden_states: (batch, channels, seq_len). Depthwise causal conv,
+ left-padded by kernel_size - 1, затем активация."""
+ channels, seq_len = hidden_states.shape[1], hidden_states.shape[2]
+ padding = weight.shape[-1] - 1
+ out = F.conv1d(hidden_states, weight=weight.unsqueeze(1), bias=None, padding=padding, groups=channels)
+ out = out[:, :, :seq_len]
+ return F.silu(out) if activation == "silu" else out
+
+
+def chunked_gated_delta_rule(query, key, value, g, beta, chunk_size=DELTA_CHUNK_SIZE):
+ """Порт torch_chunk_gated_delta_rule из modeling_qwen3_5.py -- без
+ initial_state/output_final_state (тренировка/полный форвард без
+ инкрементального кэша). query/key/value: (b, h, t, d), g/beta: (b, h, t)."""
+ initial_dtype = query.dtype
+ query = l2norm(query, dim=-1, eps=1e-6)
+ key = l2norm(key, dim=-1, eps=1e-6)
+ query, key, value, beta, g = [x.contiguous().to(torch.float32) for x in (query, key, value, beta, g)]
+
+ b, h, t, k_dim = key.shape
+ v_dim = value.shape[-1]
+ pad = (chunk_size - t % chunk_size) % chunk_size
+ query = F.pad(query, (0, 0, 0, pad))
+ key = F.pad(key, (0, 0, 0, pad))
+ value = F.pad(value, (0, 0, 0, pad))
+ beta = F.pad(beta, (0, pad))
+ g = F.pad(g, (0, pad))
+ total_t = t + pad
+ scale = 1 / (query.shape[-1] ** 0.5)
+ query = query * scale
+
+ v_beta = value * beta.unsqueeze(-1)
+ k_beta = key * beta.unsqueeze(-1)
+ query, key, value, k_beta, v_beta = [
+ x.reshape(x.shape[0], x.shape[1], -1, chunk_size, x.shape[-1]) for x in (query, key, value, k_beta, v_beta)
+ ]
+ g = g.reshape(g.shape[0], g.shape[1], -1, chunk_size)
+ mask = torch.triu(torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=0)
+
+ g = g.cumsum(dim=-1)
+ decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril()
+ attn = -((k_beta @ key.transpose(-1, -2)) * decay_mask).masked_fill(mask, 0)
+ for i in range(1, chunk_size):
+ row = attn[..., i, :i].clone()
+ sub = attn[..., :i, :i].clone()
+ attn[..., i, :i] = row + (row.unsqueeze(-1) * sub).sum(-2)
+ attn = attn + torch.eye(chunk_size, dtype=attn.dtype, device=attn.device)
+ value = attn @ v_beta
+ k_cumdecay = attn @ (k_beta * g.exp().unsqueeze(-1))
+
+ last_state = torch.zeros(b, h, k_dim, v_dim, dtype=value.dtype, device=value.device)
+ core_out = torch.zeros_like(value)
+
+ for i in range(total_t // chunk_size):
+ q_i, k_i, v_i = query[:, :, i], key[:, :, i], value[:, :, i]
+ attn_i = q_i @ k_i.transpose(-1, -2) * decay_mask[:, :, i]
+ v_prime = k_cumdecay[:, :, i] @ last_state
+ v_new = v_i - v_prime
+ attn_inter = (q_i * g[:, :, i, :, None].exp()) @ last_state
+ core_out[:, :, i] = attn_inter + attn_i @ v_new
+ last_state = (
+ last_state * g[:, :, i, -1, None, None].exp()
+ + (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2) @ v_new
+ )
+
+ core_out = core_out.reshape(core_out.shape[0], core_out.shape[1], -1, core_out.shape[-1])
+ core_out = core_out[:, :, :t]
+ core_out = core_out.transpose(1, 2).contiguous().to(initial_dtype)
+ return core_out
+
+
+# =====================================================================
+# 5. Наш собственный TransformerBlock (и подмодули linear_attn/self_attn)
+# =====================================================================
+
+class GatedDeltaNetBlock(nn.Module):
+ """linear_attention слой (Gated DeltaNet). Имена подмодулей совпадают
+ с HF state_dict, чтобы веса грузились напрямую."""
+
+ def __init__(self):
+ super().__init__()
+ self.conv1d = nn.Conv1d(
+ in_channels=DELTA_CONV_DIM, out_channels=DELTA_CONV_DIM, bias=False,
+ kernel_size=LINEAR_CONV_KERNEL_DIM, groups=DELTA_CONV_DIM,
+ padding=LINEAR_CONV_KERNEL_DIM - 1,
+ )
+ self.dt_bias = nn.Parameter(torch.ones(LINEAR_NUM_VALUE_HEADS))
+ A = torch.empty(LINEAR_NUM_VALUE_HEADS).uniform_(0.01, 16)
+ self.A_log = nn.Parameter(torch.log(A))
+ self.norm = RMSNormGated(LINEAR_VALUE_HEAD_DIM, eps=RMS_NORM_EPS)
+ self.out_proj = nn.Linear(DELTA_VALUE_DIM, HIDDEN_SIZE, bias=False)
+ self.in_proj_qkv = nn.Linear(HIDDEN_SIZE, DELTA_KEY_DIM * 2 + DELTA_VALUE_DIM, bias=False)
+ self.in_proj_z = nn.Linear(HIDDEN_SIZE, DELTA_VALUE_DIM, bias=False)
+ self.in_proj_b = nn.Linear(HIDDEN_SIZE, LINEAR_NUM_VALUE_HEADS, bias=False)
+ self.in_proj_a = nn.Linear(HIDDEN_SIZE, LINEAR_NUM_VALUE_HEADS, bias=False)
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ bsz, seq_len, _ = hidden_states.shape
+
+ mixed_qkv = self.in_proj_qkv(hidden_states).transpose(1, 2) # (b, conv_dim, t)
+ z = self.in_proj_z(hidden_states).reshape(bsz, seq_len, -1, LINEAR_VALUE_HEAD_DIM)
+ b = self.in_proj_b(hidden_states)
+ a = self.in_proj_a(hidden_states)
+
+ mixed_qkv = causal_conv1d(mixed_qkv, self.conv1d.weight.squeeze(1), activation="silu")
+ mixed_qkv = mixed_qkv.transpose(1, 2) # (b, t, conv_dim)
+
+ query, key, value = torch.split(mixed_qkv, [DELTA_KEY_DIM, DELTA_KEY_DIM, DELTA_VALUE_DIM], dim=-1)
+ query = query.reshape(bsz, seq_len, -1, LINEAR_KEY_HEAD_DIM)
+ key = key.reshape(bsz, seq_len, -1, LINEAR_KEY_HEAD_DIM)
+ value = value.reshape(bsz, seq_len, -1, LINEAR_VALUE_HEAD_DIM)
+
+ beta = b.sigmoid()
+ g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
+
+ rep = LINEAR_NUM_VALUE_HEADS // LINEAR_NUM_KEY_HEADS
+ if rep > 1:
+ query = query.repeat_interleave(rep, dim=2)
+ key = key.repeat_interleave(rep, dim=2)
+
+ # -> (b, h, t, d) for the chunked kernel
+ query_h = query.transpose(1, 2)
+ key_h = key.transpose(1, 2)
+ value_h = value.transpose(1, 2)
+ beta_h = beta.transpose(1, 2)
+ g_h = g.transpose(1, 2)
+
+ core_attn_out = chunked_gated_delta_rule(query_h, key_h, value_h, g_h, beta_h)
+
+ core_attn_out = core_attn_out.reshape(-1, LINEAR_VALUE_HEAD_DIM)
+ z_flat = z.reshape(-1, LINEAR_VALUE_HEAD_DIM)
+ core_attn_out = self.norm(core_attn_out, z_flat)
+ core_attn_out = core_attn_out.reshape(bsz, seq_len, -1)
+
+ return self.out_proj(core_attn_out)
+
+
+class AttentionBlock(nn.Module):
+ """full_attention слой (Gated Attention + partial rotary mRoPE)."""
+
+ def __init__(self):
+ super().__init__()
+ self.num_key_value_groups = NUM_ATTENTION_HEADS // NUM_KEY_VALUE_HEADS
+ self.scaling = HEAD_DIM ** -0.5
+ self.q_proj = nn.Linear(HIDDEN_SIZE, NUM_ATTENTION_HEADS * HEAD_DIM * 2, bias=ATTENTION_BIAS)
+ self.k_proj = nn.Linear(HIDDEN_SIZE, NUM_KEY_VALUE_HEADS * HEAD_DIM, bias=ATTENTION_BIAS)
+ self.v_proj = nn.Linear(HIDDEN_SIZE, NUM_KEY_VALUE_HEADS * HEAD_DIM, bias=ATTENTION_BIAS)
+ self.o_proj = nn.Linear(NUM_ATTENTION_HEADS * HEAD_DIM, HIDDEN_SIZE, bias=ATTENTION_BIAS)
+ self.q_norm = RMSNorm(HEAD_DIM, eps=RMS_NORM_EPS)
+ self.k_norm = RMSNorm(HEAD_DIM, eps=RMS_NORM_EPS)
+
+ def forward(self, hidden_states, cos, sin, causal_mask):
+ bsz, seq_len, _ = hidden_states.shape
+ hidden_shape = (bsz, seq_len, -1, HEAD_DIM)
+
+ query_states, gate = torch.chunk(
+ self.q_proj(hidden_states).view(bsz, seq_len, -1, HEAD_DIM * 2), 2, dim=-1
+ )
+ gate = gate.reshape(bsz, seq_len, -1)
+
+ query_states = self.q_norm(query_states.view(hidden_shape)).transpose(1, 2)
+ key_states = self.k_norm(self.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
+ value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
+
+ rep = self.num_key_value_groups
+ key_states = key_states.repeat_interleave(rep, dim=1)
+ value_states = value_states.repeat_interleave(rep, dim=1)
+
+ attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) * self.scaling
+ attn_weights = attn_weights + causal_mask
+ attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+ attn_output = torch.matmul(attn_weights, value_states)
+ attn_output = attn_output.transpose(1, 2).contiguous().reshape(bsz, seq_len, -1)
+ attn_output = attn_output * torch.sigmoid(gate)
+
+ return self.o_proj(attn_output)
+
+
+class MLPBlock(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate_proj = nn.Linear(HIDDEN_SIZE, INTERMEDIATE_SIZE, bias=False)
+ self.up_proj = nn.Linear(HIDDEN_SIZE, INTERMEDIATE_SIZE, bias=False)
+ self.down_proj = nn.Linear(INTERMEDIATE_SIZE, HIDDEN_SIZE, bias=False)
+
+ def forward(self, x):
+ return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
+
+
+class TransformerBlock(nn.Module):
+ """Наш decoder layer -- linear_attn ИЛИ self_attn, в зависимости от
+ layer_type, + общий MLP. Имена атрибутов совпадают с HF state_dict."""
+
+ def __init__(self, layer_idx: int, layer_type: str = None):
+ super().__init__()
+ # layer_type задан явно -> используем его (нужно для MTP-слоя,
+ # который всегда full_attention и лежит ВНЕ LAYER_TYPES).
+ self.block_type = layer_type if layer_type is not None else LAYER_TYPES[layer_idx]
+ if self.block_type == "linear_attention":
+ self.linear_attn = GatedDeltaNetBlock()
+ else:
+ self.self_attn = AttentionBlock()
+ self.mlp = MLPBlock()
+ self.input_layernorm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+ self.post_attention_layernorm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+
+ def forward(self, hidden_states, cos, sin, causal_mask):
+ residual = hidden_states
+ hidden_states = self.input_layernorm(hidden_states)
+ if self.block_type == "linear_attention":
+ hidden_states = self.linear_attn(hidden_states)
+ else:
+ hidden_states = self.self_attn(hidden_states, cos, sin, causal_mask)
+ hidden_states = residual + hidden_states
+
+ residual = hidden_states
+ hidden_states = self.post_attention_layernorm(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + hidden_states
+ return hidden_states
+
+
+# =====================================================================
+# 5b. MTP / NextN-голова (draft-слой спекулятивного декода)
+# =====================================================================
+
+class MTPBlock(nn.Module):
+ """Multi-Token Prediction голова -- ровно как в реальном чекпоинте
+ Qwen3.5/3.8 (mtp_num_hidden_layers=1, mtp_use_dedicated_embeddings=false).
+
+ Имена подмодулей СОВПАДАЮТ с ключами HF state_dict (top-level `mtp.`):
+ mtp.pre_fc_norm_embedding.weight
+ mtp.pre_fc_norm_hidden.weight
+ mtp.fc.weight [HIDDEN, 2*HIDDEN]
+ mtp.layers.0.* (обычный full_attention decoder)
+ mtp.norm.weight (final norm перед общим lm_head)
+
+ embed_tokens и lm_head ОБЩИЕ со стволом (dedicated embeddings = false),
+ здесь их нет. В GGUF это blk.{NUM_LAYERS}.nextn.eh_proj /
+ shared_head_norm + тензоры draft-слоя.
+
+ Механика (DeepSeek-style, порядок как в vLLM Qwen3NextMTP):
+ h = fc(cat(pre_fc_norm_embedding(emb_next), pre_fc_norm_hidden(h_prev)))
+ h = layers[0](h) # full attention
+ logits2 = lm_head(norm(h))
+ """
+
+ def __init__(self):
+ super().__init__()
+ self.pre_fc_norm_embedding = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+ self.pre_fc_norm_hidden = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+ self.fc = nn.Linear(2 * HIDDEN_SIZE, HIDDEN_SIZE, bias=False)
+ self.layers = nn.ModuleList([
+ TransformerBlock(layer_idx=NUM_LAYERS, layer_type="full_attention"),
+ ])
+ self.norm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+
+ def forward(self, inputs_embeds, previous_hidden, cos, sin, causal_mask):
+ """inputs_embeds -- embed_tokens(токены, сдвинутые на +1)
+ previous_hidden -- hidden ствола ДО финального model.norm
+ (см. контракт: MTP сам применяет pre_fc_norm_hidden)."""
+ h = self.fc(torch.cat(
+ (self.pre_fc_norm_embedding(inputs_embeds),
+ self.pre_fc_norm_hidden(previous_hidden)),
+ dim=-1,
+ ))
+ h = self.layers[0](h, cos, sin, causal_mask)
+ return self.norm(h)
+
+
+# =====================================================================
+# 6. Полная модель
+# =====================================================================
+
+class JiRackQwen38Model(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.embed_tokens = nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE)
+ self.layers = nn.ModuleList([TransformerBlock(i) for i in range(NUM_LAYERS)])
+ self.norm = RMSNorm(HIDDEN_SIZE, eps=RMS_NORM_EPS)
+ self.rotary_emb = TextRotaryEmbedding()
+
+ def forward(self, input_ids: torch.LongTensor, return_prenorm: bool = False):
+ bsz, seq_len = input_ids.shape
+ hidden_states = self.embed_tokens(input_ids)
+
+ position_ids = torch.arange(seq_len, device=input_ids.device).unsqueeze(0).expand(bsz, -1)
+ cos, sin = self.rotary_emb(hidden_states, position_ids)
+
+ causal_mask = torch.full((seq_len, seq_len), float("-inf"), device=input_ids.device, dtype=torch.float32)
+ causal_mask = torch.triu(causal_mask, diagonal=1)[None, None, :, :].to(hidden_states.dtype)
+
+ for layer in self.layers:
+ hidden_states = layer(hidden_states, cos, sin, causal_mask)
+
+ if return_prenorm:
+ # Для MTP: голова сама применяет pre_fc_norm_hidden, финальный
+ # model.norm сюда НЕ входит (иначе двойная нормализация).
+ return self.norm(hidden_states), hidden_states, (cos, sin, causal_mask)
+ return self.norm(hidden_states)
+
+
+class JiRackQwen38ForCausalLM(nn.Module):
+ """Верхнеуровневая модель. Имена атрибутов (`model`, `lm_head`)
+ совпадают с HF Qwen3_5ForCausalLM -- state_dict грузится напрямую."""
+
+ def __init__(self):
+ super().__init__()
+ self.model = JiRackQwen38Model()
+ self.lm_head = nn.Linear(HIDDEN_SIZE, VOCAB_SIZE, bias=False)
+ # MTP/NextN-голова: top-level атрибут `mtp` -- ключи state_dict
+ # получаются ровно mtp.*, как в реальном HF-чекпоинте.
+ self.mtp = MTPBlock()
+
+ def forward(self, input_ids: torch.LongTensor) -> torch.Tensor:
+ hidden_states = self.model(input_ids)
+ return self.lm_head(hidden_states)
+
+ def mtp_forward(self, input_ids: torch.LongTensor):
+ """Возвращает (logits_t1, logits_t2): обычные логиты ствола и
+ draft-логиты MTP-головы (предсказание токена t+2 из позиции t).
+ Для инференса/валидации MTP; в GGUF-конвертации не участвует."""
+ _, prenorm_hidden, (cos, sin, causal_mask) = self.model(
+ input_ids, return_prenorm=True)
+ logits_t1 = self.lm_head(self.model.norm(prenorm_hidden))
+
+ # Вход MTP -- эмбеддинги токенов, сдвинутых на +1 (следующий токен
+ # уже известен на шаге верификации). Последнюю позицию дублируем,
+ # чтобы сохранить форму (draft для неё не используется).
+ shifted = torch.cat([input_ids[:, 1:], input_ids[:, -1:]], dim=1)
+ emb_next = self.model.embed_tokens(shifted)
+ h2 = self.mtp(emb_next, prenorm_hidden, cos, sin, causal_mask)
+ logits_t2 = self.lm_head(h2)
+ return logits_t1, logits_t2
+
+ # -----------------------------------------------------------------
+ # Загрузка реальных весов (state_dict уже скачан конвертером через
+ # HF Hub -- сюда просто передаётся словарь тензоров)
+ # -----------------------------------------------------------------
+ def load_hf_state_dict(self, state_dict: dict, strict: bool = False):
+ """Грузит state_dict в формате оригинального Qwen3_5ForCausalLM.
+ mtp.* ключи теперь ГРУЗЯТСЯ в self.mtp (MTPBlock) -- фильтра
+ больше нет. strict=False оставлен из-за служебных lambda_-буферов
+ BitLinear, которых нет в реальном чекпоинте."""
+ missing, unexpected = self.load_state_dict(state_dict, strict=strict)
+ real_missing = [k for k in missing if not k.endswith("lambda_")]
+ if real_missing:
+ print(f"⚠️ {len(real_missing)} missing keys (первые 10): {real_missing[:10]}")
+ if unexpected:
+ print(f"⚠️ {len(unexpected)} unexpected keys (первые 10): {unexpected[:10]}")
+ return missing, unexpected
+
+ # -----------------------------------------------------------------
+ # Подсадка РЕАЛЬНЫХ mtp.* весов из оригинального HF-чекпоинта.
+ # Нужна, потому что старые JiRack .pt сохранены ДО появления MTPBlock
+ # и mtp-весов не содержат (там была бы случайная инициализация).
+ # -----------------------------------------------------------------
+ def load_mtp_from_hf_dir(self, hf_dir: str):
+ """Читает из папки оригинального HF-чекпоинта ТОЛЬКО mtp.* тензоры
+ (по index.json или перебором *.safetensors) и грузит их в self.mtp.
+ Возвращает число загруженных тензоров."""
+ import glob
+ import json as _json
+ from safetensors import safe_open
+
+ index_path = os.path.join(hf_dir, "model.safetensors.index.json")
+ mtp_map = {} # key -> shard file
+ if os.path.exists(index_path):
+ with open(index_path) as f:
+ weight_map = _json.load(f)["weight_map"]
+ for k, shard in weight_map.items():
+ if k.startswith("mtp."):
+ mtp_map[k] = os.path.join(hf_dir, shard)
+ else:
+ for shard in glob.glob(os.path.join(hf_dir, "*.safetensors")):
+ with safe_open(shard, framework="pt", device="cpu") as f:
+ for k in f.keys():
+ if k.startswith("mtp."):
+ mtp_map[k] = shard
+
+ if not mtp_map:
+ raise FileNotFoundError(
+ f"В {hf_dir} не найдено ни одного mtp.* тензора. Нужен "
+ f"ОРИГИНАЛЬНЫЙ чекпоинт Qwen3.8-27B (не JiRack-экспорт!). "
+ f"Если оригинальные шарды удалены -- перекачай с HF Hub "
+ f"только шарды с mtp.* по model.safetensors.index.json.")
+
+ own_dtype = next(self.mtp.parameters()).dtype
+ loaded = {}
+ by_shard = {}
+ for k, shard in mtp_map.items():
+ by_shard.setdefault(shard, []).append(k)
+ for shard, keys in by_shard.items():
+ with safe_open(shard, framework="pt", device="cpu") as f:
+ for k in keys:
+ loaded[k] = f.get_tensor(k).to(own_dtype)
+
+ missing, unexpected = self.load_state_dict(loaded, strict=False)
+ n_ok = len(loaded)
+ print(f"✅ MTP: загружено {n_ok} тензоров из {len(by_shard)} шардов ({hf_dir})")
+ if unexpected:
+ print(f"⚠️ MTP: unexpected keys: {unexpected[:10]}")
+ return n_ok
+
+ # -----------------------------------------------------------------
+ # JiRack-чекпоинт (свой формат, с lambda/patched_linear_names)
+ # -----------------------------------------------------------------
+ def save_checkpoint(self, path: str, lam: float, global_step: int = 0, patched_linear_names=None):
+ torch.save({
+ "model": self.state_dict(),
+ "lambda": lam,
+ "global_step": global_step,
+ "patched_linear_names": patched_linear_names or [],
+ }, path)
+
+ @classmethod
+ def load_checkpoint(cls, path: str, dtype=torch.bfloat16, include_gate_proj: bool = False):
+ ckpt = torch.load(path, map_location="cpu", weights_only=False)
+ model = cls()
+ patched_names = ckpt.get("patched_linear_names", [])
+ if patched_names:
+ for name in patched_names:
+ *parent_path, leaf = name.split(".")
+ parent = model
+ for p in parent_path:
+ parent = getattr(parent, p)
+ old = getattr(parent, leaf)
+ if not isinstance(old, BitLinear):
+ setattr(parent, leaf, BitLinear.from_linear(old))
+ else:
+ apply_bitlinear_patch(model, include_gate_proj=include_gate_proj)
+
+ has_mtp_in_ckpt = any(k.startswith("mtp.") for k in ckpt["model"])
+ model.load_state_dict(ckpt["model"], strict=False)
+ model = model.to(dtype)
+
+ if not has_mtp_in_ckpt:
+ print(
+ "⚠️ В этом .pt НЕТ mtp.* весов (чекпоинт сохранён до "
+ "появления MTPBlock). Голова сейчас со СЛУЧАЙНОЙ "
+ "инициализацией! Перед экспортом обязательно вызови "
+ "model.load_mtp_from_hf_dir('<папка оригинального HF-чекпоинта>')."
+ )
+
+ lam = ckpt.get("lambda", 0.0)
+ for m in model.modules():
+ if isinstance(m, BitLinear):
+ m.set_lambda(lam)
+
+ print(f"✅ Загружено. global_step={ckpt.get('global_step', 0)}, lambda={lam}")
+ return model
+
+
+# =====================================================================
+# 7. Демонстрация (без скачивания весов -- только структура)
+# =====================================================================
+
+if __name__ == "__main__":
+ print(ARCH_SUMMARY)
+ print(KEPT_FULL_PRECISION)
+
+ model = JiRackQwen38ForCausalLM()
+ n_params = sum(p.numel() for p in model.parameters())
+ print(f"✅ Модель собрана (случайные веса). Параметров: {n_params / 1e9:.2f}B")
+
+ patched = apply_bitlinear_patch(model)
+ print(f"✅ Заменено подмодулей на BitLinear: {len(patched)}")
+ print("Примеры первых 10:")
+ for n in patched[:10]:
+ print(f" {n}")
+
+ ids = torch.randint(0, VOCAB_SIZE, (1, 8))
+ with torch.no_grad():
+ logits = model(ids)
+ print(f"✅ Пробный forward (случайные веса) -> logits shape: {tuple(logits.shape)}")
diff --git a/sft_final_fp8/audit_depth.json b/sft_final_fp8/audit_depth.json
new file mode 100644
index 0000000000000000000000000000000000000000..420de2a404139630bbe23cc934c6f7fbae0b572d
--- /dev/null
+++ b/sft_final_fp8/audit_depth.json
@@ -0,0 +1,14 @@
+{
+ "kind": "depth",
+ "src": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded",
+ "out": "/mnt/nfs_share/JiRackDeltaNet_70b/expanded",
+ "tensors": 1714,
+ "ops": {
+ "copy": 1586,
+ "zero": 128
+ },
+ "errors": [],
+ "n_errors": 0,
+ "verdict": "PASS",
+ "minutes": 9.8
+}
\ No newline at end of file
diff --git a/sft_final_fp8/chat_fp8.py b/sft_final_fp8/chat_fp8.py
new file mode 100644
index 0000000000000000000000000000000000000000..58e24fdb907fc379ded6edbd4e94291c4b677552
--- /dev/null
+++ b/sft_final_fp8/chat_fp8.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+# =============================================================================
+# chat_fp8.py - chat_jirack.py for the FP8 block-quantized copy (convert_fp8_block.py).
+# python3 chat_fp8.py --model-dir /home/ubuntu/deltanet70b_sft/final_fp8 --no-think --temperature 0
+# Same commands/options as chat_jirack.py (it is imported from the same folder).
+# The quantized GEMM weights stay FP8 in memory (~72 GB for 70B) and are dequantized
+# per layer call (weight * 128x128 block scale -> bf16), so it runs on any GPU
+# (A100, H100, Blackwell); no FP8 tensor-core kernels are used.
+# =============================================================================
+import argparse
+import json
+import os
+import sys
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import chat_jirack as cj # noqa: E402
+
+B = 128
+
+
+class FP8Linear(nn.Module):
+ def __init__(self, in_features, out_features):
+ super().__init__()
+ assert in_features % B == 0 and out_features % B == 0, (in_features, out_features)
+ self.in_features, self.out_features = in_features, out_features
+ self.register_buffer("weight", torch.empty(out_features, in_features, dtype=torch.float8_e4m3fn))
+ self.register_buffer("weight_scale_inv", torch.empty(out_features // B, in_features // B,
+ dtype=torch.float32))
+
+ def forward(self, x):
+ R, C = self.out_features, self.in_features
+ w = self.weight.to(x.dtype).view(R // B, B, C // B, B) * self.weight_scale_inv.to(x.dtype)[:, None, :, None]
+ return F.linear(x, w.view(R, C))
+
+
+def _model_dir():
+ p = argparse.ArgumentParser(add_help=False)
+ p.add_argument("--model-dir", default="/mnt/nfs_share/JiRackDeltaNet_70b/expanded")
+ return p.parse_known_args()[0].model_dir
+
+
+def build_fp8(mod):
+ d = _model_dir()
+ wm = json.load(open(os.path.join(d, "model.safetensors.index.json")))["weight_map"]
+ qnames = {k[:-len(".weight_scale_inv")] for k in wm if k.endswith(".weight_scale_inv")}
+ if not qnames:
+ sys.exit(f"❌ no *.weight_scale_inv in {d} - not an FP8 copy, use chat_jirack.py")
+ torch.set_default_dtype(torch.bfloat16)
+ try:
+ with torch.device("meta"):
+ model = mod.JiRackQwen38ForCausalLM()
+ for name in qnames:
+ parent, _, leaf = name.rpartition(".")
+ pm = model.get_submodule(parent)
+ old = getattr(pm, leaf)
+ setattr(pm, leaf, FP8Linear(old.in_features, old.out_features))
+ finally:
+ torch.set_default_dtype(torch.float32)
+ model.to_empty(device="cpu")
+ for m in model.modules(): # non-persistent buffer, not in the checkpoint
+ if isinstance(m, mod.TextRotaryEmbedding):
+ m.inv_freq = 1.0 / (mod.ROPE_THETA ** (torch.arange(0, m.rotary_dim, 2, dtype=torch.float32)
+ / m.rotary_dim))
+ print(f"🧮 FP8: {len(qnames)} linear layers kept in FP8 (block {B}x{B})", flush=True)
+ return model.eval()
+
+
+cj.build = build_fp8
+
+if __name__ == "__main__":
+ cj.main()
diff --git a/sft_final_fp8/chat_jirack.py b/sft_final_fp8/chat_jirack.py
new file mode 100644
index 0000000000000000000000000000000000000000..b528fc3d015a027a33f5abf2c60d1dcd3b7d1711
--- /dev/null
+++ b/sft_final_fp8/chat_jirack.py
@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+# =============================================================================
+# chat_jirack.py - chat with JiRackDeltaNet (27B / 36B / 70B) from a safetensors folder.
+#
+# python3 chat_jirack.py --model-dir /mnt/nfs_share/JiRackDeltaNet_70b/expanded
+# python3 chat_jirack.py --model-dir /mnt/nfs_share/JiRackDeltaNet_27b_cp # 27B
+#
+# - model code: JiRackDeltaNet_*.py from the same folder (or --model-file)
+# - several GPUs: the TransformerBlocks are split across them evenly (70B bf16 = 140 GB,
+# so at least 2x80 GB); 1 GPU if it fits; CPU works but is VERY slow for 70B
+# - answer streams token by token, stops on <|im_end|> / <|endoftext|>
+# - commands: /reset (clear history), /think (thinking on/off), exit
+# =============================================================================
+import argparse
+import glob
+import importlib.util
+import itertools
+import json
+import os
+import sys
+import time
+
+import torch
+import torch.nn as nn
+
+INIT_FNS = ("uniform_", "normal_", "trunc_normal_", "kaiming_uniform_", "kaiming_normal_",
+ "xavier_uniform_", "xavier_normal_", "orthogonal_")
+
+
+def find_model_file(d):
+ files = sorted(glob.glob(os.path.join(d, "JiRackDeltaNet_*.py")))
+ if not files:
+ sys.exit(f"❌ no JiRackDeltaNet_*.py in {d}; pass --model-file")
+ return files[-1]
+
+
+def load_module(path):
+ spec = importlib.util.spec_from_file_location("jirack_chat_model", path)
+ mod = importlib.util.module_from_spec(spec)
+ sys.modules["jirack_chat_model"] = mod
+ spec.loader.exec_module(mod)
+ return mod
+
+
+def build(mod):
+ """bf16 skeleton without random init (all weights are loaded afterwards) - as in deltanet_gate.py."""
+ saved = {n: getattr(nn.init, n) for n in INIT_FNS if hasattr(nn.init, n)}
+ for n in saved:
+ setattr(nn.init, n, lambda t, *a, **k: t)
+ torch.set_default_dtype(torch.bfloat16)
+ try:
+ model = mod.JiRackQwen38ForCausalLM()
+ finally:
+ torch.set_default_dtype(torch.float32)
+ for n, f in saved.items():
+ setattr(nn.init, n, f)
+ return model.eval()
+
+
+def load_weights(model, d):
+ from safetensors import safe_open
+ idx = os.path.join(d, "model.safetensors.index.json")
+ files = sorted(set(json.load(open(idx))["weight_map"].values())) if os.path.exists(idx) else \
+ sorted(os.path.basename(p) for p in glob.glob(os.path.join(d, "*.safetensors")))
+ sd = model.state_dict()
+ params = {n for n, _ in model.named_parameters()}
+ loaded = set()
+ t0 = time.time()
+ with torch.no_grad():
+ for i, f in enumerate(files):
+ with safe_open(os.path.join(d, f), framework="pt", device="cpu") as h:
+ for k in h.keys():
+ if k in sd:
+ sd[k].copy_(h.get_tensor(k))
+ loaded.add(k)
+ print(f" {i + 1}/{len(files)} {f} {time.time() - t0:.0f} s", flush=True)
+ missing = sorted(params - loaded)
+ if missing:
+ sys.exit(f"❌ {len(missing)} parameters not in {d}: {missing[:5]}")
+
+
+class ChatTokenizer:
+ """tokenizer.json (tokenizers) + chat template (jinja2) - no transformers needed, so the
+ tokenizer_config.json format (e.g. extra_special_tokens as a list) does not matter."""
+
+ def __init__(self, d):
+ import jinja2
+ from jinja2.sandbox import ImmutableSandboxedEnvironment
+ from tokenizers import Tokenizer
+ self.tk = Tokenizer.from_file(os.path.join(d, "tokenizer.json"))
+ cfg = {}
+ if os.path.exists(os.path.join(d, "tokenizer_config.json")):
+ cfg = json.load(open(os.path.join(d, "tokenizer_config.json"), encoding="utf-8"))
+ path = os.path.join(d, "chat_template.jinja")
+ text = open(path, encoding="utf-8").read() if os.path.exists(path) else cfg.get("chat_template")
+ if isinstance(text, list):
+ text = next(t["template"] for t in text if t.get("name") == "default")
+ if not text:
+ sys.exit(f"❌ no chat template in {d}")
+
+ def raise_exception(msg):
+ raise jinja2.exceptions.TemplateError(msg)
+
+ def tojson(x, ensure_ascii=False, indent=None, separators=None, sort_keys=False):
+ return json.dumps(x, ensure_ascii=ensure_ascii, indent=indent, separators=separators,
+ sort_keys=sort_keys)
+
+ env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True)
+ env.filters["tojson"] = tojson
+ env.globals["raise_exception"] = raise_exception
+ self.tmpl = env.from_string(text)
+ s = lambda v: v.get("content") if isinstance(v, dict) else v
+ self.bos, self.eos = s(cfg.get("bos_token")) or "", s(cfg.get("eos_token")) or ""
+ self.special = [t.content for t in self.tk.get_added_tokens_decoder().values() if t.special]
+ self.stop_ids = {i for i in (self.tk.token_to_id("<|im_end|>"), self.tk.token_to_id("<|endoftext|>"),
+ self.tk.token_to_id(self.eos) if self.eos else None) if i is not None}
+
+ def prompt_ids(self, msgs, think):
+ text = self.tmpl.render(messages=msgs, add_generation_prompt=True, enable_thinking=think,
+ bos_token=self.bos, eos_token=self.eos)
+ return self.tk.encode(text, add_special_tokens=False).ids
+
+ def decode(self, ids):
+ return self.tk.decode(ids, skip_special_tokens=False)
+
+
+def _to(x, dev):
+ if torch.is_tensor(x):
+ return x.to(dev)
+ if isinstance(x, (tuple, list)):
+ return type(x)(_to(v, dev) for v in x)
+ if isinstance(x, dict):
+ return {k: _to(v, dev) for k, v in x.items()}
+ return x
+
+
+def place(model, devices):
+ """Blocks split evenly over the GPUs (moved there directly - the whole model never sits on one
+ GPU); everything else on the first GPU. Pre-hooks move each module's inputs to its device,
+ so the model code needs no changes."""
+ blocks = [m for m in model.modules() if type(m).__name__ == "TransformerBlock"]
+ in_block = {id(x) for b in blocks for x in b.modules()}
+ for i, b in enumerate(blocks):
+ dev = devices[i * len(devices) // len(blocks)]
+ b.to(dev)
+ if len(devices) > 1:
+ b.register_forward_pre_hook(lambda m, a, k, dev=dev: (_to(a, dev), _to(k, dev)), with_kwargs=True)
+ for m in model.modules(): # embeddings / norms / lm_head / mtp / rotary
+ if id(m) in in_block:
+ continue
+ m._apply(lambda t: t.to(devices[0]), recurse=False)
+ if len(devices) > 1 and not list(m.children()):
+ m.register_forward_pre_hook(lambda mm, a, k: (_to(a, devices[0]), _to(k, devices[0])),
+ with_kwargs=True)
+ return devices[0]
+
+
+def _cast(x, dtype):
+ if torch.is_tensor(x):
+ return x.to(dtype) if x.is_floating_point() and x.dtype != dtype else x
+ if isinstance(x, (tuple, list)):
+ return type(x)(_cast(v, dtype) for v in x)
+ if isinstance(x, dict):
+ return {k: _cast(v, dtype) for k, v in x.items()}
+ return x
+
+
+def fp32_blocks(model):
+ """CPU (AVX-only Xeon has no fast bf16 matmul): every TransformerBlock and lm_head runs with
+ fp32 weights/inputs, converted on entry and back on exit - stored weights stay bf16 (RAM),
+ same as the 'fp32blocks' compute of deltanet_gate.py."""
+ def pre(mod, args, kwargs):
+ saved = []
+ for t in itertools.chain(mod.parameters(), mod.buffers()):
+ if t.is_floating_point() and t.dtype != torch.float32:
+ saved.append((t, t.data))
+ t.data = t.data.float()
+ mod._jr_saved = saved
+ return _cast(args, torch.float32), _cast(kwargs, torch.float32)
+
+ def post(mod, args, out, back=torch.bfloat16):
+ for t, d in mod._jr_saved:
+ t.data = d
+ mod._jr_saved = []
+ return _cast(out, back) if back is not None else out
+
+ for m in model.modules():
+ if type(m).__name__ == "TransformerBlock":
+ m.register_forward_pre_hook(pre, with_kwargs=True)
+ m.register_forward_hook(post)
+ head = getattr(model, "lm_head", None)
+ if isinstance(head, nn.Module):
+ head.register_forward_pre_hook(pre, with_kwargs=True)
+ head.register_forward_hook(lambda m, a, o: post(m, a, o, None))
+
+
+def logits_of(out):
+ if isinstance(out, (tuple, list)):
+ return out[0]
+ return out.logits if hasattr(out, "logits") else out
+
+
+def sample(logits, temperature, top_p):
+ if temperature <= 0:
+ return int(logits.argmax())
+ probs = torch.softmax(logits.float() / temperature, dim=-1)
+ sp, si = probs.sort(descending=True)
+ cut = (sp.cumsum(-1) - sp) > top_p
+ sp[cut] = 0
+ return int(si[torch.multinomial(sp / sp.sum(), 1)])
+
+
+def main():
+ p = argparse.ArgumentParser()
+ p.add_argument("--model-dir", default="/mnt/nfs_share/JiRackDeltaNet_70b/expanded")
+ p.add_argument("--model-file", default=None)
+ p.add_argument("--tokenizer-dir", default=None, help="default: --model-dir")
+ p.add_argument("--max-new", type=int, default=2048)
+ p.add_argument("--temperature", type=float, default=0.6, help="0 = greedy")
+ p.add_argument("--top-p", type=float, default=0.95)
+ p.add_argument("--system", default=None, help="file with a system prompt")
+ p.add_argument("--no-think", action="store_true", help="start with thinking off")
+ p.add_argument("--cpu", action="store_true")
+ p.add_argument("--cpu-bf16", action="store_true", help="CPU: plain bf16 instead of fp32 blocks")
+ a = p.parse_args()
+
+ tok = ChatTokenizer(a.tokenizer_dir or a.model_dir)
+ stop = tok.stop_ids
+ print(f"stop tokens {sorted(stop)}", flush=True)
+
+ mf = a.model_file or find_model_file(a.model_dir)
+ print(f"model file {mf}\nweights {a.model_dir}", flush=True)
+ model = build(load_module(mf))
+ load_weights(model, a.model_dir)
+ n = sum(q.numel() for q in model.parameters())
+ if a.cpu or not torch.cuda.is_available():
+ dev = torch.device("cpu")
+ try:
+ torch.backends.nnpack.set_flags(False) # silences "Could not initialize NNPACK"
+ except Exception:
+ pass
+ if not a.cpu_bf16:
+ fp32_blocks(model)
+ print(f"⚠️ CPU: {n / 1e9:.1f}B params, compute {'bf16' if a.cpu_bf16 else 'fp32 blocks'} - "
+ f"every token is a full forward pass, expect minutes per token. Ctrl+C stops the answer.")
+ else:
+ devs = [torch.device("cuda", i) for i in range(torch.cuda.device_count())]
+ dev = place(model, devs)
+ print(f"on {len(devs)} GPU(s): {n / 1e9:.1f}B params, "
+ f"{[round(torch.cuda.memory_allocated(d) / 2 ** 30) for d in devs]} GB")
+
+ system = open(a.system, encoding="utf-8").read().strip() if a.system else None
+ think = not a.no_think
+ history = []
+ print("\nReady. /reset - clear history, /think - thinking on/off, exit - quit")
+ while True:
+ try:
+ user = input("\nYou: ").strip()
+ except (EOFError, KeyboardInterrupt):
+ break
+ if not user:
+ continue
+ if user.lower() in ("exit", "quit"):
+ break
+ if user == "/reset":
+ history = []
+ print("(history cleared)")
+ continue
+ if user == "/think":
+ think = not think
+ print(f"(thinking {'on' if think else 'off'})")
+ continue
+ history.append({"role": "user", "content": user})
+ msgs = ([{"role": "system", "content": system}] if system else []) + history
+ ids = torch.tensor([tok.prompt_ids(msgs, think)], dtype=torch.long, device=dev)
+ start = ids.shape[1]
+ print("\nAssistant: ", end="", flush=True)
+ printed, t0 = "", time.time()
+ try:
+ with torch.no_grad():
+ for _ in range(a.max_new):
+ nxt = sample(logits_of(model(ids))[0, -1], a.temperature, a.top_p)
+ if nxt in stop:
+ break
+ ids = torch.cat([ids, torch.tensor([[nxt]], device=ids.device)], dim=1)
+ text = tok.decode(ids[0, start:].tolist())
+ if not text.endswith("�"): # wait for a complete UTF-8 character
+ print(text[len(printed):], end="", flush=True)
+ printed = text
+ except KeyboardInterrupt:
+ print(" [stopped]", end="")
+ gen = ids.shape[1] - start
+ raw = tok.decode(ids[0, start:].tolist())
+ print(f"\n [{gen} tokens, {gen / max(time.time() - t0, 1e-6):.2f} tok/s]")
+ # Qwen: earlier turns go back into the history without the thinking part
+ reply = raw.split("")[-1]
+ for t in tok.special:
+ reply = reply.replace(t, "")
+ history.append({"role": "assistant", "content": reply.strip()})
+
+
+if __name__ == "__main__":
+ main()
diff --git a/sft_final_fp8/chat_template.jinja b/sft_final_fp8/chat_template.jinja
new file mode 100644
index 0000000000000000000000000000000000000000..2e0b08f0d19001d28b1b801b64a7a52c0b4cc088
--- /dev/null
+++ b/sft_final_fp8/chat_template.jinja
@@ -0,0 +1,171 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+ {%- if content is string %}
+ {{- content }}
+ {%- elif content is iterable and content is not mapping %}
+ {%- for item in content %}
+ {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+ {%- if is_system_content %}
+ {{- raise_exception('System message cannot contain images.') }}
+ {%- endif %}
+ {%- if do_vision_count %}
+ {%- set image_count.value = image_count.value + 1 %}
+ {%- endif %}
+ {%- if add_vision_id %}
+ {{- 'Picture ' ~ image_count.value ~ ': ' }}
+ {%- endif %}
+ {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+ {%- elif 'video' in item or item.type == 'video' %}
+ {%- if is_system_content %}
+ {{- raise_exception('System message cannot contain videos.') }}
+ {%- endif %}
+ {%- if do_vision_count %}
+ {%- set video_count.value = video_count.value + 1 %}
+ {%- endif %}
+ {%- if add_vision_id %}
+ {{- 'Video ' ~ video_count.value ~ ': ' }}
+ {%- endif %}
+ {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+ {%- elif 'text' in item %}
+ {{- item.text }}
+ {%- else %}
+ {{- raise_exception('Unexpected item type in content.') }}
+ {%- endif %}
+ {%- endfor %}
+ {%- elif content is none or content is undefined %}
+ {{- '' }}
+ {%- else %}
+ {{- raise_exception('Unexpected content type.') }}
+ {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+ {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- set reasoning_instructions = '' %}
+{%- if enable_thinking is defined and enable_thinking is true %}
+ {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}
+ {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}
+ {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}
+ {%- endif %}
+ {%- if resolved_reasoning_effort == 'xhigh' %}
+ {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}
+ {%- elif resolved_reasoning_effort == 'low' %}
+ {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}
+ {%- endif %}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+ {{- '<|im_start|>system\n' }}
+ {%- if reasoning_instructions %}
+ {{- reasoning_instructions + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou have access to the following functions:\n\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n" }}
+ {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }}
+ {%- if messages[0].role == 'system' %}
+ {%- set content = render_content(messages[0].content, false, true)|trim %}
+ {%- if content %}
+ {{- '\n\n' + content }}
+ {%- endif %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {%- set content = render_content(messages[0].content, false, true)|trim %}
+ {%- if content %}
+ {{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + content + '<|im_end|>\n' }}
+ {%- elif reasoning_instructions %}
+ {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
+ {%- endif %}
+ {%- elif reasoning_instructions %}
+ {{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" %}
+ {%- set content = render_content(message.content, false)|trim %}
+ {%- if not(content.startswith('') and content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+ {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+ {%- set content = render_content(message.content, true)|trim %}
+ {%- if message.role == "system" %}
+ {%- if not loop.first %}
+ {{- raise_exception('System message must be at the beginning.') }}
+ {%- endif %}
+ {%- elif message.role == "user" %}
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is string %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- endif %}
+ {%- set reasoning_content = reasoning_content|trim %}
+ {%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function is defined %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {%- if loop.first %}
+ {%- if content|trim %}
+ {{- '\n\n\n\n' }}
+ {%- else %}
+ {{- '\n\n' }}
+ {%- endif %}
+ {%- else %}
+ {{- '\n\n\n' }}
+ {%- endif %}
+ {%- if tool_call.arguments is defined and tool_call.arguments != '' %}
+ {%- for args_name, args_value in tool_call.arguments|items %}
+ {{- '\n' }}
+ {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}
+ {{- args_value }}
+ {{- '\n\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.previtem and loop.previtem.role != "tool" %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- content }}
+ {{- '\n' }}
+ {%- if not loop.last and loop.nextitem.role != "tool" %}
+ {{- '<|im_end|>\n' }}
+ {%- elif loop.last %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- else %}
+ {{- raise_exception('Unexpected message role.') }}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {%- if enable_thinking is defined and enable_thinking is true %}
+ {{- '\n' }}
+ {%- else %}
+ {{- '\n\n\n\n' }}
+ {%- endif %}
+{%- endif %}
+
diff --git a/sft_final_fp8/config.json b/sft_final_fp8/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..53631b3bd4a22fd8baeca6b0520a26edf32148f4
--- /dev/null
+++ b/sft_final_fp8/config.json
@@ -0,0 +1,201 @@
+{
+ "attention_bias": false,
+ "attention_dropout": 0.0,
+ "attn_output_gate": true,
+ "bos_token_id": 248044,
+ "dtype": "bfloat16",
+ "eos_token_id": 248044,
+ "full_attention_interval": 4,
+ "head_dim": 256,
+ "hidden_act": "silu",
+ "hidden_size": 5120,
+ "initializer_range": 0.02,
+ "intermediate_size": 26624,
+ "layer_types": [
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention",
+ "linear_attention",
+ "linear_attention",
+ "linear_attention",
+ "full_attention"
+ ],
+ "linear_conv_kernel_dim": 4,
+ "linear_key_head_dim": 128,
+ "linear_num_key_heads": 16,
+ "linear_num_value_heads": 48,
+ "linear_value_head_dim": 128,
+ "mamba_ssm_dtype": "float32",
+ "max_position_embeddings": 262144,
+ "model_type": "qwen3_5_text",
+ "mtp_num_hidden_layers": 1,
+ "mtp_use_dedicated_embeddings": false,
+ "num_attention_heads": 24,
+ "num_hidden_layers": 128,
+ "num_key_value_heads": 4,
+ "output_gate_type": "swish",
+ "pad_token_id": null,
+ "partial_rotary_factor": 0.25,
+ "rms_norm_eps": 1e-06,
+ "rope_parameters": {
+ "mrope_interleaved": true,
+ "mrope_section": [
+ 11,
+ 11,
+ 10
+ ],
+ "partial_rotary_factor": 0.25,
+ "rope_theta": 10000000,
+ "rope_type": "default"
+ },
+ "tie_word_embeddings": false,
+ "use_cache": true,
+ "vocab_size": 248320,
+ "architectures": [
+ "Qwen3_5ForCausalLM"
+ ],
+ "quantization_config": {
+ "quant_method": "fp8",
+ "fmt": "e4m3",
+ "activation_scheme": "dynamic",
+ "weight_block_size": [
+ 128,
+ 128
+ ],
+ "modules_to_not_convert": [
+ "lm_head",
+ "model.embed_tokens",
+ "mtp",
+ "in_proj_a",
+ "in_proj_b"
+ ],
+ "ignored_layers": [
+ "lm_head",
+ "model.embed_tokens",
+ "mtp",
+ "in_proj_a",
+ "in_proj_b"
+ ]
+ }
+}
\ No newline at end of file
diff --git a/sft_final_fp8/expansion_depth.json b/sft_final_fp8/expansion_depth.json
new file mode 100644
index 0000000000000000000000000000000000000000..b7787cc90e337393983e2860798df1e45d7dc67c
--- /dev/null
+++ b/sft_final_fp8/expansion_depth.json
@@ -0,0 +1,17623 @@
+{
+ "kind": "depth",
+ "src": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded",
+ "layers_old": 64,
+ "layers_new": 128,
+ "hidden": 5120,
+ "intermediate": 26624,
+ "copies": [
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1,
+ 1
+ ],
+ "layout": [
+ [
+ 0,
+ false
+ ],
+ [
+ 1,
+ false
+ ],
+ [
+ 2,
+ false
+ ],
+ [
+ 3,
+ false
+ ],
+ [
+ 0,
+ true
+ ],
+ [
+ 1,
+ true
+ ],
+ [
+ 2,
+ true
+ ],
+ [
+ 3,
+ true
+ ],
+ [
+ 4,
+ false
+ ],
+ [
+ 5,
+ false
+ ],
+ [
+ 6,
+ false
+ ],
+ [
+ 7,
+ false
+ ],
+ [
+ 4,
+ true
+ ],
+ [
+ 5,
+ true
+ ],
+ [
+ 6,
+ true
+ ],
+ [
+ 7,
+ true
+ ],
+ [
+ 8,
+ false
+ ],
+ [
+ 9,
+ false
+ ],
+ [
+ 10,
+ false
+ ],
+ [
+ 11,
+ false
+ ],
+ [
+ 8,
+ true
+ ],
+ [
+ 9,
+ true
+ ],
+ [
+ 10,
+ true
+ ],
+ [
+ 11,
+ true
+ ],
+ [
+ 12,
+ false
+ ],
+ [
+ 13,
+ false
+ ],
+ [
+ 14,
+ false
+ ],
+ [
+ 15,
+ false
+ ],
+ [
+ 12,
+ true
+ ],
+ [
+ 13,
+ true
+ ],
+ [
+ 14,
+ true
+ ],
+ [
+ 15,
+ true
+ ],
+ [
+ 16,
+ false
+ ],
+ [
+ 17,
+ false
+ ],
+ [
+ 18,
+ false
+ ],
+ [
+ 19,
+ false
+ ],
+ [
+ 16,
+ true
+ ],
+ [
+ 17,
+ true
+ ],
+ [
+ 18,
+ true
+ ],
+ [
+ 19,
+ true
+ ],
+ [
+ 20,
+ false
+ ],
+ [
+ 21,
+ false
+ ],
+ [
+ 22,
+ false
+ ],
+ [
+ 23,
+ false
+ ],
+ [
+ 20,
+ true
+ ],
+ [
+ 21,
+ true
+ ],
+ [
+ 22,
+ true
+ ],
+ [
+ 23,
+ true
+ ],
+ [
+ 24,
+ false
+ ],
+ [
+ 25,
+ false
+ ],
+ [
+ 26,
+ false
+ ],
+ [
+ 27,
+ false
+ ],
+ [
+ 24,
+ true
+ ],
+ [
+ 25,
+ true
+ ],
+ [
+ 26,
+ true
+ ],
+ [
+ 27,
+ true
+ ],
+ [
+ 28,
+ false
+ ],
+ [
+ 29,
+ false
+ ],
+ [
+ 30,
+ false
+ ],
+ [
+ 31,
+ false
+ ],
+ [
+ 28,
+ true
+ ],
+ [
+ 29,
+ true
+ ],
+ [
+ 30,
+ true
+ ],
+ [
+ 31,
+ true
+ ],
+ [
+ 32,
+ false
+ ],
+ [
+ 33,
+ false
+ ],
+ [
+ 34,
+ false
+ ],
+ [
+ 35,
+ false
+ ],
+ [
+ 32,
+ true
+ ],
+ [
+ 33,
+ true
+ ],
+ [
+ 34,
+ true
+ ],
+ [
+ 35,
+ true
+ ],
+ [
+ 36,
+ false
+ ],
+ [
+ 37,
+ false
+ ],
+ [
+ 38,
+ false
+ ],
+ [
+ 39,
+ false
+ ],
+ [
+ 36,
+ true
+ ],
+ [
+ 37,
+ true
+ ],
+ [
+ 38,
+ true
+ ],
+ [
+ 39,
+ true
+ ],
+ [
+ 40,
+ false
+ ],
+ [
+ 41,
+ false
+ ],
+ [
+ 42,
+ false
+ ],
+ [
+ 43,
+ false
+ ],
+ [
+ 40,
+ true
+ ],
+ [
+ 41,
+ true
+ ],
+ [
+ 42,
+ true
+ ],
+ [
+ 43,
+ true
+ ],
+ [
+ 44,
+ false
+ ],
+ [
+ 45,
+ false
+ ],
+ [
+ 46,
+ false
+ ],
+ [
+ 47,
+ false
+ ],
+ [
+ 44,
+ true
+ ],
+ [
+ 45,
+ true
+ ],
+ [
+ 46,
+ true
+ ],
+ [
+ 47,
+ true
+ ],
+ [
+ 48,
+ false
+ ],
+ [
+ 49,
+ false
+ ],
+ [
+ 50,
+ false
+ ],
+ [
+ 51,
+ false
+ ],
+ [
+ 48,
+ true
+ ],
+ [
+ 49,
+ true
+ ],
+ [
+ 50,
+ true
+ ],
+ [
+ 51,
+ true
+ ],
+ [
+ 52,
+ false
+ ],
+ [
+ 53,
+ false
+ ],
+ [
+ 54,
+ false
+ ],
+ [
+ 55,
+ false
+ ],
+ [
+ 52,
+ true
+ ],
+ [
+ 53,
+ true
+ ],
+ [
+ 54,
+ true
+ ],
+ [
+ 55,
+ true
+ ],
+ [
+ 56,
+ false
+ ],
+ [
+ 57,
+ false
+ ],
+ [
+ 58,
+ false
+ ],
+ [
+ 59,
+ false
+ ],
+ [
+ 56,
+ true
+ ],
+ [
+ 57,
+ true
+ ],
+ [
+ 58,
+ true
+ ],
+ [
+ 59,
+ true
+ ],
+ [
+ 60,
+ false
+ ],
+ [
+ 61,
+ false
+ ],
+ [
+ 62,
+ false
+ ],
+ [
+ 63,
+ false
+ ],
+ [
+ 60,
+ true
+ ],
+ [
+ 61,
+ true
+ ],
+ [
+ 62,
+ true
+ ],
+ [
+ 63,
+ true
+ ]
+ ],
+ "zeroed_in_copies": [
+ "linear_attn.out_proj.weight",
+ "self_attn.o_proj.weight",
+ "mlp.down_proj.weight"
+ ],
+ "params_before": 36521952256,
+ "params_after": 69934845440,
+ "shards": 27,
+ "bytes": 139869690880,
+ "side_files": [
+ "chat_template.jinja",
+ "config.json",
+ "gguf.txt",
+ "system_prompt.txt",
+ "tokenizer.json",
+ "tokenizer_config.json"
+ ],
+ "config_changes": [
+ "layer_types: 64 entries -> 128 (same 3 linear + 1 full pattern)",
+ "num_hidden_layers: 64 -> 128"
+ ],
+ "model_file_diff": [
+ "--- JiRackDeltaNet_27b.py",
+ "+++ JiRackDeltaNet_70b.py",
+ "@@ -52,2 +52,2 @@",
+ "-INTERMEDIATE_SIZE = 17408",
+ "-NUM_LAYERS = 64",
+ "+INTERMEDIATE_SIZE = 26624",
+ "+NUM_LAYERS = 128"
+ ],
+ "minutes": 8.5,
+ "recipe": [
+ {
+ "key": "model.embed_tokens.weight",
+ "src": "model.embed_tokens.weight",
+ "op": "copy"
+ },
+ {
+ "key": "model.layers.0.input_layernorm.weight",
+ "src": "model.layers.0.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.A_log",
+ "src": "model.layers.0.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.conv1d.weight",
+ "src": "model.layers.0.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.dt_bias",
+ "src": "model.layers.0.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.in_proj_a.weight",
+ "src": "model.layers.0.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.in_proj_b.weight",
+ "src": "model.layers.0.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.0.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.in_proj_z.weight",
+ "src": "model.layers.0.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.norm.weight",
+ "src": "model.layers.0.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.linear_attn.out_proj.weight",
+ "src": "model.layers.0.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.mlp.down_proj.weight",
+ "src": "model.layers.0.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.mlp.gate_proj.weight",
+ "src": "model.layers.0.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.mlp.up_proj.weight",
+ "src": "model.layers.0.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.0.post_attention_layernorm.weight",
+ "src": "model.layers.0.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 0,
+ 0,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.input_layernorm.weight",
+ "src": "model.layers.1.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.A_log",
+ "src": "model.layers.1.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.conv1d.weight",
+ "src": "model.layers.1.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.dt_bias",
+ "src": "model.layers.1.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.in_proj_a.weight",
+ "src": "model.layers.1.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.in_proj_b.weight",
+ "src": "model.layers.1.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.1.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.in_proj_z.weight",
+ "src": "model.layers.1.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.norm.weight",
+ "src": "model.layers.1.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.linear_attn.out_proj.weight",
+ "src": "model.layers.1.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.mlp.down_proj.weight",
+ "src": "model.layers.1.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.mlp.gate_proj.weight",
+ "src": "model.layers.1.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.mlp.up_proj.weight",
+ "src": "model.layers.1.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.1.post_attention_layernorm.weight",
+ "src": "model.layers.1.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 1,
+ 1,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.input_layernorm.weight",
+ "src": "model.layers.2.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.A_log",
+ "src": "model.layers.2.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.conv1d.weight",
+ "src": "model.layers.2.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.dt_bias",
+ "src": "model.layers.2.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.in_proj_a.weight",
+ "src": "model.layers.2.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.in_proj_b.weight",
+ "src": "model.layers.2.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.2.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.in_proj_z.weight",
+ "src": "model.layers.2.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.norm.weight",
+ "src": "model.layers.2.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.linear_attn.out_proj.weight",
+ "src": "model.layers.2.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.mlp.down_proj.weight",
+ "src": "model.layers.2.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.mlp.gate_proj.weight",
+ "src": "model.layers.2.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.mlp.up_proj.weight",
+ "src": "model.layers.2.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.2.post_attention_layernorm.weight",
+ "src": "model.layers.2.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 2,
+ 2,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.input_layernorm.weight",
+ "src": "model.layers.3.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.mlp.down_proj.weight",
+ "src": "model.layers.3.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.mlp.gate_proj.weight",
+ "src": "model.layers.3.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.mlp.up_proj.weight",
+ "src": "model.layers.3.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.post_attention_layernorm.weight",
+ "src": "model.layers.3.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.self_attn.k_norm.weight",
+ "src": "model.layers.3.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.self_attn.k_proj.weight",
+ "src": "model.layers.3.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.self_attn.o_proj.weight",
+ "src": "model.layers.3.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.self_attn.q_norm.weight",
+ "src": "model.layers.3.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.self_attn.q_proj.weight",
+ "src": "model.layers.3.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.3.self_attn.v_proj.weight",
+ "src": "model.layers.3.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 3,
+ 3,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.4.input_layernorm.weight",
+ "src": "model.layers.0.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.A_log",
+ "src": "model.layers.0.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.conv1d.weight",
+ "src": "model.layers.0.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.dt_bias",
+ "src": "model.layers.0.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.in_proj_a.weight",
+ "src": "model.layers.0.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.in_proj_b.weight",
+ "src": "model.layers.0.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.0.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.in_proj_z.weight",
+ "src": "model.layers.0.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.norm.weight",
+ "src": "model.layers.0.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.linear_attn.out_proj.weight",
+ "src": "model.layers.0.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.mlp.down_proj.weight",
+ "src": "model.layers.0.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.mlp.gate_proj.weight",
+ "src": "model.layers.0.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.mlp.up_proj.weight",
+ "src": "model.layers.0.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.4.post_attention_layernorm.weight",
+ "src": "model.layers.0.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 4,
+ 0,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.input_layernorm.weight",
+ "src": "model.layers.1.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.A_log",
+ "src": "model.layers.1.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.conv1d.weight",
+ "src": "model.layers.1.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.dt_bias",
+ "src": "model.layers.1.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.in_proj_a.weight",
+ "src": "model.layers.1.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.in_proj_b.weight",
+ "src": "model.layers.1.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.1.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.in_proj_z.weight",
+ "src": "model.layers.1.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.norm.weight",
+ "src": "model.layers.1.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.linear_attn.out_proj.weight",
+ "src": "model.layers.1.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.mlp.down_proj.weight",
+ "src": "model.layers.1.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.mlp.gate_proj.weight",
+ "src": "model.layers.1.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.mlp.up_proj.weight",
+ "src": "model.layers.1.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.5.post_attention_layernorm.weight",
+ "src": "model.layers.1.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 5,
+ 1,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.input_layernorm.weight",
+ "src": "model.layers.2.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.A_log",
+ "src": "model.layers.2.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.conv1d.weight",
+ "src": "model.layers.2.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.dt_bias",
+ "src": "model.layers.2.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.in_proj_a.weight",
+ "src": "model.layers.2.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.in_proj_b.weight",
+ "src": "model.layers.2.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.2.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.in_proj_z.weight",
+ "src": "model.layers.2.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.norm.weight",
+ "src": "model.layers.2.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.linear_attn.out_proj.weight",
+ "src": "model.layers.2.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.mlp.down_proj.weight",
+ "src": "model.layers.2.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.mlp.gate_proj.weight",
+ "src": "model.layers.2.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.mlp.up_proj.weight",
+ "src": "model.layers.2.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.6.post_attention_layernorm.weight",
+ "src": "model.layers.2.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 6,
+ 2,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.input_layernorm.weight",
+ "src": "model.layers.3.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.mlp.down_proj.weight",
+ "src": "model.layers.3.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.mlp.gate_proj.weight",
+ "src": "model.layers.3.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.mlp.up_proj.weight",
+ "src": "model.layers.3.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.post_attention_layernorm.weight",
+ "src": "model.layers.3.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.self_attn.k_norm.weight",
+ "src": "model.layers.3.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.self_attn.k_proj.weight",
+ "src": "model.layers.3.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.self_attn.o_proj.weight",
+ "src": "model.layers.3.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.self_attn.q_norm.weight",
+ "src": "model.layers.3.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.self_attn.q_proj.weight",
+ "src": "model.layers.3.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.7.self_attn.v_proj.weight",
+ "src": "model.layers.3.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 7,
+ 3,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.8.input_layernorm.weight",
+ "src": "model.layers.4.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.A_log",
+ "src": "model.layers.4.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.conv1d.weight",
+ "src": "model.layers.4.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.dt_bias",
+ "src": "model.layers.4.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.in_proj_a.weight",
+ "src": "model.layers.4.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.in_proj_b.weight",
+ "src": "model.layers.4.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.4.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.in_proj_z.weight",
+ "src": "model.layers.4.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.norm.weight",
+ "src": "model.layers.4.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.linear_attn.out_proj.weight",
+ "src": "model.layers.4.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.mlp.down_proj.weight",
+ "src": "model.layers.4.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.mlp.gate_proj.weight",
+ "src": "model.layers.4.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.mlp.up_proj.weight",
+ "src": "model.layers.4.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.8.post_attention_layernorm.weight",
+ "src": "model.layers.4.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 8,
+ 4,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.input_layernorm.weight",
+ "src": "model.layers.5.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.A_log",
+ "src": "model.layers.5.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.conv1d.weight",
+ "src": "model.layers.5.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.dt_bias",
+ "src": "model.layers.5.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.in_proj_a.weight",
+ "src": "model.layers.5.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.in_proj_b.weight",
+ "src": "model.layers.5.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.5.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.in_proj_z.weight",
+ "src": "model.layers.5.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.norm.weight",
+ "src": "model.layers.5.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.linear_attn.out_proj.weight",
+ "src": "model.layers.5.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.mlp.down_proj.weight",
+ "src": "model.layers.5.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.mlp.gate_proj.weight",
+ "src": "model.layers.5.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.mlp.up_proj.weight",
+ "src": "model.layers.5.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.9.post_attention_layernorm.weight",
+ "src": "model.layers.5.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 9,
+ 5,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.input_layernorm.weight",
+ "src": "model.layers.6.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.A_log",
+ "src": "model.layers.6.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.conv1d.weight",
+ "src": "model.layers.6.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.dt_bias",
+ "src": "model.layers.6.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.in_proj_a.weight",
+ "src": "model.layers.6.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.in_proj_b.weight",
+ "src": "model.layers.6.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.6.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.in_proj_z.weight",
+ "src": "model.layers.6.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.norm.weight",
+ "src": "model.layers.6.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.linear_attn.out_proj.weight",
+ "src": "model.layers.6.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.mlp.down_proj.weight",
+ "src": "model.layers.6.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.mlp.gate_proj.weight",
+ "src": "model.layers.6.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.mlp.up_proj.weight",
+ "src": "model.layers.6.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.10.post_attention_layernorm.weight",
+ "src": "model.layers.6.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 10,
+ 6,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.input_layernorm.weight",
+ "src": "model.layers.7.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.mlp.down_proj.weight",
+ "src": "model.layers.7.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.mlp.gate_proj.weight",
+ "src": "model.layers.7.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.mlp.up_proj.weight",
+ "src": "model.layers.7.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.post_attention_layernorm.weight",
+ "src": "model.layers.7.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.self_attn.k_norm.weight",
+ "src": "model.layers.7.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.self_attn.k_proj.weight",
+ "src": "model.layers.7.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.self_attn.o_proj.weight",
+ "src": "model.layers.7.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.self_attn.q_norm.weight",
+ "src": "model.layers.7.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.self_attn.q_proj.weight",
+ "src": "model.layers.7.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.11.self_attn.v_proj.weight",
+ "src": "model.layers.7.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 11,
+ 7,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.12.input_layernorm.weight",
+ "src": "model.layers.4.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.A_log",
+ "src": "model.layers.4.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.conv1d.weight",
+ "src": "model.layers.4.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.dt_bias",
+ "src": "model.layers.4.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.in_proj_a.weight",
+ "src": "model.layers.4.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.in_proj_b.weight",
+ "src": "model.layers.4.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.4.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.in_proj_z.weight",
+ "src": "model.layers.4.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.norm.weight",
+ "src": "model.layers.4.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.linear_attn.out_proj.weight",
+ "src": "model.layers.4.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.mlp.down_proj.weight",
+ "src": "model.layers.4.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.mlp.gate_proj.weight",
+ "src": "model.layers.4.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.mlp.up_proj.weight",
+ "src": "model.layers.4.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.12.post_attention_layernorm.weight",
+ "src": "model.layers.4.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 12,
+ 4,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.input_layernorm.weight",
+ "src": "model.layers.5.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.A_log",
+ "src": "model.layers.5.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.conv1d.weight",
+ "src": "model.layers.5.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.dt_bias",
+ "src": "model.layers.5.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.in_proj_a.weight",
+ "src": "model.layers.5.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.in_proj_b.weight",
+ "src": "model.layers.5.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.5.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.in_proj_z.weight",
+ "src": "model.layers.5.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.norm.weight",
+ "src": "model.layers.5.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.linear_attn.out_proj.weight",
+ "src": "model.layers.5.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.mlp.down_proj.weight",
+ "src": "model.layers.5.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.mlp.gate_proj.weight",
+ "src": "model.layers.5.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.mlp.up_proj.weight",
+ "src": "model.layers.5.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.13.post_attention_layernorm.weight",
+ "src": "model.layers.5.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 13,
+ 5,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.input_layernorm.weight",
+ "src": "model.layers.6.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.A_log",
+ "src": "model.layers.6.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.conv1d.weight",
+ "src": "model.layers.6.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.dt_bias",
+ "src": "model.layers.6.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.in_proj_a.weight",
+ "src": "model.layers.6.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.in_proj_b.weight",
+ "src": "model.layers.6.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.6.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.in_proj_z.weight",
+ "src": "model.layers.6.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.norm.weight",
+ "src": "model.layers.6.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.linear_attn.out_proj.weight",
+ "src": "model.layers.6.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.mlp.down_proj.weight",
+ "src": "model.layers.6.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.mlp.gate_proj.weight",
+ "src": "model.layers.6.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.mlp.up_proj.weight",
+ "src": "model.layers.6.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.14.post_attention_layernorm.weight",
+ "src": "model.layers.6.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 14,
+ 6,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.input_layernorm.weight",
+ "src": "model.layers.7.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.mlp.down_proj.weight",
+ "src": "model.layers.7.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.mlp.gate_proj.weight",
+ "src": "model.layers.7.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.mlp.up_proj.weight",
+ "src": "model.layers.7.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.post_attention_layernorm.weight",
+ "src": "model.layers.7.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.self_attn.k_norm.weight",
+ "src": "model.layers.7.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.self_attn.k_proj.weight",
+ "src": "model.layers.7.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.self_attn.o_proj.weight",
+ "src": "model.layers.7.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.self_attn.q_norm.weight",
+ "src": "model.layers.7.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.self_attn.q_proj.weight",
+ "src": "model.layers.7.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.15.self_attn.v_proj.weight",
+ "src": "model.layers.7.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 15,
+ 7,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.16.input_layernorm.weight",
+ "src": "model.layers.8.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.A_log",
+ "src": "model.layers.8.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.conv1d.weight",
+ "src": "model.layers.8.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.dt_bias",
+ "src": "model.layers.8.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.in_proj_a.weight",
+ "src": "model.layers.8.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.in_proj_b.weight",
+ "src": "model.layers.8.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.8.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.in_proj_z.weight",
+ "src": "model.layers.8.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.norm.weight",
+ "src": "model.layers.8.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.linear_attn.out_proj.weight",
+ "src": "model.layers.8.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.mlp.down_proj.weight",
+ "src": "model.layers.8.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.mlp.gate_proj.weight",
+ "src": "model.layers.8.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.mlp.up_proj.weight",
+ "src": "model.layers.8.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.16.post_attention_layernorm.weight",
+ "src": "model.layers.8.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 16,
+ 8,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.input_layernorm.weight",
+ "src": "model.layers.9.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.A_log",
+ "src": "model.layers.9.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.conv1d.weight",
+ "src": "model.layers.9.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.dt_bias",
+ "src": "model.layers.9.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.in_proj_a.weight",
+ "src": "model.layers.9.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.in_proj_b.weight",
+ "src": "model.layers.9.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.9.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.in_proj_z.weight",
+ "src": "model.layers.9.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.norm.weight",
+ "src": "model.layers.9.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.linear_attn.out_proj.weight",
+ "src": "model.layers.9.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.mlp.down_proj.weight",
+ "src": "model.layers.9.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.mlp.gate_proj.weight",
+ "src": "model.layers.9.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.mlp.up_proj.weight",
+ "src": "model.layers.9.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.17.post_attention_layernorm.weight",
+ "src": "model.layers.9.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 17,
+ 9,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.input_layernorm.weight",
+ "src": "model.layers.10.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.A_log",
+ "src": "model.layers.10.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.conv1d.weight",
+ "src": "model.layers.10.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.dt_bias",
+ "src": "model.layers.10.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.in_proj_a.weight",
+ "src": "model.layers.10.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.in_proj_b.weight",
+ "src": "model.layers.10.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.10.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.in_proj_z.weight",
+ "src": "model.layers.10.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.norm.weight",
+ "src": "model.layers.10.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.linear_attn.out_proj.weight",
+ "src": "model.layers.10.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.mlp.down_proj.weight",
+ "src": "model.layers.10.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.mlp.gate_proj.weight",
+ "src": "model.layers.10.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.mlp.up_proj.weight",
+ "src": "model.layers.10.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.18.post_attention_layernorm.weight",
+ "src": "model.layers.10.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 18,
+ 10,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.input_layernorm.weight",
+ "src": "model.layers.11.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.mlp.down_proj.weight",
+ "src": "model.layers.11.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.mlp.gate_proj.weight",
+ "src": "model.layers.11.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.mlp.up_proj.weight",
+ "src": "model.layers.11.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.post_attention_layernorm.weight",
+ "src": "model.layers.11.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.self_attn.k_norm.weight",
+ "src": "model.layers.11.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.self_attn.k_proj.weight",
+ "src": "model.layers.11.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.self_attn.o_proj.weight",
+ "src": "model.layers.11.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.self_attn.q_norm.weight",
+ "src": "model.layers.11.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.self_attn.q_proj.weight",
+ "src": "model.layers.11.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.19.self_attn.v_proj.weight",
+ "src": "model.layers.11.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 19,
+ 11,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.20.input_layernorm.weight",
+ "src": "model.layers.8.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.A_log",
+ "src": "model.layers.8.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.conv1d.weight",
+ "src": "model.layers.8.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.dt_bias",
+ "src": "model.layers.8.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.in_proj_a.weight",
+ "src": "model.layers.8.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.in_proj_b.weight",
+ "src": "model.layers.8.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.8.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.in_proj_z.weight",
+ "src": "model.layers.8.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.norm.weight",
+ "src": "model.layers.8.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.linear_attn.out_proj.weight",
+ "src": "model.layers.8.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.mlp.down_proj.weight",
+ "src": "model.layers.8.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.mlp.gate_proj.weight",
+ "src": "model.layers.8.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.mlp.up_proj.weight",
+ "src": "model.layers.8.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.20.post_attention_layernorm.weight",
+ "src": "model.layers.8.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 20,
+ 8,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.input_layernorm.weight",
+ "src": "model.layers.9.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.A_log",
+ "src": "model.layers.9.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.conv1d.weight",
+ "src": "model.layers.9.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.dt_bias",
+ "src": "model.layers.9.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.in_proj_a.weight",
+ "src": "model.layers.9.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.in_proj_b.weight",
+ "src": "model.layers.9.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.9.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.in_proj_z.weight",
+ "src": "model.layers.9.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.norm.weight",
+ "src": "model.layers.9.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.linear_attn.out_proj.weight",
+ "src": "model.layers.9.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.mlp.down_proj.weight",
+ "src": "model.layers.9.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.mlp.gate_proj.weight",
+ "src": "model.layers.9.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.mlp.up_proj.weight",
+ "src": "model.layers.9.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.21.post_attention_layernorm.weight",
+ "src": "model.layers.9.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 21,
+ 9,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.input_layernorm.weight",
+ "src": "model.layers.10.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.A_log",
+ "src": "model.layers.10.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.conv1d.weight",
+ "src": "model.layers.10.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.dt_bias",
+ "src": "model.layers.10.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.in_proj_a.weight",
+ "src": "model.layers.10.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.in_proj_b.weight",
+ "src": "model.layers.10.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.10.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.in_proj_z.weight",
+ "src": "model.layers.10.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.norm.weight",
+ "src": "model.layers.10.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.linear_attn.out_proj.weight",
+ "src": "model.layers.10.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.mlp.down_proj.weight",
+ "src": "model.layers.10.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.mlp.gate_proj.weight",
+ "src": "model.layers.10.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.mlp.up_proj.weight",
+ "src": "model.layers.10.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.22.post_attention_layernorm.weight",
+ "src": "model.layers.10.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 22,
+ 10,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.input_layernorm.weight",
+ "src": "model.layers.11.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.mlp.down_proj.weight",
+ "src": "model.layers.11.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.mlp.gate_proj.weight",
+ "src": "model.layers.11.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.mlp.up_proj.weight",
+ "src": "model.layers.11.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.post_attention_layernorm.weight",
+ "src": "model.layers.11.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.self_attn.k_norm.weight",
+ "src": "model.layers.11.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.self_attn.k_proj.weight",
+ "src": "model.layers.11.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.self_attn.o_proj.weight",
+ "src": "model.layers.11.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.self_attn.q_norm.weight",
+ "src": "model.layers.11.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.self_attn.q_proj.weight",
+ "src": "model.layers.11.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.23.self_attn.v_proj.weight",
+ "src": "model.layers.11.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 23,
+ 11,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.24.input_layernorm.weight",
+ "src": "model.layers.12.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.A_log",
+ "src": "model.layers.12.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.conv1d.weight",
+ "src": "model.layers.12.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.dt_bias",
+ "src": "model.layers.12.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.in_proj_a.weight",
+ "src": "model.layers.12.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.in_proj_b.weight",
+ "src": "model.layers.12.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.12.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.in_proj_z.weight",
+ "src": "model.layers.12.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.norm.weight",
+ "src": "model.layers.12.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.linear_attn.out_proj.weight",
+ "src": "model.layers.12.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.mlp.down_proj.weight",
+ "src": "model.layers.12.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.mlp.gate_proj.weight",
+ "src": "model.layers.12.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.mlp.up_proj.weight",
+ "src": "model.layers.12.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.24.post_attention_layernorm.weight",
+ "src": "model.layers.12.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 24,
+ 12,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.input_layernorm.weight",
+ "src": "model.layers.13.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.A_log",
+ "src": "model.layers.13.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.conv1d.weight",
+ "src": "model.layers.13.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.dt_bias",
+ "src": "model.layers.13.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.in_proj_a.weight",
+ "src": "model.layers.13.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.in_proj_b.weight",
+ "src": "model.layers.13.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.13.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.in_proj_z.weight",
+ "src": "model.layers.13.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.norm.weight",
+ "src": "model.layers.13.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.linear_attn.out_proj.weight",
+ "src": "model.layers.13.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.mlp.down_proj.weight",
+ "src": "model.layers.13.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.mlp.gate_proj.weight",
+ "src": "model.layers.13.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.mlp.up_proj.weight",
+ "src": "model.layers.13.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.25.post_attention_layernorm.weight",
+ "src": "model.layers.13.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 25,
+ 13,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.input_layernorm.weight",
+ "src": "model.layers.14.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.A_log",
+ "src": "model.layers.14.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.conv1d.weight",
+ "src": "model.layers.14.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.dt_bias",
+ "src": "model.layers.14.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.in_proj_a.weight",
+ "src": "model.layers.14.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.in_proj_b.weight",
+ "src": "model.layers.14.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.14.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.in_proj_z.weight",
+ "src": "model.layers.14.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.norm.weight",
+ "src": "model.layers.14.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.linear_attn.out_proj.weight",
+ "src": "model.layers.14.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.mlp.down_proj.weight",
+ "src": "model.layers.14.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.mlp.gate_proj.weight",
+ "src": "model.layers.14.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.mlp.up_proj.weight",
+ "src": "model.layers.14.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.26.post_attention_layernorm.weight",
+ "src": "model.layers.14.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 26,
+ 14,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.input_layernorm.weight",
+ "src": "model.layers.15.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.mlp.down_proj.weight",
+ "src": "model.layers.15.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.mlp.gate_proj.weight",
+ "src": "model.layers.15.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.mlp.up_proj.weight",
+ "src": "model.layers.15.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.post_attention_layernorm.weight",
+ "src": "model.layers.15.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.self_attn.k_norm.weight",
+ "src": "model.layers.15.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.self_attn.k_proj.weight",
+ "src": "model.layers.15.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.self_attn.o_proj.weight",
+ "src": "model.layers.15.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.self_attn.q_norm.weight",
+ "src": "model.layers.15.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.self_attn.q_proj.weight",
+ "src": "model.layers.15.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.27.self_attn.v_proj.weight",
+ "src": "model.layers.15.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 27,
+ 15,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.28.input_layernorm.weight",
+ "src": "model.layers.12.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.A_log",
+ "src": "model.layers.12.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.conv1d.weight",
+ "src": "model.layers.12.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.dt_bias",
+ "src": "model.layers.12.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.in_proj_a.weight",
+ "src": "model.layers.12.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.in_proj_b.weight",
+ "src": "model.layers.12.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.12.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.in_proj_z.weight",
+ "src": "model.layers.12.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.norm.weight",
+ "src": "model.layers.12.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.linear_attn.out_proj.weight",
+ "src": "model.layers.12.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.mlp.down_proj.weight",
+ "src": "model.layers.12.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.mlp.gate_proj.weight",
+ "src": "model.layers.12.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.mlp.up_proj.weight",
+ "src": "model.layers.12.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.28.post_attention_layernorm.weight",
+ "src": "model.layers.12.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 28,
+ 12,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.input_layernorm.weight",
+ "src": "model.layers.13.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.A_log",
+ "src": "model.layers.13.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.conv1d.weight",
+ "src": "model.layers.13.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.dt_bias",
+ "src": "model.layers.13.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.in_proj_a.weight",
+ "src": "model.layers.13.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.in_proj_b.weight",
+ "src": "model.layers.13.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.13.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.in_proj_z.weight",
+ "src": "model.layers.13.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.norm.weight",
+ "src": "model.layers.13.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.linear_attn.out_proj.weight",
+ "src": "model.layers.13.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.mlp.down_proj.weight",
+ "src": "model.layers.13.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.mlp.gate_proj.weight",
+ "src": "model.layers.13.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.mlp.up_proj.weight",
+ "src": "model.layers.13.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.29.post_attention_layernorm.weight",
+ "src": "model.layers.13.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 29,
+ 13,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.input_layernorm.weight",
+ "src": "model.layers.14.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.A_log",
+ "src": "model.layers.14.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.conv1d.weight",
+ "src": "model.layers.14.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.dt_bias",
+ "src": "model.layers.14.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.in_proj_a.weight",
+ "src": "model.layers.14.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.in_proj_b.weight",
+ "src": "model.layers.14.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.14.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.in_proj_z.weight",
+ "src": "model.layers.14.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.norm.weight",
+ "src": "model.layers.14.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.linear_attn.out_proj.weight",
+ "src": "model.layers.14.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.mlp.down_proj.weight",
+ "src": "model.layers.14.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.mlp.gate_proj.weight",
+ "src": "model.layers.14.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.mlp.up_proj.weight",
+ "src": "model.layers.14.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.30.post_attention_layernorm.weight",
+ "src": "model.layers.14.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 30,
+ 14,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.input_layernorm.weight",
+ "src": "model.layers.15.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.mlp.down_proj.weight",
+ "src": "model.layers.15.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.mlp.gate_proj.weight",
+ "src": "model.layers.15.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.mlp.up_proj.weight",
+ "src": "model.layers.15.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.post_attention_layernorm.weight",
+ "src": "model.layers.15.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.self_attn.k_norm.weight",
+ "src": "model.layers.15.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.self_attn.k_proj.weight",
+ "src": "model.layers.15.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.self_attn.o_proj.weight",
+ "src": "model.layers.15.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.self_attn.q_norm.weight",
+ "src": "model.layers.15.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.self_attn.q_proj.weight",
+ "src": "model.layers.15.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.31.self_attn.v_proj.weight",
+ "src": "model.layers.15.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 31,
+ 15,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.32.input_layernorm.weight",
+ "src": "model.layers.16.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.A_log",
+ "src": "model.layers.16.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.conv1d.weight",
+ "src": "model.layers.16.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.dt_bias",
+ "src": "model.layers.16.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.in_proj_a.weight",
+ "src": "model.layers.16.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.in_proj_b.weight",
+ "src": "model.layers.16.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.16.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.in_proj_z.weight",
+ "src": "model.layers.16.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.norm.weight",
+ "src": "model.layers.16.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.linear_attn.out_proj.weight",
+ "src": "model.layers.16.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.mlp.down_proj.weight",
+ "src": "model.layers.16.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.mlp.gate_proj.weight",
+ "src": "model.layers.16.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.mlp.up_proj.weight",
+ "src": "model.layers.16.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.32.post_attention_layernorm.weight",
+ "src": "model.layers.16.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 32,
+ 16,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.input_layernorm.weight",
+ "src": "model.layers.17.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.A_log",
+ "src": "model.layers.17.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.conv1d.weight",
+ "src": "model.layers.17.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.dt_bias",
+ "src": "model.layers.17.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.in_proj_a.weight",
+ "src": "model.layers.17.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.in_proj_b.weight",
+ "src": "model.layers.17.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.17.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.in_proj_z.weight",
+ "src": "model.layers.17.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.norm.weight",
+ "src": "model.layers.17.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.linear_attn.out_proj.weight",
+ "src": "model.layers.17.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.mlp.down_proj.weight",
+ "src": "model.layers.17.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.mlp.gate_proj.weight",
+ "src": "model.layers.17.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.mlp.up_proj.weight",
+ "src": "model.layers.17.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.33.post_attention_layernorm.weight",
+ "src": "model.layers.17.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 33,
+ 17,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.input_layernorm.weight",
+ "src": "model.layers.18.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.A_log",
+ "src": "model.layers.18.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.conv1d.weight",
+ "src": "model.layers.18.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.dt_bias",
+ "src": "model.layers.18.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.in_proj_a.weight",
+ "src": "model.layers.18.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.in_proj_b.weight",
+ "src": "model.layers.18.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.18.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.in_proj_z.weight",
+ "src": "model.layers.18.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.norm.weight",
+ "src": "model.layers.18.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.linear_attn.out_proj.weight",
+ "src": "model.layers.18.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.mlp.down_proj.weight",
+ "src": "model.layers.18.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.mlp.gate_proj.weight",
+ "src": "model.layers.18.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.mlp.up_proj.weight",
+ "src": "model.layers.18.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.34.post_attention_layernorm.weight",
+ "src": "model.layers.18.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 34,
+ 18,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.input_layernorm.weight",
+ "src": "model.layers.19.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.mlp.down_proj.weight",
+ "src": "model.layers.19.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.mlp.gate_proj.weight",
+ "src": "model.layers.19.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.mlp.up_proj.weight",
+ "src": "model.layers.19.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.post_attention_layernorm.weight",
+ "src": "model.layers.19.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.self_attn.k_norm.weight",
+ "src": "model.layers.19.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.self_attn.k_proj.weight",
+ "src": "model.layers.19.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.self_attn.o_proj.weight",
+ "src": "model.layers.19.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.self_attn.q_norm.weight",
+ "src": "model.layers.19.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.self_attn.q_proj.weight",
+ "src": "model.layers.19.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.35.self_attn.v_proj.weight",
+ "src": "model.layers.19.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 35,
+ 19,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.36.input_layernorm.weight",
+ "src": "model.layers.16.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.A_log",
+ "src": "model.layers.16.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.conv1d.weight",
+ "src": "model.layers.16.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.dt_bias",
+ "src": "model.layers.16.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.in_proj_a.weight",
+ "src": "model.layers.16.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.in_proj_b.weight",
+ "src": "model.layers.16.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.16.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.in_proj_z.weight",
+ "src": "model.layers.16.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.norm.weight",
+ "src": "model.layers.16.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.linear_attn.out_proj.weight",
+ "src": "model.layers.16.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.mlp.down_proj.weight",
+ "src": "model.layers.16.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.mlp.gate_proj.weight",
+ "src": "model.layers.16.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.mlp.up_proj.weight",
+ "src": "model.layers.16.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.36.post_attention_layernorm.weight",
+ "src": "model.layers.16.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 36,
+ 16,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.input_layernorm.weight",
+ "src": "model.layers.17.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.A_log",
+ "src": "model.layers.17.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.conv1d.weight",
+ "src": "model.layers.17.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.dt_bias",
+ "src": "model.layers.17.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.in_proj_a.weight",
+ "src": "model.layers.17.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.in_proj_b.weight",
+ "src": "model.layers.17.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.17.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.in_proj_z.weight",
+ "src": "model.layers.17.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.norm.weight",
+ "src": "model.layers.17.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.linear_attn.out_proj.weight",
+ "src": "model.layers.17.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.mlp.down_proj.weight",
+ "src": "model.layers.17.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.mlp.gate_proj.weight",
+ "src": "model.layers.17.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.mlp.up_proj.weight",
+ "src": "model.layers.17.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.37.post_attention_layernorm.weight",
+ "src": "model.layers.17.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 37,
+ 17,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.input_layernorm.weight",
+ "src": "model.layers.18.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.A_log",
+ "src": "model.layers.18.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.conv1d.weight",
+ "src": "model.layers.18.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.dt_bias",
+ "src": "model.layers.18.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.in_proj_a.weight",
+ "src": "model.layers.18.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.in_proj_b.weight",
+ "src": "model.layers.18.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.18.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.in_proj_z.weight",
+ "src": "model.layers.18.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.norm.weight",
+ "src": "model.layers.18.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.linear_attn.out_proj.weight",
+ "src": "model.layers.18.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.mlp.down_proj.weight",
+ "src": "model.layers.18.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.mlp.gate_proj.weight",
+ "src": "model.layers.18.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.mlp.up_proj.weight",
+ "src": "model.layers.18.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.38.post_attention_layernorm.weight",
+ "src": "model.layers.18.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 38,
+ 18,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.input_layernorm.weight",
+ "src": "model.layers.19.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.mlp.down_proj.weight",
+ "src": "model.layers.19.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.mlp.gate_proj.weight",
+ "src": "model.layers.19.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.mlp.up_proj.weight",
+ "src": "model.layers.19.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.post_attention_layernorm.weight",
+ "src": "model.layers.19.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.self_attn.k_norm.weight",
+ "src": "model.layers.19.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.self_attn.k_proj.weight",
+ "src": "model.layers.19.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.self_attn.o_proj.weight",
+ "src": "model.layers.19.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.self_attn.q_norm.weight",
+ "src": "model.layers.19.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.self_attn.q_proj.weight",
+ "src": "model.layers.19.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.39.self_attn.v_proj.weight",
+ "src": "model.layers.19.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 39,
+ 19,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.40.input_layernorm.weight",
+ "src": "model.layers.20.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.A_log",
+ "src": "model.layers.20.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.conv1d.weight",
+ "src": "model.layers.20.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.dt_bias",
+ "src": "model.layers.20.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.in_proj_a.weight",
+ "src": "model.layers.20.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.in_proj_b.weight",
+ "src": "model.layers.20.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.20.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.in_proj_z.weight",
+ "src": "model.layers.20.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.norm.weight",
+ "src": "model.layers.20.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.linear_attn.out_proj.weight",
+ "src": "model.layers.20.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.mlp.down_proj.weight",
+ "src": "model.layers.20.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.mlp.gate_proj.weight",
+ "src": "model.layers.20.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.mlp.up_proj.weight",
+ "src": "model.layers.20.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.40.post_attention_layernorm.weight",
+ "src": "model.layers.20.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 40,
+ 20,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.input_layernorm.weight",
+ "src": "model.layers.21.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.A_log",
+ "src": "model.layers.21.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.conv1d.weight",
+ "src": "model.layers.21.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.dt_bias",
+ "src": "model.layers.21.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.in_proj_a.weight",
+ "src": "model.layers.21.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.in_proj_b.weight",
+ "src": "model.layers.21.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.21.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.in_proj_z.weight",
+ "src": "model.layers.21.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.norm.weight",
+ "src": "model.layers.21.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.linear_attn.out_proj.weight",
+ "src": "model.layers.21.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.mlp.down_proj.weight",
+ "src": "model.layers.21.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.mlp.gate_proj.weight",
+ "src": "model.layers.21.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.mlp.up_proj.weight",
+ "src": "model.layers.21.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.41.post_attention_layernorm.weight",
+ "src": "model.layers.21.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 41,
+ 21,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.input_layernorm.weight",
+ "src": "model.layers.22.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.A_log",
+ "src": "model.layers.22.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.conv1d.weight",
+ "src": "model.layers.22.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.dt_bias",
+ "src": "model.layers.22.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.in_proj_a.weight",
+ "src": "model.layers.22.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.in_proj_b.weight",
+ "src": "model.layers.22.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.22.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.in_proj_z.weight",
+ "src": "model.layers.22.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.norm.weight",
+ "src": "model.layers.22.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.linear_attn.out_proj.weight",
+ "src": "model.layers.22.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.mlp.down_proj.weight",
+ "src": "model.layers.22.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.mlp.gate_proj.weight",
+ "src": "model.layers.22.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.mlp.up_proj.weight",
+ "src": "model.layers.22.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.42.post_attention_layernorm.weight",
+ "src": "model.layers.22.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 42,
+ 22,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.input_layernorm.weight",
+ "src": "model.layers.23.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.mlp.down_proj.weight",
+ "src": "model.layers.23.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.mlp.gate_proj.weight",
+ "src": "model.layers.23.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.mlp.up_proj.weight",
+ "src": "model.layers.23.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.post_attention_layernorm.weight",
+ "src": "model.layers.23.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.self_attn.k_norm.weight",
+ "src": "model.layers.23.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.self_attn.k_proj.weight",
+ "src": "model.layers.23.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.self_attn.o_proj.weight",
+ "src": "model.layers.23.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.self_attn.q_norm.weight",
+ "src": "model.layers.23.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.self_attn.q_proj.weight",
+ "src": "model.layers.23.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.43.self_attn.v_proj.weight",
+ "src": "model.layers.23.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 43,
+ 23,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.44.input_layernorm.weight",
+ "src": "model.layers.20.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.A_log",
+ "src": "model.layers.20.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.conv1d.weight",
+ "src": "model.layers.20.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.dt_bias",
+ "src": "model.layers.20.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.in_proj_a.weight",
+ "src": "model.layers.20.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.in_proj_b.weight",
+ "src": "model.layers.20.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.20.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.in_proj_z.weight",
+ "src": "model.layers.20.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.norm.weight",
+ "src": "model.layers.20.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.linear_attn.out_proj.weight",
+ "src": "model.layers.20.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.mlp.down_proj.weight",
+ "src": "model.layers.20.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.mlp.gate_proj.weight",
+ "src": "model.layers.20.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.mlp.up_proj.weight",
+ "src": "model.layers.20.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.44.post_attention_layernorm.weight",
+ "src": "model.layers.20.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 44,
+ 20,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.input_layernorm.weight",
+ "src": "model.layers.21.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.A_log",
+ "src": "model.layers.21.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.conv1d.weight",
+ "src": "model.layers.21.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.dt_bias",
+ "src": "model.layers.21.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.in_proj_a.weight",
+ "src": "model.layers.21.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.in_proj_b.weight",
+ "src": "model.layers.21.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.21.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.in_proj_z.weight",
+ "src": "model.layers.21.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.norm.weight",
+ "src": "model.layers.21.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.linear_attn.out_proj.weight",
+ "src": "model.layers.21.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.mlp.down_proj.weight",
+ "src": "model.layers.21.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.mlp.gate_proj.weight",
+ "src": "model.layers.21.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.mlp.up_proj.weight",
+ "src": "model.layers.21.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.45.post_attention_layernorm.weight",
+ "src": "model.layers.21.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 45,
+ 21,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.input_layernorm.weight",
+ "src": "model.layers.22.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.A_log",
+ "src": "model.layers.22.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.conv1d.weight",
+ "src": "model.layers.22.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.dt_bias",
+ "src": "model.layers.22.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.in_proj_a.weight",
+ "src": "model.layers.22.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.in_proj_b.weight",
+ "src": "model.layers.22.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.22.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.in_proj_z.weight",
+ "src": "model.layers.22.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.norm.weight",
+ "src": "model.layers.22.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.linear_attn.out_proj.weight",
+ "src": "model.layers.22.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.mlp.down_proj.weight",
+ "src": "model.layers.22.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.mlp.gate_proj.weight",
+ "src": "model.layers.22.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.mlp.up_proj.weight",
+ "src": "model.layers.22.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.46.post_attention_layernorm.weight",
+ "src": "model.layers.22.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 46,
+ 22,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.input_layernorm.weight",
+ "src": "model.layers.23.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.mlp.down_proj.weight",
+ "src": "model.layers.23.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.mlp.gate_proj.weight",
+ "src": "model.layers.23.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.mlp.up_proj.weight",
+ "src": "model.layers.23.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.post_attention_layernorm.weight",
+ "src": "model.layers.23.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.self_attn.k_norm.weight",
+ "src": "model.layers.23.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.self_attn.k_proj.weight",
+ "src": "model.layers.23.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.self_attn.o_proj.weight",
+ "src": "model.layers.23.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.self_attn.q_norm.weight",
+ "src": "model.layers.23.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.self_attn.q_proj.weight",
+ "src": "model.layers.23.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.47.self_attn.v_proj.weight",
+ "src": "model.layers.23.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 47,
+ 23,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.48.input_layernorm.weight",
+ "src": "model.layers.24.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.A_log",
+ "src": "model.layers.24.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.conv1d.weight",
+ "src": "model.layers.24.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.dt_bias",
+ "src": "model.layers.24.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.in_proj_a.weight",
+ "src": "model.layers.24.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.in_proj_b.weight",
+ "src": "model.layers.24.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.24.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.in_proj_z.weight",
+ "src": "model.layers.24.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.norm.weight",
+ "src": "model.layers.24.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.linear_attn.out_proj.weight",
+ "src": "model.layers.24.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.mlp.down_proj.weight",
+ "src": "model.layers.24.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.mlp.gate_proj.weight",
+ "src": "model.layers.24.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.mlp.up_proj.weight",
+ "src": "model.layers.24.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.48.post_attention_layernorm.weight",
+ "src": "model.layers.24.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 48,
+ 24,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.input_layernorm.weight",
+ "src": "model.layers.25.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.A_log",
+ "src": "model.layers.25.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.conv1d.weight",
+ "src": "model.layers.25.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.dt_bias",
+ "src": "model.layers.25.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.in_proj_a.weight",
+ "src": "model.layers.25.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.in_proj_b.weight",
+ "src": "model.layers.25.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.25.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.in_proj_z.weight",
+ "src": "model.layers.25.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.norm.weight",
+ "src": "model.layers.25.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.linear_attn.out_proj.weight",
+ "src": "model.layers.25.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.mlp.down_proj.weight",
+ "src": "model.layers.25.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.mlp.gate_proj.weight",
+ "src": "model.layers.25.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.mlp.up_proj.weight",
+ "src": "model.layers.25.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.49.post_attention_layernorm.weight",
+ "src": "model.layers.25.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 49,
+ 25,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.input_layernorm.weight",
+ "src": "model.layers.26.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.A_log",
+ "src": "model.layers.26.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.conv1d.weight",
+ "src": "model.layers.26.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.dt_bias",
+ "src": "model.layers.26.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.in_proj_a.weight",
+ "src": "model.layers.26.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.in_proj_b.weight",
+ "src": "model.layers.26.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.26.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.in_proj_z.weight",
+ "src": "model.layers.26.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.norm.weight",
+ "src": "model.layers.26.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.linear_attn.out_proj.weight",
+ "src": "model.layers.26.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.mlp.down_proj.weight",
+ "src": "model.layers.26.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.mlp.gate_proj.weight",
+ "src": "model.layers.26.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.mlp.up_proj.weight",
+ "src": "model.layers.26.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.50.post_attention_layernorm.weight",
+ "src": "model.layers.26.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 50,
+ 26,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.input_layernorm.weight",
+ "src": "model.layers.27.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.mlp.down_proj.weight",
+ "src": "model.layers.27.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.mlp.gate_proj.weight",
+ "src": "model.layers.27.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.mlp.up_proj.weight",
+ "src": "model.layers.27.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.post_attention_layernorm.weight",
+ "src": "model.layers.27.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.self_attn.k_norm.weight",
+ "src": "model.layers.27.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.self_attn.k_proj.weight",
+ "src": "model.layers.27.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.self_attn.o_proj.weight",
+ "src": "model.layers.27.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.self_attn.q_norm.weight",
+ "src": "model.layers.27.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.self_attn.q_proj.weight",
+ "src": "model.layers.27.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.51.self_attn.v_proj.weight",
+ "src": "model.layers.27.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 51,
+ 27,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.52.input_layernorm.weight",
+ "src": "model.layers.24.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.A_log",
+ "src": "model.layers.24.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.conv1d.weight",
+ "src": "model.layers.24.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.dt_bias",
+ "src": "model.layers.24.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.in_proj_a.weight",
+ "src": "model.layers.24.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.in_proj_b.weight",
+ "src": "model.layers.24.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.24.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.in_proj_z.weight",
+ "src": "model.layers.24.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.norm.weight",
+ "src": "model.layers.24.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.linear_attn.out_proj.weight",
+ "src": "model.layers.24.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.mlp.down_proj.weight",
+ "src": "model.layers.24.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.mlp.gate_proj.weight",
+ "src": "model.layers.24.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.mlp.up_proj.weight",
+ "src": "model.layers.24.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.52.post_attention_layernorm.weight",
+ "src": "model.layers.24.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 52,
+ 24,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.input_layernorm.weight",
+ "src": "model.layers.25.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.A_log",
+ "src": "model.layers.25.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.conv1d.weight",
+ "src": "model.layers.25.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.dt_bias",
+ "src": "model.layers.25.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.in_proj_a.weight",
+ "src": "model.layers.25.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.in_proj_b.weight",
+ "src": "model.layers.25.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.25.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.in_proj_z.weight",
+ "src": "model.layers.25.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.norm.weight",
+ "src": "model.layers.25.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.linear_attn.out_proj.weight",
+ "src": "model.layers.25.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.mlp.down_proj.weight",
+ "src": "model.layers.25.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.mlp.gate_proj.weight",
+ "src": "model.layers.25.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.mlp.up_proj.weight",
+ "src": "model.layers.25.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.53.post_attention_layernorm.weight",
+ "src": "model.layers.25.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 53,
+ 25,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.input_layernorm.weight",
+ "src": "model.layers.26.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.A_log",
+ "src": "model.layers.26.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.conv1d.weight",
+ "src": "model.layers.26.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.dt_bias",
+ "src": "model.layers.26.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.in_proj_a.weight",
+ "src": "model.layers.26.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.in_proj_b.weight",
+ "src": "model.layers.26.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.26.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.in_proj_z.weight",
+ "src": "model.layers.26.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.norm.weight",
+ "src": "model.layers.26.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.linear_attn.out_proj.weight",
+ "src": "model.layers.26.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.mlp.down_proj.weight",
+ "src": "model.layers.26.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.mlp.gate_proj.weight",
+ "src": "model.layers.26.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.mlp.up_proj.weight",
+ "src": "model.layers.26.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.54.post_attention_layernorm.weight",
+ "src": "model.layers.26.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 54,
+ 26,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.input_layernorm.weight",
+ "src": "model.layers.27.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.mlp.down_proj.weight",
+ "src": "model.layers.27.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.mlp.gate_proj.weight",
+ "src": "model.layers.27.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.mlp.up_proj.weight",
+ "src": "model.layers.27.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.post_attention_layernorm.weight",
+ "src": "model.layers.27.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.self_attn.k_norm.weight",
+ "src": "model.layers.27.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.self_attn.k_proj.weight",
+ "src": "model.layers.27.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.self_attn.o_proj.weight",
+ "src": "model.layers.27.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.self_attn.q_norm.weight",
+ "src": "model.layers.27.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.self_attn.q_proj.weight",
+ "src": "model.layers.27.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.55.self_attn.v_proj.weight",
+ "src": "model.layers.27.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 55,
+ 27,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.56.input_layernorm.weight",
+ "src": "model.layers.28.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.A_log",
+ "src": "model.layers.28.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.conv1d.weight",
+ "src": "model.layers.28.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.dt_bias",
+ "src": "model.layers.28.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.in_proj_a.weight",
+ "src": "model.layers.28.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.in_proj_b.weight",
+ "src": "model.layers.28.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.28.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.in_proj_z.weight",
+ "src": "model.layers.28.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.norm.weight",
+ "src": "model.layers.28.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.linear_attn.out_proj.weight",
+ "src": "model.layers.28.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.mlp.down_proj.weight",
+ "src": "model.layers.28.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.mlp.gate_proj.weight",
+ "src": "model.layers.28.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.mlp.up_proj.weight",
+ "src": "model.layers.28.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.56.post_attention_layernorm.weight",
+ "src": "model.layers.28.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 56,
+ 28,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.input_layernorm.weight",
+ "src": "model.layers.29.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.A_log",
+ "src": "model.layers.29.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.conv1d.weight",
+ "src": "model.layers.29.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.dt_bias",
+ "src": "model.layers.29.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.in_proj_a.weight",
+ "src": "model.layers.29.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.in_proj_b.weight",
+ "src": "model.layers.29.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.29.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.in_proj_z.weight",
+ "src": "model.layers.29.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.norm.weight",
+ "src": "model.layers.29.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.linear_attn.out_proj.weight",
+ "src": "model.layers.29.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.mlp.down_proj.weight",
+ "src": "model.layers.29.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.mlp.gate_proj.weight",
+ "src": "model.layers.29.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.mlp.up_proj.weight",
+ "src": "model.layers.29.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.57.post_attention_layernorm.weight",
+ "src": "model.layers.29.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 57,
+ 29,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.input_layernorm.weight",
+ "src": "model.layers.30.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.A_log",
+ "src": "model.layers.30.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.conv1d.weight",
+ "src": "model.layers.30.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.dt_bias",
+ "src": "model.layers.30.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.in_proj_a.weight",
+ "src": "model.layers.30.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.in_proj_b.weight",
+ "src": "model.layers.30.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.30.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.in_proj_z.weight",
+ "src": "model.layers.30.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.norm.weight",
+ "src": "model.layers.30.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.linear_attn.out_proj.weight",
+ "src": "model.layers.30.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.mlp.down_proj.weight",
+ "src": "model.layers.30.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.mlp.gate_proj.weight",
+ "src": "model.layers.30.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.mlp.up_proj.weight",
+ "src": "model.layers.30.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.58.post_attention_layernorm.weight",
+ "src": "model.layers.30.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 58,
+ 30,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.input_layernorm.weight",
+ "src": "model.layers.31.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.mlp.down_proj.weight",
+ "src": "model.layers.31.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.mlp.gate_proj.weight",
+ "src": "model.layers.31.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.mlp.up_proj.weight",
+ "src": "model.layers.31.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.post_attention_layernorm.weight",
+ "src": "model.layers.31.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.self_attn.k_norm.weight",
+ "src": "model.layers.31.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.self_attn.k_proj.weight",
+ "src": "model.layers.31.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.self_attn.o_proj.weight",
+ "src": "model.layers.31.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.self_attn.q_norm.weight",
+ "src": "model.layers.31.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.self_attn.q_proj.weight",
+ "src": "model.layers.31.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.59.self_attn.v_proj.weight",
+ "src": "model.layers.31.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 59,
+ 31,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.60.input_layernorm.weight",
+ "src": "model.layers.28.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.A_log",
+ "src": "model.layers.28.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.conv1d.weight",
+ "src": "model.layers.28.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.dt_bias",
+ "src": "model.layers.28.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.in_proj_a.weight",
+ "src": "model.layers.28.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.in_proj_b.weight",
+ "src": "model.layers.28.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.28.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.in_proj_z.weight",
+ "src": "model.layers.28.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.norm.weight",
+ "src": "model.layers.28.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.linear_attn.out_proj.weight",
+ "src": "model.layers.28.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.mlp.down_proj.weight",
+ "src": "model.layers.28.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.mlp.gate_proj.weight",
+ "src": "model.layers.28.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.mlp.up_proj.weight",
+ "src": "model.layers.28.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.60.post_attention_layernorm.weight",
+ "src": "model.layers.28.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 60,
+ 28,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.input_layernorm.weight",
+ "src": "model.layers.29.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.A_log",
+ "src": "model.layers.29.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.conv1d.weight",
+ "src": "model.layers.29.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.dt_bias",
+ "src": "model.layers.29.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.in_proj_a.weight",
+ "src": "model.layers.29.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.in_proj_b.weight",
+ "src": "model.layers.29.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.29.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.in_proj_z.weight",
+ "src": "model.layers.29.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.norm.weight",
+ "src": "model.layers.29.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.linear_attn.out_proj.weight",
+ "src": "model.layers.29.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.mlp.down_proj.weight",
+ "src": "model.layers.29.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.mlp.gate_proj.weight",
+ "src": "model.layers.29.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.mlp.up_proj.weight",
+ "src": "model.layers.29.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.61.post_attention_layernorm.weight",
+ "src": "model.layers.29.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 61,
+ 29,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.input_layernorm.weight",
+ "src": "model.layers.30.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.A_log",
+ "src": "model.layers.30.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.conv1d.weight",
+ "src": "model.layers.30.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.dt_bias",
+ "src": "model.layers.30.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.in_proj_a.weight",
+ "src": "model.layers.30.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.in_proj_b.weight",
+ "src": "model.layers.30.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.30.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.in_proj_z.weight",
+ "src": "model.layers.30.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.norm.weight",
+ "src": "model.layers.30.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.linear_attn.out_proj.weight",
+ "src": "model.layers.30.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.mlp.down_proj.weight",
+ "src": "model.layers.30.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.mlp.gate_proj.weight",
+ "src": "model.layers.30.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.mlp.up_proj.weight",
+ "src": "model.layers.30.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.62.post_attention_layernorm.weight",
+ "src": "model.layers.30.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 62,
+ 30,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.input_layernorm.weight",
+ "src": "model.layers.31.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.mlp.down_proj.weight",
+ "src": "model.layers.31.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.mlp.gate_proj.weight",
+ "src": "model.layers.31.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.mlp.up_proj.weight",
+ "src": "model.layers.31.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.post_attention_layernorm.weight",
+ "src": "model.layers.31.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.self_attn.k_norm.weight",
+ "src": "model.layers.31.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.self_attn.k_proj.weight",
+ "src": "model.layers.31.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.self_attn.o_proj.weight",
+ "src": "model.layers.31.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.self_attn.q_norm.weight",
+ "src": "model.layers.31.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.self_attn.q_proj.weight",
+ "src": "model.layers.31.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.63.self_attn.v_proj.weight",
+ "src": "model.layers.31.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 63,
+ 31,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.64.input_layernorm.weight",
+ "src": "model.layers.32.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.A_log",
+ "src": "model.layers.32.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.conv1d.weight",
+ "src": "model.layers.32.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.dt_bias",
+ "src": "model.layers.32.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.in_proj_a.weight",
+ "src": "model.layers.32.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.in_proj_b.weight",
+ "src": "model.layers.32.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.32.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.in_proj_z.weight",
+ "src": "model.layers.32.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.norm.weight",
+ "src": "model.layers.32.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.linear_attn.out_proj.weight",
+ "src": "model.layers.32.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.mlp.down_proj.weight",
+ "src": "model.layers.32.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.mlp.gate_proj.weight",
+ "src": "model.layers.32.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.mlp.up_proj.weight",
+ "src": "model.layers.32.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.64.post_attention_layernorm.weight",
+ "src": "model.layers.32.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 64,
+ 32,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.input_layernorm.weight",
+ "src": "model.layers.33.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.A_log",
+ "src": "model.layers.33.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.conv1d.weight",
+ "src": "model.layers.33.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.dt_bias",
+ "src": "model.layers.33.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.in_proj_a.weight",
+ "src": "model.layers.33.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.in_proj_b.weight",
+ "src": "model.layers.33.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.33.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.in_proj_z.weight",
+ "src": "model.layers.33.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.norm.weight",
+ "src": "model.layers.33.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.linear_attn.out_proj.weight",
+ "src": "model.layers.33.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.mlp.down_proj.weight",
+ "src": "model.layers.33.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.mlp.gate_proj.weight",
+ "src": "model.layers.33.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.mlp.up_proj.weight",
+ "src": "model.layers.33.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.65.post_attention_layernorm.weight",
+ "src": "model.layers.33.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 65,
+ 33,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.input_layernorm.weight",
+ "src": "model.layers.34.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.A_log",
+ "src": "model.layers.34.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.conv1d.weight",
+ "src": "model.layers.34.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.dt_bias",
+ "src": "model.layers.34.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.in_proj_a.weight",
+ "src": "model.layers.34.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.in_proj_b.weight",
+ "src": "model.layers.34.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.34.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.in_proj_z.weight",
+ "src": "model.layers.34.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.norm.weight",
+ "src": "model.layers.34.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.linear_attn.out_proj.weight",
+ "src": "model.layers.34.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.mlp.down_proj.weight",
+ "src": "model.layers.34.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.mlp.gate_proj.weight",
+ "src": "model.layers.34.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.mlp.up_proj.weight",
+ "src": "model.layers.34.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.66.post_attention_layernorm.weight",
+ "src": "model.layers.34.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 66,
+ 34,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.input_layernorm.weight",
+ "src": "model.layers.35.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.mlp.down_proj.weight",
+ "src": "model.layers.35.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.mlp.gate_proj.weight",
+ "src": "model.layers.35.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.mlp.up_proj.weight",
+ "src": "model.layers.35.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.post_attention_layernorm.weight",
+ "src": "model.layers.35.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.self_attn.k_norm.weight",
+ "src": "model.layers.35.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.self_attn.k_proj.weight",
+ "src": "model.layers.35.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.self_attn.o_proj.weight",
+ "src": "model.layers.35.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.self_attn.q_norm.weight",
+ "src": "model.layers.35.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.self_attn.q_proj.weight",
+ "src": "model.layers.35.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.67.self_attn.v_proj.weight",
+ "src": "model.layers.35.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 67,
+ 35,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.68.input_layernorm.weight",
+ "src": "model.layers.32.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.A_log",
+ "src": "model.layers.32.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.conv1d.weight",
+ "src": "model.layers.32.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.dt_bias",
+ "src": "model.layers.32.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.in_proj_a.weight",
+ "src": "model.layers.32.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.in_proj_b.weight",
+ "src": "model.layers.32.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.32.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.in_proj_z.weight",
+ "src": "model.layers.32.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.norm.weight",
+ "src": "model.layers.32.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.linear_attn.out_proj.weight",
+ "src": "model.layers.32.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.mlp.down_proj.weight",
+ "src": "model.layers.32.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.mlp.gate_proj.weight",
+ "src": "model.layers.32.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.mlp.up_proj.weight",
+ "src": "model.layers.32.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.68.post_attention_layernorm.weight",
+ "src": "model.layers.32.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 68,
+ 32,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.input_layernorm.weight",
+ "src": "model.layers.33.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.A_log",
+ "src": "model.layers.33.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.conv1d.weight",
+ "src": "model.layers.33.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.dt_bias",
+ "src": "model.layers.33.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.in_proj_a.weight",
+ "src": "model.layers.33.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.in_proj_b.weight",
+ "src": "model.layers.33.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.33.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.in_proj_z.weight",
+ "src": "model.layers.33.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.norm.weight",
+ "src": "model.layers.33.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.linear_attn.out_proj.weight",
+ "src": "model.layers.33.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.mlp.down_proj.weight",
+ "src": "model.layers.33.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.mlp.gate_proj.weight",
+ "src": "model.layers.33.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.mlp.up_proj.weight",
+ "src": "model.layers.33.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.69.post_attention_layernorm.weight",
+ "src": "model.layers.33.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 69,
+ 33,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.input_layernorm.weight",
+ "src": "model.layers.34.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.A_log",
+ "src": "model.layers.34.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.conv1d.weight",
+ "src": "model.layers.34.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.dt_bias",
+ "src": "model.layers.34.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.in_proj_a.weight",
+ "src": "model.layers.34.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.in_proj_b.weight",
+ "src": "model.layers.34.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.34.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.in_proj_z.weight",
+ "src": "model.layers.34.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.norm.weight",
+ "src": "model.layers.34.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.linear_attn.out_proj.weight",
+ "src": "model.layers.34.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.mlp.down_proj.weight",
+ "src": "model.layers.34.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.mlp.gate_proj.weight",
+ "src": "model.layers.34.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.mlp.up_proj.weight",
+ "src": "model.layers.34.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.70.post_attention_layernorm.weight",
+ "src": "model.layers.34.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 70,
+ 34,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.input_layernorm.weight",
+ "src": "model.layers.35.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.mlp.down_proj.weight",
+ "src": "model.layers.35.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.mlp.gate_proj.weight",
+ "src": "model.layers.35.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.mlp.up_proj.weight",
+ "src": "model.layers.35.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.post_attention_layernorm.weight",
+ "src": "model.layers.35.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.self_attn.k_norm.weight",
+ "src": "model.layers.35.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.self_attn.k_proj.weight",
+ "src": "model.layers.35.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.self_attn.o_proj.weight",
+ "src": "model.layers.35.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.self_attn.q_norm.weight",
+ "src": "model.layers.35.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.self_attn.q_proj.weight",
+ "src": "model.layers.35.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.71.self_attn.v_proj.weight",
+ "src": "model.layers.35.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 71,
+ 35,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.72.input_layernorm.weight",
+ "src": "model.layers.36.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.A_log",
+ "src": "model.layers.36.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.conv1d.weight",
+ "src": "model.layers.36.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.dt_bias",
+ "src": "model.layers.36.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.in_proj_a.weight",
+ "src": "model.layers.36.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.in_proj_b.weight",
+ "src": "model.layers.36.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.36.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.in_proj_z.weight",
+ "src": "model.layers.36.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.norm.weight",
+ "src": "model.layers.36.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.linear_attn.out_proj.weight",
+ "src": "model.layers.36.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.mlp.down_proj.weight",
+ "src": "model.layers.36.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.mlp.gate_proj.weight",
+ "src": "model.layers.36.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.mlp.up_proj.weight",
+ "src": "model.layers.36.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.72.post_attention_layernorm.weight",
+ "src": "model.layers.36.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 72,
+ 36,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.input_layernorm.weight",
+ "src": "model.layers.37.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.A_log",
+ "src": "model.layers.37.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.conv1d.weight",
+ "src": "model.layers.37.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.dt_bias",
+ "src": "model.layers.37.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.in_proj_a.weight",
+ "src": "model.layers.37.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.in_proj_b.weight",
+ "src": "model.layers.37.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.37.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.in_proj_z.weight",
+ "src": "model.layers.37.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.norm.weight",
+ "src": "model.layers.37.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.linear_attn.out_proj.weight",
+ "src": "model.layers.37.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.mlp.down_proj.weight",
+ "src": "model.layers.37.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.mlp.gate_proj.weight",
+ "src": "model.layers.37.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.mlp.up_proj.weight",
+ "src": "model.layers.37.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.73.post_attention_layernorm.weight",
+ "src": "model.layers.37.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 73,
+ 37,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.input_layernorm.weight",
+ "src": "model.layers.38.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.A_log",
+ "src": "model.layers.38.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.conv1d.weight",
+ "src": "model.layers.38.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.dt_bias",
+ "src": "model.layers.38.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.in_proj_a.weight",
+ "src": "model.layers.38.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.in_proj_b.weight",
+ "src": "model.layers.38.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.38.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.in_proj_z.weight",
+ "src": "model.layers.38.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.norm.weight",
+ "src": "model.layers.38.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.linear_attn.out_proj.weight",
+ "src": "model.layers.38.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.mlp.down_proj.weight",
+ "src": "model.layers.38.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.mlp.gate_proj.weight",
+ "src": "model.layers.38.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.mlp.up_proj.weight",
+ "src": "model.layers.38.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.74.post_attention_layernorm.weight",
+ "src": "model.layers.38.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 74,
+ 38,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.input_layernorm.weight",
+ "src": "model.layers.39.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.mlp.down_proj.weight",
+ "src": "model.layers.39.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.mlp.gate_proj.weight",
+ "src": "model.layers.39.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.mlp.up_proj.weight",
+ "src": "model.layers.39.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.post_attention_layernorm.weight",
+ "src": "model.layers.39.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.self_attn.k_norm.weight",
+ "src": "model.layers.39.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.self_attn.k_proj.weight",
+ "src": "model.layers.39.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.self_attn.o_proj.weight",
+ "src": "model.layers.39.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.self_attn.q_norm.weight",
+ "src": "model.layers.39.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.self_attn.q_proj.weight",
+ "src": "model.layers.39.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.75.self_attn.v_proj.weight",
+ "src": "model.layers.39.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 75,
+ 39,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.76.input_layernorm.weight",
+ "src": "model.layers.36.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.A_log",
+ "src": "model.layers.36.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.conv1d.weight",
+ "src": "model.layers.36.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.dt_bias",
+ "src": "model.layers.36.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.in_proj_a.weight",
+ "src": "model.layers.36.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.in_proj_b.weight",
+ "src": "model.layers.36.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.36.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.in_proj_z.weight",
+ "src": "model.layers.36.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.norm.weight",
+ "src": "model.layers.36.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.linear_attn.out_proj.weight",
+ "src": "model.layers.36.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.mlp.down_proj.weight",
+ "src": "model.layers.36.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.mlp.gate_proj.weight",
+ "src": "model.layers.36.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.mlp.up_proj.weight",
+ "src": "model.layers.36.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.76.post_attention_layernorm.weight",
+ "src": "model.layers.36.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 76,
+ 36,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.input_layernorm.weight",
+ "src": "model.layers.37.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.A_log",
+ "src": "model.layers.37.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.conv1d.weight",
+ "src": "model.layers.37.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.dt_bias",
+ "src": "model.layers.37.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.in_proj_a.weight",
+ "src": "model.layers.37.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.in_proj_b.weight",
+ "src": "model.layers.37.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.37.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.in_proj_z.weight",
+ "src": "model.layers.37.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.norm.weight",
+ "src": "model.layers.37.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.linear_attn.out_proj.weight",
+ "src": "model.layers.37.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.mlp.down_proj.weight",
+ "src": "model.layers.37.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.mlp.gate_proj.weight",
+ "src": "model.layers.37.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.mlp.up_proj.weight",
+ "src": "model.layers.37.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.77.post_attention_layernorm.weight",
+ "src": "model.layers.37.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 77,
+ 37,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.input_layernorm.weight",
+ "src": "model.layers.38.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.A_log",
+ "src": "model.layers.38.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.conv1d.weight",
+ "src": "model.layers.38.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.dt_bias",
+ "src": "model.layers.38.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.in_proj_a.weight",
+ "src": "model.layers.38.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.in_proj_b.weight",
+ "src": "model.layers.38.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.38.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.in_proj_z.weight",
+ "src": "model.layers.38.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.norm.weight",
+ "src": "model.layers.38.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.linear_attn.out_proj.weight",
+ "src": "model.layers.38.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.mlp.down_proj.weight",
+ "src": "model.layers.38.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.mlp.gate_proj.weight",
+ "src": "model.layers.38.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.mlp.up_proj.weight",
+ "src": "model.layers.38.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.78.post_attention_layernorm.weight",
+ "src": "model.layers.38.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 78,
+ 38,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.input_layernorm.weight",
+ "src": "model.layers.39.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.mlp.down_proj.weight",
+ "src": "model.layers.39.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.mlp.gate_proj.weight",
+ "src": "model.layers.39.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.mlp.up_proj.weight",
+ "src": "model.layers.39.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.post_attention_layernorm.weight",
+ "src": "model.layers.39.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.self_attn.k_norm.weight",
+ "src": "model.layers.39.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.self_attn.k_proj.weight",
+ "src": "model.layers.39.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.self_attn.o_proj.weight",
+ "src": "model.layers.39.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.self_attn.q_norm.weight",
+ "src": "model.layers.39.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.self_attn.q_proj.weight",
+ "src": "model.layers.39.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.79.self_attn.v_proj.weight",
+ "src": "model.layers.39.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 79,
+ 39,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.80.input_layernorm.weight",
+ "src": "model.layers.40.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.A_log",
+ "src": "model.layers.40.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.conv1d.weight",
+ "src": "model.layers.40.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.dt_bias",
+ "src": "model.layers.40.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.in_proj_a.weight",
+ "src": "model.layers.40.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.in_proj_b.weight",
+ "src": "model.layers.40.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.40.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.in_proj_z.weight",
+ "src": "model.layers.40.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.norm.weight",
+ "src": "model.layers.40.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.linear_attn.out_proj.weight",
+ "src": "model.layers.40.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.mlp.down_proj.weight",
+ "src": "model.layers.40.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.mlp.gate_proj.weight",
+ "src": "model.layers.40.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.mlp.up_proj.weight",
+ "src": "model.layers.40.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.80.post_attention_layernorm.weight",
+ "src": "model.layers.40.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 80,
+ 40,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.input_layernorm.weight",
+ "src": "model.layers.41.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.A_log",
+ "src": "model.layers.41.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.conv1d.weight",
+ "src": "model.layers.41.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.dt_bias",
+ "src": "model.layers.41.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.in_proj_a.weight",
+ "src": "model.layers.41.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.in_proj_b.weight",
+ "src": "model.layers.41.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.41.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.in_proj_z.weight",
+ "src": "model.layers.41.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.norm.weight",
+ "src": "model.layers.41.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.linear_attn.out_proj.weight",
+ "src": "model.layers.41.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.mlp.down_proj.weight",
+ "src": "model.layers.41.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.mlp.gate_proj.weight",
+ "src": "model.layers.41.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.mlp.up_proj.weight",
+ "src": "model.layers.41.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.81.post_attention_layernorm.weight",
+ "src": "model.layers.41.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 81,
+ 41,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.input_layernorm.weight",
+ "src": "model.layers.42.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.A_log",
+ "src": "model.layers.42.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.conv1d.weight",
+ "src": "model.layers.42.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.dt_bias",
+ "src": "model.layers.42.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.in_proj_a.weight",
+ "src": "model.layers.42.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.in_proj_b.weight",
+ "src": "model.layers.42.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.42.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.in_proj_z.weight",
+ "src": "model.layers.42.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.norm.weight",
+ "src": "model.layers.42.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.linear_attn.out_proj.weight",
+ "src": "model.layers.42.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.mlp.down_proj.weight",
+ "src": "model.layers.42.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.mlp.gate_proj.weight",
+ "src": "model.layers.42.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.mlp.up_proj.weight",
+ "src": "model.layers.42.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.82.post_attention_layernorm.weight",
+ "src": "model.layers.42.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 82,
+ 42,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.input_layernorm.weight",
+ "src": "model.layers.43.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.mlp.down_proj.weight",
+ "src": "model.layers.43.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.mlp.gate_proj.weight",
+ "src": "model.layers.43.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.mlp.up_proj.weight",
+ "src": "model.layers.43.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.post_attention_layernorm.weight",
+ "src": "model.layers.43.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.self_attn.k_norm.weight",
+ "src": "model.layers.43.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.self_attn.k_proj.weight",
+ "src": "model.layers.43.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.self_attn.o_proj.weight",
+ "src": "model.layers.43.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.self_attn.q_norm.weight",
+ "src": "model.layers.43.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.self_attn.q_proj.weight",
+ "src": "model.layers.43.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.83.self_attn.v_proj.weight",
+ "src": "model.layers.43.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 83,
+ 43,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.84.input_layernorm.weight",
+ "src": "model.layers.40.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.A_log",
+ "src": "model.layers.40.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.conv1d.weight",
+ "src": "model.layers.40.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.dt_bias",
+ "src": "model.layers.40.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.in_proj_a.weight",
+ "src": "model.layers.40.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.in_proj_b.weight",
+ "src": "model.layers.40.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.40.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.in_proj_z.weight",
+ "src": "model.layers.40.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.norm.weight",
+ "src": "model.layers.40.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.linear_attn.out_proj.weight",
+ "src": "model.layers.40.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.mlp.down_proj.weight",
+ "src": "model.layers.40.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.mlp.gate_proj.weight",
+ "src": "model.layers.40.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.mlp.up_proj.weight",
+ "src": "model.layers.40.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.84.post_attention_layernorm.weight",
+ "src": "model.layers.40.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 84,
+ 40,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.input_layernorm.weight",
+ "src": "model.layers.41.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.A_log",
+ "src": "model.layers.41.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.conv1d.weight",
+ "src": "model.layers.41.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.dt_bias",
+ "src": "model.layers.41.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.in_proj_a.weight",
+ "src": "model.layers.41.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.in_proj_b.weight",
+ "src": "model.layers.41.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.41.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.in_proj_z.weight",
+ "src": "model.layers.41.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.norm.weight",
+ "src": "model.layers.41.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.linear_attn.out_proj.weight",
+ "src": "model.layers.41.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.mlp.down_proj.weight",
+ "src": "model.layers.41.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.mlp.gate_proj.weight",
+ "src": "model.layers.41.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.mlp.up_proj.weight",
+ "src": "model.layers.41.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.85.post_attention_layernorm.weight",
+ "src": "model.layers.41.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 85,
+ 41,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.input_layernorm.weight",
+ "src": "model.layers.42.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.A_log",
+ "src": "model.layers.42.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.conv1d.weight",
+ "src": "model.layers.42.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.dt_bias",
+ "src": "model.layers.42.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.in_proj_a.weight",
+ "src": "model.layers.42.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.in_proj_b.weight",
+ "src": "model.layers.42.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.42.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.in_proj_z.weight",
+ "src": "model.layers.42.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.norm.weight",
+ "src": "model.layers.42.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.linear_attn.out_proj.weight",
+ "src": "model.layers.42.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.mlp.down_proj.weight",
+ "src": "model.layers.42.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.mlp.gate_proj.weight",
+ "src": "model.layers.42.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.mlp.up_proj.weight",
+ "src": "model.layers.42.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.86.post_attention_layernorm.weight",
+ "src": "model.layers.42.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 86,
+ 42,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.input_layernorm.weight",
+ "src": "model.layers.43.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.mlp.down_proj.weight",
+ "src": "model.layers.43.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.mlp.gate_proj.weight",
+ "src": "model.layers.43.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.mlp.up_proj.weight",
+ "src": "model.layers.43.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.post_attention_layernorm.weight",
+ "src": "model.layers.43.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.self_attn.k_norm.weight",
+ "src": "model.layers.43.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.self_attn.k_proj.weight",
+ "src": "model.layers.43.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.self_attn.o_proj.weight",
+ "src": "model.layers.43.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.self_attn.q_norm.weight",
+ "src": "model.layers.43.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.self_attn.q_proj.weight",
+ "src": "model.layers.43.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.87.self_attn.v_proj.weight",
+ "src": "model.layers.43.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 87,
+ 43,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.88.input_layernorm.weight",
+ "src": "model.layers.44.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.A_log",
+ "src": "model.layers.44.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.conv1d.weight",
+ "src": "model.layers.44.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.dt_bias",
+ "src": "model.layers.44.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.in_proj_a.weight",
+ "src": "model.layers.44.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.in_proj_b.weight",
+ "src": "model.layers.44.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.44.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.in_proj_z.weight",
+ "src": "model.layers.44.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.norm.weight",
+ "src": "model.layers.44.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.linear_attn.out_proj.weight",
+ "src": "model.layers.44.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.mlp.down_proj.weight",
+ "src": "model.layers.44.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.mlp.gate_proj.weight",
+ "src": "model.layers.44.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.mlp.up_proj.weight",
+ "src": "model.layers.44.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.88.post_attention_layernorm.weight",
+ "src": "model.layers.44.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 88,
+ 44,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.input_layernorm.weight",
+ "src": "model.layers.45.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.A_log",
+ "src": "model.layers.45.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.conv1d.weight",
+ "src": "model.layers.45.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.dt_bias",
+ "src": "model.layers.45.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.in_proj_a.weight",
+ "src": "model.layers.45.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.in_proj_b.weight",
+ "src": "model.layers.45.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.45.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.in_proj_z.weight",
+ "src": "model.layers.45.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.norm.weight",
+ "src": "model.layers.45.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.linear_attn.out_proj.weight",
+ "src": "model.layers.45.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.mlp.down_proj.weight",
+ "src": "model.layers.45.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.mlp.gate_proj.weight",
+ "src": "model.layers.45.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.mlp.up_proj.weight",
+ "src": "model.layers.45.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.89.post_attention_layernorm.weight",
+ "src": "model.layers.45.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 89,
+ 45,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.input_layernorm.weight",
+ "src": "model.layers.46.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.A_log",
+ "src": "model.layers.46.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.conv1d.weight",
+ "src": "model.layers.46.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.dt_bias",
+ "src": "model.layers.46.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.in_proj_a.weight",
+ "src": "model.layers.46.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.in_proj_b.weight",
+ "src": "model.layers.46.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.46.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.in_proj_z.weight",
+ "src": "model.layers.46.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.norm.weight",
+ "src": "model.layers.46.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.linear_attn.out_proj.weight",
+ "src": "model.layers.46.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.mlp.down_proj.weight",
+ "src": "model.layers.46.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.mlp.gate_proj.weight",
+ "src": "model.layers.46.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.mlp.up_proj.weight",
+ "src": "model.layers.46.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.90.post_attention_layernorm.weight",
+ "src": "model.layers.46.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 90,
+ 46,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.input_layernorm.weight",
+ "src": "model.layers.47.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.mlp.down_proj.weight",
+ "src": "model.layers.47.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.mlp.gate_proj.weight",
+ "src": "model.layers.47.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.mlp.up_proj.weight",
+ "src": "model.layers.47.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.post_attention_layernorm.weight",
+ "src": "model.layers.47.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.self_attn.k_norm.weight",
+ "src": "model.layers.47.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.self_attn.k_proj.weight",
+ "src": "model.layers.47.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.self_attn.o_proj.weight",
+ "src": "model.layers.47.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.self_attn.q_norm.weight",
+ "src": "model.layers.47.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.self_attn.q_proj.weight",
+ "src": "model.layers.47.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.91.self_attn.v_proj.weight",
+ "src": "model.layers.47.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 91,
+ 47,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.92.input_layernorm.weight",
+ "src": "model.layers.44.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.A_log",
+ "src": "model.layers.44.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.conv1d.weight",
+ "src": "model.layers.44.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.dt_bias",
+ "src": "model.layers.44.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.in_proj_a.weight",
+ "src": "model.layers.44.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.in_proj_b.weight",
+ "src": "model.layers.44.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.44.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.in_proj_z.weight",
+ "src": "model.layers.44.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.norm.weight",
+ "src": "model.layers.44.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.linear_attn.out_proj.weight",
+ "src": "model.layers.44.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.mlp.down_proj.weight",
+ "src": "model.layers.44.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.mlp.gate_proj.weight",
+ "src": "model.layers.44.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.mlp.up_proj.weight",
+ "src": "model.layers.44.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.92.post_attention_layernorm.weight",
+ "src": "model.layers.44.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 92,
+ 44,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.input_layernorm.weight",
+ "src": "model.layers.45.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.A_log",
+ "src": "model.layers.45.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.conv1d.weight",
+ "src": "model.layers.45.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.dt_bias",
+ "src": "model.layers.45.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.in_proj_a.weight",
+ "src": "model.layers.45.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.in_proj_b.weight",
+ "src": "model.layers.45.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.45.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.in_proj_z.weight",
+ "src": "model.layers.45.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.norm.weight",
+ "src": "model.layers.45.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.linear_attn.out_proj.weight",
+ "src": "model.layers.45.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.mlp.down_proj.weight",
+ "src": "model.layers.45.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.mlp.gate_proj.weight",
+ "src": "model.layers.45.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.mlp.up_proj.weight",
+ "src": "model.layers.45.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.93.post_attention_layernorm.weight",
+ "src": "model.layers.45.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 93,
+ 45,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.input_layernorm.weight",
+ "src": "model.layers.46.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.A_log",
+ "src": "model.layers.46.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.conv1d.weight",
+ "src": "model.layers.46.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.dt_bias",
+ "src": "model.layers.46.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.in_proj_a.weight",
+ "src": "model.layers.46.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.in_proj_b.weight",
+ "src": "model.layers.46.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.46.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.in_proj_z.weight",
+ "src": "model.layers.46.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.norm.weight",
+ "src": "model.layers.46.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.linear_attn.out_proj.weight",
+ "src": "model.layers.46.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.mlp.down_proj.weight",
+ "src": "model.layers.46.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.mlp.gate_proj.weight",
+ "src": "model.layers.46.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.mlp.up_proj.weight",
+ "src": "model.layers.46.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.94.post_attention_layernorm.weight",
+ "src": "model.layers.46.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 94,
+ 46,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.input_layernorm.weight",
+ "src": "model.layers.47.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.mlp.down_proj.weight",
+ "src": "model.layers.47.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.mlp.gate_proj.weight",
+ "src": "model.layers.47.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.mlp.up_proj.weight",
+ "src": "model.layers.47.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.post_attention_layernorm.weight",
+ "src": "model.layers.47.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.self_attn.k_norm.weight",
+ "src": "model.layers.47.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.self_attn.k_proj.weight",
+ "src": "model.layers.47.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.self_attn.o_proj.weight",
+ "src": "model.layers.47.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.self_attn.q_norm.weight",
+ "src": "model.layers.47.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.self_attn.q_proj.weight",
+ "src": "model.layers.47.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.95.self_attn.v_proj.weight",
+ "src": "model.layers.47.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 95,
+ 47,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.96.input_layernorm.weight",
+ "src": "model.layers.48.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.A_log",
+ "src": "model.layers.48.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.conv1d.weight",
+ "src": "model.layers.48.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.dt_bias",
+ "src": "model.layers.48.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.in_proj_a.weight",
+ "src": "model.layers.48.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.in_proj_b.weight",
+ "src": "model.layers.48.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.48.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.in_proj_z.weight",
+ "src": "model.layers.48.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.norm.weight",
+ "src": "model.layers.48.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.linear_attn.out_proj.weight",
+ "src": "model.layers.48.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.mlp.down_proj.weight",
+ "src": "model.layers.48.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.mlp.gate_proj.weight",
+ "src": "model.layers.48.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.mlp.up_proj.weight",
+ "src": "model.layers.48.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.96.post_attention_layernorm.weight",
+ "src": "model.layers.48.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 96,
+ 48,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.input_layernorm.weight",
+ "src": "model.layers.49.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.A_log",
+ "src": "model.layers.49.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.conv1d.weight",
+ "src": "model.layers.49.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.dt_bias",
+ "src": "model.layers.49.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.in_proj_a.weight",
+ "src": "model.layers.49.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.in_proj_b.weight",
+ "src": "model.layers.49.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.49.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.in_proj_z.weight",
+ "src": "model.layers.49.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.norm.weight",
+ "src": "model.layers.49.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.linear_attn.out_proj.weight",
+ "src": "model.layers.49.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.mlp.down_proj.weight",
+ "src": "model.layers.49.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.mlp.gate_proj.weight",
+ "src": "model.layers.49.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.mlp.up_proj.weight",
+ "src": "model.layers.49.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.97.post_attention_layernorm.weight",
+ "src": "model.layers.49.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 97,
+ 49,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.input_layernorm.weight",
+ "src": "model.layers.50.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.A_log",
+ "src": "model.layers.50.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.conv1d.weight",
+ "src": "model.layers.50.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.dt_bias",
+ "src": "model.layers.50.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.in_proj_a.weight",
+ "src": "model.layers.50.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.in_proj_b.weight",
+ "src": "model.layers.50.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.50.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.in_proj_z.weight",
+ "src": "model.layers.50.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.norm.weight",
+ "src": "model.layers.50.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.linear_attn.out_proj.weight",
+ "src": "model.layers.50.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.mlp.down_proj.weight",
+ "src": "model.layers.50.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.mlp.gate_proj.weight",
+ "src": "model.layers.50.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.mlp.up_proj.weight",
+ "src": "model.layers.50.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.98.post_attention_layernorm.weight",
+ "src": "model.layers.50.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 98,
+ 50,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.input_layernorm.weight",
+ "src": "model.layers.51.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.mlp.down_proj.weight",
+ "src": "model.layers.51.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.mlp.gate_proj.weight",
+ "src": "model.layers.51.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.mlp.up_proj.weight",
+ "src": "model.layers.51.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.post_attention_layernorm.weight",
+ "src": "model.layers.51.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.self_attn.k_norm.weight",
+ "src": "model.layers.51.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.self_attn.k_proj.weight",
+ "src": "model.layers.51.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.self_attn.o_proj.weight",
+ "src": "model.layers.51.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.self_attn.q_norm.weight",
+ "src": "model.layers.51.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.self_attn.q_proj.weight",
+ "src": "model.layers.51.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.99.self_attn.v_proj.weight",
+ "src": "model.layers.51.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 99,
+ 51,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.100.input_layernorm.weight",
+ "src": "model.layers.48.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.A_log",
+ "src": "model.layers.48.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.conv1d.weight",
+ "src": "model.layers.48.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.dt_bias",
+ "src": "model.layers.48.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.in_proj_a.weight",
+ "src": "model.layers.48.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.in_proj_b.weight",
+ "src": "model.layers.48.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.48.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.in_proj_z.weight",
+ "src": "model.layers.48.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.norm.weight",
+ "src": "model.layers.48.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.linear_attn.out_proj.weight",
+ "src": "model.layers.48.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.mlp.down_proj.weight",
+ "src": "model.layers.48.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.mlp.gate_proj.weight",
+ "src": "model.layers.48.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.mlp.up_proj.weight",
+ "src": "model.layers.48.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.100.post_attention_layernorm.weight",
+ "src": "model.layers.48.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 100,
+ 48,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.input_layernorm.weight",
+ "src": "model.layers.49.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.A_log",
+ "src": "model.layers.49.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.conv1d.weight",
+ "src": "model.layers.49.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.dt_bias",
+ "src": "model.layers.49.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.in_proj_a.weight",
+ "src": "model.layers.49.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.in_proj_b.weight",
+ "src": "model.layers.49.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.49.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.in_proj_z.weight",
+ "src": "model.layers.49.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.norm.weight",
+ "src": "model.layers.49.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.linear_attn.out_proj.weight",
+ "src": "model.layers.49.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.mlp.down_proj.weight",
+ "src": "model.layers.49.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.mlp.gate_proj.weight",
+ "src": "model.layers.49.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.mlp.up_proj.weight",
+ "src": "model.layers.49.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.101.post_attention_layernorm.weight",
+ "src": "model.layers.49.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 101,
+ 49,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.input_layernorm.weight",
+ "src": "model.layers.50.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.A_log",
+ "src": "model.layers.50.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.conv1d.weight",
+ "src": "model.layers.50.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.dt_bias",
+ "src": "model.layers.50.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.in_proj_a.weight",
+ "src": "model.layers.50.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.in_proj_b.weight",
+ "src": "model.layers.50.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.50.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.in_proj_z.weight",
+ "src": "model.layers.50.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.norm.weight",
+ "src": "model.layers.50.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.linear_attn.out_proj.weight",
+ "src": "model.layers.50.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.mlp.down_proj.weight",
+ "src": "model.layers.50.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.mlp.gate_proj.weight",
+ "src": "model.layers.50.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.mlp.up_proj.weight",
+ "src": "model.layers.50.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.102.post_attention_layernorm.weight",
+ "src": "model.layers.50.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 102,
+ 50,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.input_layernorm.weight",
+ "src": "model.layers.51.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.mlp.down_proj.weight",
+ "src": "model.layers.51.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.mlp.gate_proj.weight",
+ "src": "model.layers.51.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.mlp.up_proj.weight",
+ "src": "model.layers.51.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.post_attention_layernorm.weight",
+ "src": "model.layers.51.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.self_attn.k_norm.weight",
+ "src": "model.layers.51.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.self_attn.k_proj.weight",
+ "src": "model.layers.51.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.self_attn.o_proj.weight",
+ "src": "model.layers.51.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.self_attn.q_norm.weight",
+ "src": "model.layers.51.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.self_attn.q_proj.weight",
+ "src": "model.layers.51.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.103.self_attn.v_proj.weight",
+ "src": "model.layers.51.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 103,
+ 51,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.104.input_layernorm.weight",
+ "src": "model.layers.52.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.A_log",
+ "src": "model.layers.52.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.conv1d.weight",
+ "src": "model.layers.52.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.dt_bias",
+ "src": "model.layers.52.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.in_proj_a.weight",
+ "src": "model.layers.52.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.in_proj_b.weight",
+ "src": "model.layers.52.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.52.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.in_proj_z.weight",
+ "src": "model.layers.52.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.norm.weight",
+ "src": "model.layers.52.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.linear_attn.out_proj.weight",
+ "src": "model.layers.52.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.mlp.down_proj.weight",
+ "src": "model.layers.52.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.mlp.gate_proj.weight",
+ "src": "model.layers.52.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.mlp.up_proj.weight",
+ "src": "model.layers.52.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.104.post_attention_layernorm.weight",
+ "src": "model.layers.52.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 104,
+ 52,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.input_layernorm.weight",
+ "src": "model.layers.53.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.A_log",
+ "src": "model.layers.53.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.conv1d.weight",
+ "src": "model.layers.53.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.dt_bias",
+ "src": "model.layers.53.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.in_proj_a.weight",
+ "src": "model.layers.53.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.in_proj_b.weight",
+ "src": "model.layers.53.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.53.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.in_proj_z.weight",
+ "src": "model.layers.53.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.norm.weight",
+ "src": "model.layers.53.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.linear_attn.out_proj.weight",
+ "src": "model.layers.53.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.mlp.down_proj.weight",
+ "src": "model.layers.53.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.mlp.gate_proj.weight",
+ "src": "model.layers.53.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.mlp.up_proj.weight",
+ "src": "model.layers.53.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.105.post_attention_layernorm.weight",
+ "src": "model.layers.53.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 105,
+ 53,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.input_layernorm.weight",
+ "src": "model.layers.54.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.A_log",
+ "src": "model.layers.54.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.conv1d.weight",
+ "src": "model.layers.54.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.dt_bias",
+ "src": "model.layers.54.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.in_proj_a.weight",
+ "src": "model.layers.54.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.in_proj_b.weight",
+ "src": "model.layers.54.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.54.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.in_proj_z.weight",
+ "src": "model.layers.54.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.norm.weight",
+ "src": "model.layers.54.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.linear_attn.out_proj.weight",
+ "src": "model.layers.54.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.mlp.down_proj.weight",
+ "src": "model.layers.54.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.mlp.gate_proj.weight",
+ "src": "model.layers.54.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.mlp.up_proj.weight",
+ "src": "model.layers.54.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.106.post_attention_layernorm.weight",
+ "src": "model.layers.54.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 106,
+ 54,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.input_layernorm.weight",
+ "src": "model.layers.55.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.mlp.down_proj.weight",
+ "src": "model.layers.55.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.mlp.gate_proj.weight",
+ "src": "model.layers.55.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.mlp.up_proj.weight",
+ "src": "model.layers.55.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.post_attention_layernorm.weight",
+ "src": "model.layers.55.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.self_attn.k_norm.weight",
+ "src": "model.layers.55.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.self_attn.k_proj.weight",
+ "src": "model.layers.55.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.self_attn.o_proj.weight",
+ "src": "model.layers.55.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.self_attn.q_norm.weight",
+ "src": "model.layers.55.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.self_attn.q_proj.weight",
+ "src": "model.layers.55.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.107.self_attn.v_proj.weight",
+ "src": "model.layers.55.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 107,
+ 55,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.108.input_layernorm.weight",
+ "src": "model.layers.52.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.A_log",
+ "src": "model.layers.52.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.conv1d.weight",
+ "src": "model.layers.52.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.dt_bias",
+ "src": "model.layers.52.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.in_proj_a.weight",
+ "src": "model.layers.52.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.in_proj_b.weight",
+ "src": "model.layers.52.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.52.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.in_proj_z.weight",
+ "src": "model.layers.52.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.norm.weight",
+ "src": "model.layers.52.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.linear_attn.out_proj.weight",
+ "src": "model.layers.52.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.mlp.down_proj.weight",
+ "src": "model.layers.52.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.mlp.gate_proj.weight",
+ "src": "model.layers.52.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.mlp.up_proj.weight",
+ "src": "model.layers.52.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.108.post_attention_layernorm.weight",
+ "src": "model.layers.52.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 108,
+ 52,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.input_layernorm.weight",
+ "src": "model.layers.53.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.A_log",
+ "src": "model.layers.53.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.conv1d.weight",
+ "src": "model.layers.53.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.dt_bias",
+ "src": "model.layers.53.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.in_proj_a.weight",
+ "src": "model.layers.53.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.in_proj_b.weight",
+ "src": "model.layers.53.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.53.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.in_proj_z.weight",
+ "src": "model.layers.53.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.norm.weight",
+ "src": "model.layers.53.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.linear_attn.out_proj.weight",
+ "src": "model.layers.53.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.mlp.down_proj.weight",
+ "src": "model.layers.53.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.mlp.gate_proj.weight",
+ "src": "model.layers.53.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.mlp.up_proj.weight",
+ "src": "model.layers.53.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.109.post_attention_layernorm.weight",
+ "src": "model.layers.53.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 109,
+ 53,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.input_layernorm.weight",
+ "src": "model.layers.54.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.A_log",
+ "src": "model.layers.54.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.conv1d.weight",
+ "src": "model.layers.54.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.dt_bias",
+ "src": "model.layers.54.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.in_proj_a.weight",
+ "src": "model.layers.54.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.in_proj_b.weight",
+ "src": "model.layers.54.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.54.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.in_proj_z.weight",
+ "src": "model.layers.54.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.norm.weight",
+ "src": "model.layers.54.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.linear_attn.out_proj.weight",
+ "src": "model.layers.54.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.mlp.down_proj.weight",
+ "src": "model.layers.54.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.mlp.gate_proj.weight",
+ "src": "model.layers.54.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.mlp.up_proj.weight",
+ "src": "model.layers.54.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.110.post_attention_layernorm.weight",
+ "src": "model.layers.54.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 110,
+ 54,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.input_layernorm.weight",
+ "src": "model.layers.55.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.mlp.down_proj.weight",
+ "src": "model.layers.55.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.mlp.gate_proj.weight",
+ "src": "model.layers.55.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.mlp.up_proj.weight",
+ "src": "model.layers.55.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.post_attention_layernorm.weight",
+ "src": "model.layers.55.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.self_attn.k_norm.weight",
+ "src": "model.layers.55.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.self_attn.k_proj.weight",
+ "src": "model.layers.55.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.self_attn.o_proj.weight",
+ "src": "model.layers.55.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.self_attn.q_norm.weight",
+ "src": "model.layers.55.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.self_attn.q_proj.weight",
+ "src": "model.layers.55.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.111.self_attn.v_proj.weight",
+ "src": "model.layers.55.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 111,
+ 55,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.112.input_layernorm.weight",
+ "src": "model.layers.56.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.A_log",
+ "src": "model.layers.56.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.conv1d.weight",
+ "src": "model.layers.56.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.dt_bias",
+ "src": "model.layers.56.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.in_proj_a.weight",
+ "src": "model.layers.56.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.in_proj_b.weight",
+ "src": "model.layers.56.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.56.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.in_proj_z.weight",
+ "src": "model.layers.56.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.norm.weight",
+ "src": "model.layers.56.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.linear_attn.out_proj.weight",
+ "src": "model.layers.56.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.mlp.down_proj.weight",
+ "src": "model.layers.56.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.mlp.gate_proj.weight",
+ "src": "model.layers.56.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.mlp.up_proj.weight",
+ "src": "model.layers.56.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.112.post_attention_layernorm.weight",
+ "src": "model.layers.56.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 112,
+ 56,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.input_layernorm.weight",
+ "src": "model.layers.57.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.A_log",
+ "src": "model.layers.57.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.conv1d.weight",
+ "src": "model.layers.57.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.dt_bias",
+ "src": "model.layers.57.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.in_proj_a.weight",
+ "src": "model.layers.57.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.in_proj_b.weight",
+ "src": "model.layers.57.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.57.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.in_proj_z.weight",
+ "src": "model.layers.57.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.norm.weight",
+ "src": "model.layers.57.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.linear_attn.out_proj.weight",
+ "src": "model.layers.57.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.mlp.down_proj.weight",
+ "src": "model.layers.57.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.mlp.gate_proj.weight",
+ "src": "model.layers.57.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.mlp.up_proj.weight",
+ "src": "model.layers.57.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.113.post_attention_layernorm.weight",
+ "src": "model.layers.57.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 113,
+ 57,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.input_layernorm.weight",
+ "src": "model.layers.58.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.A_log",
+ "src": "model.layers.58.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.conv1d.weight",
+ "src": "model.layers.58.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.dt_bias",
+ "src": "model.layers.58.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.in_proj_a.weight",
+ "src": "model.layers.58.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.in_proj_b.weight",
+ "src": "model.layers.58.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.58.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.in_proj_z.weight",
+ "src": "model.layers.58.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.norm.weight",
+ "src": "model.layers.58.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.linear_attn.out_proj.weight",
+ "src": "model.layers.58.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.mlp.down_proj.weight",
+ "src": "model.layers.58.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.mlp.gate_proj.weight",
+ "src": "model.layers.58.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.mlp.up_proj.weight",
+ "src": "model.layers.58.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.114.post_attention_layernorm.weight",
+ "src": "model.layers.58.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 114,
+ 58,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.input_layernorm.weight",
+ "src": "model.layers.59.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.mlp.down_proj.weight",
+ "src": "model.layers.59.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.mlp.gate_proj.weight",
+ "src": "model.layers.59.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.mlp.up_proj.weight",
+ "src": "model.layers.59.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.post_attention_layernorm.weight",
+ "src": "model.layers.59.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.self_attn.k_norm.weight",
+ "src": "model.layers.59.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.self_attn.k_proj.weight",
+ "src": "model.layers.59.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.self_attn.o_proj.weight",
+ "src": "model.layers.59.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.self_attn.q_norm.weight",
+ "src": "model.layers.59.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.self_attn.q_proj.weight",
+ "src": "model.layers.59.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.115.self_attn.v_proj.weight",
+ "src": "model.layers.59.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 115,
+ 59,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.116.input_layernorm.weight",
+ "src": "model.layers.56.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.A_log",
+ "src": "model.layers.56.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.conv1d.weight",
+ "src": "model.layers.56.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.dt_bias",
+ "src": "model.layers.56.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.in_proj_a.weight",
+ "src": "model.layers.56.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.in_proj_b.weight",
+ "src": "model.layers.56.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.56.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.in_proj_z.weight",
+ "src": "model.layers.56.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.norm.weight",
+ "src": "model.layers.56.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.linear_attn.out_proj.weight",
+ "src": "model.layers.56.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.mlp.down_proj.weight",
+ "src": "model.layers.56.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.mlp.gate_proj.weight",
+ "src": "model.layers.56.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.mlp.up_proj.weight",
+ "src": "model.layers.56.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.116.post_attention_layernorm.weight",
+ "src": "model.layers.56.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 116,
+ 56,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.input_layernorm.weight",
+ "src": "model.layers.57.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.A_log",
+ "src": "model.layers.57.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.conv1d.weight",
+ "src": "model.layers.57.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.dt_bias",
+ "src": "model.layers.57.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.in_proj_a.weight",
+ "src": "model.layers.57.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.in_proj_b.weight",
+ "src": "model.layers.57.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.57.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.in_proj_z.weight",
+ "src": "model.layers.57.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.norm.weight",
+ "src": "model.layers.57.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.linear_attn.out_proj.weight",
+ "src": "model.layers.57.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.mlp.down_proj.weight",
+ "src": "model.layers.57.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.mlp.gate_proj.weight",
+ "src": "model.layers.57.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.mlp.up_proj.weight",
+ "src": "model.layers.57.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.117.post_attention_layernorm.weight",
+ "src": "model.layers.57.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 117,
+ 57,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.input_layernorm.weight",
+ "src": "model.layers.58.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.A_log",
+ "src": "model.layers.58.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.conv1d.weight",
+ "src": "model.layers.58.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.dt_bias",
+ "src": "model.layers.58.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.in_proj_a.weight",
+ "src": "model.layers.58.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.in_proj_b.weight",
+ "src": "model.layers.58.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.58.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.in_proj_z.weight",
+ "src": "model.layers.58.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.norm.weight",
+ "src": "model.layers.58.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.linear_attn.out_proj.weight",
+ "src": "model.layers.58.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.mlp.down_proj.weight",
+ "src": "model.layers.58.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.mlp.gate_proj.weight",
+ "src": "model.layers.58.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.mlp.up_proj.weight",
+ "src": "model.layers.58.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.118.post_attention_layernorm.weight",
+ "src": "model.layers.58.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 118,
+ 58,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.input_layernorm.weight",
+ "src": "model.layers.59.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.mlp.down_proj.weight",
+ "src": "model.layers.59.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.mlp.gate_proj.weight",
+ "src": "model.layers.59.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.mlp.up_proj.weight",
+ "src": "model.layers.59.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.post_attention_layernorm.weight",
+ "src": "model.layers.59.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.self_attn.k_norm.weight",
+ "src": "model.layers.59.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.self_attn.k_proj.weight",
+ "src": "model.layers.59.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.self_attn.o_proj.weight",
+ "src": "model.layers.59.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.self_attn.q_norm.weight",
+ "src": "model.layers.59.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.self_attn.q_proj.weight",
+ "src": "model.layers.59.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.119.self_attn.v_proj.weight",
+ "src": "model.layers.59.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 119,
+ 59,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.120.input_layernorm.weight",
+ "src": "model.layers.60.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.A_log",
+ "src": "model.layers.60.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.conv1d.weight",
+ "src": "model.layers.60.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.dt_bias",
+ "src": "model.layers.60.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.in_proj_a.weight",
+ "src": "model.layers.60.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.in_proj_b.weight",
+ "src": "model.layers.60.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.60.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.in_proj_z.weight",
+ "src": "model.layers.60.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.norm.weight",
+ "src": "model.layers.60.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.linear_attn.out_proj.weight",
+ "src": "model.layers.60.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.mlp.down_proj.weight",
+ "src": "model.layers.60.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.mlp.gate_proj.weight",
+ "src": "model.layers.60.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.mlp.up_proj.weight",
+ "src": "model.layers.60.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.120.post_attention_layernorm.weight",
+ "src": "model.layers.60.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 120,
+ 60,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.input_layernorm.weight",
+ "src": "model.layers.61.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.A_log",
+ "src": "model.layers.61.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.conv1d.weight",
+ "src": "model.layers.61.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.dt_bias",
+ "src": "model.layers.61.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.in_proj_a.weight",
+ "src": "model.layers.61.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.in_proj_b.weight",
+ "src": "model.layers.61.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.61.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.in_proj_z.weight",
+ "src": "model.layers.61.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.norm.weight",
+ "src": "model.layers.61.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.linear_attn.out_proj.weight",
+ "src": "model.layers.61.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.mlp.down_proj.weight",
+ "src": "model.layers.61.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.mlp.gate_proj.weight",
+ "src": "model.layers.61.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.mlp.up_proj.weight",
+ "src": "model.layers.61.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.121.post_attention_layernorm.weight",
+ "src": "model.layers.61.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 121,
+ 61,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.input_layernorm.weight",
+ "src": "model.layers.62.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.A_log",
+ "src": "model.layers.62.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.conv1d.weight",
+ "src": "model.layers.62.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.dt_bias",
+ "src": "model.layers.62.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.in_proj_a.weight",
+ "src": "model.layers.62.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.in_proj_b.weight",
+ "src": "model.layers.62.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.62.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.in_proj_z.weight",
+ "src": "model.layers.62.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.norm.weight",
+ "src": "model.layers.62.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.linear_attn.out_proj.weight",
+ "src": "model.layers.62.linear_attn.out_proj.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.mlp.down_proj.weight",
+ "src": "model.layers.62.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.mlp.gate_proj.weight",
+ "src": "model.layers.62.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.mlp.up_proj.weight",
+ "src": "model.layers.62.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.122.post_attention_layernorm.weight",
+ "src": "model.layers.62.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 122,
+ 62,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.input_layernorm.weight",
+ "src": "model.layers.63.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.mlp.down_proj.weight",
+ "src": "model.layers.63.mlp.down_proj.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.mlp.gate_proj.weight",
+ "src": "model.layers.63.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.mlp.up_proj.weight",
+ "src": "model.layers.63.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.post_attention_layernorm.weight",
+ "src": "model.layers.63.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.self_attn.k_norm.weight",
+ "src": "model.layers.63.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.self_attn.k_proj.weight",
+ "src": "model.layers.63.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.self_attn.o_proj.weight",
+ "src": "model.layers.63.self_attn.o_proj.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.self_attn.q_norm.weight",
+ "src": "model.layers.63.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.self_attn.q_proj.weight",
+ "src": "model.layers.63.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.123.self_attn.v_proj.weight",
+ "src": "model.layers.63.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 123,
+ 63,
+ false
+ ]
+ },
+ {
+ "key": "model.layers.124.input_layernorm.weight",
+ "src": "model.layers.60.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.A_log",
+ "src": "model.layers.60.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.conv1d.weight",
+ "src": "model.layers.60.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.dt_bias",
+ "src": "model.layers.60.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.in_proj_a.weight",
+ "src": "model.layers.60.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.in_proj_b.weight",
+ "src": "model.layers.60.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.60.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.in_proj_z.weight",
+ "src": "model.layers.60.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.norm.weight",
+ "src": "model.layers.60.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.linear_attn.out_proj.weight",
+ "src": "model.layers.60.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.mlp.down_proj.weight",
+ "src": "model.layers.60.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.mlp.gate_proj.weight",
+ "src": "model.layers.60.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.mlp.up_proj.weight",
+ "src": "model.layers.60.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.124.post_attention_layernorm.weight",
+ "src": "model.layers.60.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 124,
+ 60,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.input_layernorm.weight",
+ "src": "model.layers.61.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.A_log",
+ "src": "model.layers.61.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.conv1d.weight",
+ "src": "model.layers.61.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.dt_bias",
+ "src": "model.layers.61.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.in_proj_a.weight",
+ "src": "model.layers.61.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.in_proj_b.weight",
+ "src": "model.layers.61.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.61.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.in_proj_z.weight",
+ "src": "model.layers.61.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.norm.weight",
+ "src": "model.layers.61.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.linear_attn.out_proj.weight",
+ "src": "model.layers.61.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.mlp.down_proj.weight",
+ "src": "model.layers.61.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.mlp.gate_proj.weight",
+ "src": "model.layers.61.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.mlp.up_proj.weight",
+ "src": "model.layers.61.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.125.post_attention_layernorm.weight",
+ "src": "model.layers.61.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 125,
+ 61,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.input_layernorm.weight",
+ "src": "model.layers.62.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.A_log",
+ "src": "model.layers.62.linear_attn.A_log",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.conv1d.weight",
+ "src": "model.layers.62.linear_attn.conv1d.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.dt_bias",
+ "src": "model.layers.62.linear_attn.dt_bias",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.in_proj_a.weight",
+ "src": "model.layers.62.linear_attn.in_proj_a.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.in_proj_b.weight",
+ "src": "model.layers.62.linear_attn.in_proj_b.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.in_proj_qkv.weight",
+ "src": "model.layers.62.linear_attn.in_proj_qkv.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.in_proj_z.weight",
+ "src": "model.layers.62.linear_attn.in_proj_z.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.norm.weight",
+ "src": "model.layers.62.linear_attn.norm.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.linear_attn.out_proj.weight",
+ "src": "model.layers.62.linear_attn.out_proj.weight",
+ "op": "zero",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.mlp.down_proj.weight",
+ "src": "model.layers.62.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.mlp.gate_proj.weight",
+ "src": "model.layers.62.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.mlp.up_proj.weight",
+ "src": "model.layers.62.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.126.post_attention_layernorm.weight",
+ "src": "model.layers.62.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 126,
+ 62,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.input_layernorm.weight",
+ "src": "model.layers.63.input_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.mlp.down_proj.weight",
+ "src": "model.layers.63.mlp.down_proj.weight",
+ "op": "zero",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.mlp.gate_proj.weight",
+ "src": "model.layers.63.mlp.gate_proj.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.mlp.up_proj.weight",
+ "src": "model.layers.63.mlp.up_proj.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.post_attention_layernorm.weight",
+ "src": "model.layers.63.post_attention_layernorm.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.self_attn.k_norm.weight",
+ "src": "model.layers.63.self_attn.k_norm.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.self_attn.k_proj.weight",
+ "src": "model.layers.63.self_attn.k_proj.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.self_attn.o_proj.weight",
+ "src": "model.layers.63.self_attn.o_proj.weight",
+ "op": "zero",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.self_attn.q_norm.weight",
+ "src": "model.layers.63.self_attn.q_norm.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.self_attn.q_proj.weight",
+ "src": "model.layers.63.self_attn.q_proj.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.layers.127.self_attn.v_proj.weight",
+ "src": "model.layers.63.self_attn.v_proj.weight",
+ "op": "copy",
+ "layer": [
+ 127,
+ 63,
+ true
+ ]
+ },
+ {
+ "key": "model.norm.weight",
+ "src": "model.norm.weight",
+ "op": "copy"
+ },
+ {
+ "key": "lm_head.weight",
+ "src": "lm_head.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.fc.weight",
+ "src": "mtp.fc.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.input_layernorm.weight",
+ "src": "mtp.layers.0.input_layernorm.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.mlp.down_proj.weight",
+ "src": "mtp.layers.0.mlp.down_proj.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.mlp.gate_proj.weight",
+ "src": "mtp.layers.0.mlp.gate_proj.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.mlp.up_proj.weight",
+ "src": "mtp.layers.0.mlp.up_proj.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.post_attention_layernorm.weight",
+ "src": "mtp.layers.0.post_attention_layernorm.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.self_attn.k_norm.weight",
+ "src": "mtp.layers.0.self_attn.k_norm.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.self_attn.k_proj.weight",
+ "src": "mtp.layers.0.self_attn.k_proj.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.self_attn.o_proj.weight",
+ "src": "mtp.layers.0.self_attn.o_proj.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.self_attn.q_norm.weight",
+ "src": "mtp.layers.0.self_attn.q_norm.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.self_attn.q_proj.weight",
+ "src": "mtp.layers.0.self_attn.q_proj.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.layers.0.self_attn.v_proj.weight",
+ "src": "mtp.layers.0.self_attn.v_proj.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.norm.weight",
+ "src": "mtp.norm.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.pre_fc_norm_embedding.weight",
+ "src": "mtp.pre_fc_norm_embedding.weight",
+ "op": "copy"
+ },
+ {
+ "key": "mtp.pre_fc_norm_hidden.weight",
+ "src": "mtp.pre_fc_norm_hidden.weight",
+ "op": "copy"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/sft_final_fp8/gate_full.json b/sft_final_fp8/gate_full.json
new file mode 100644
index 0000000000000000000000000000000000000000..fd994473314956d3469f37eadfafa60e03341fbf
--- /dev/null
+++ b/sft_final_fp8/gate_full.json
@@ -0,0 +1,81 @@
+{
+ "verdict": "PASS",
+ "expected": "exact",
+ "mini": false,
+ "compute": "fp32blocks",
+ "a": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded",
+ "b": "/mnt/nfs_share/JiRackDeltaNet_70b/expanded",
+ "layers_run": [
+ 64,
+ 128
+ ],
+ "intermediate": [
+ 26624,
+ 26624
+ ],
+ "prompts": [
+ "The capital of France is",
+ "def fibonacci(n):\n ",
+ "Столица России —"
+ ],
+ "ids": [
+ [
+ 760,
+ 6511,
+ 314,
+ 9338,
+ 369
+ ],
+ [
+ 727,
+ 73111,
+ 1393,
+ 1590,
+ 198,
+ 257
+ ],
+ [
+ 212396,
+ 168314,
+ 170312,
+ 1892
+ ]
+ ],
+ "rows": [
+ {
+ "T": 5,
+ "identical": true,
+ "max_abs": 0.0,
+ "mean_abs": 0.0,
+ "max_kl": 0.0,
+ "top1_agree": 5
+ },
+ {
+ "T": 6,
+ "identical": true,
+ "max_abs": 0.0,
+ "mean_abs": 0.0,
+ "max_kl": 0.0,
+ "top1_agree": 6
+ },
+ {
+ "T": 4,
+ "identical": true,
+ "max_abs": 0.0,
+ "mean_abs": 0.0,
+ "max_kl": 0.0,
+ "top1_agree": 4
+ }
+ ],
+ "a_info": {
+ "cached": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded/gate_logits_fp32blocks_L64.npz"
+ },
+ "b_info": {
+ "loaded": 1714,
+ "skipped": 0,
+ "unexpected": [],
+ "missing_buffers": [],
+ "seconds": 2016
+ },
+ "torch": "2.13.0+cu130"
+}
\ No newline at end of file
diff --git a/sft_final_fp8/gate_logits_fp32blocks_L128.npz b/sft_final_fp8/gate_logits_fp32blocks_L128.npz
new file mode 100644
index 0000000000000000000000000000000000000000..f27c114c7eb269402bd0dd3e1ddb0caca7507239
--- /dev/null
+++ b/sft_final_fp8/gate_logits_fp32blocks_L128.npz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a589ae8136832e11c0c408e43a560cf110f011e3ecef3b60852db63eb256d025
+size 14900556
diff --git a/sft_final_fp8/gate_logits_fp32blocks_L8.npz b/sft_final_fp8/gate_logits_fp32blocks_L8.npz
new file mode 100644
index 0000000000000000000000000000000000000000..b368f7b678cbc848a11eb979478a0eab0d1e5143
--- /dev/null
+++ b/sft_final_fp8/gate_logits_fp32blocks_L8.npz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8975717cc37cabfaffcea30896d5aaefceb526cb2d0b1a0b993830d57358e29e
+size 14900556
diff --git a/sft_final_fp8/gate_mini.json b/sft_final_fp8/gate_mini.json
new file mode 100644
index 0000000000000000000000000000000000000000..1dee6cad5b932734ff025c277058bef355c5eea5
--- /dev/null
+++ b/sft_final_fp8/gate_mini.json
@@ -0,0 +1,81 @@
+{
+ "verdict": "PASS",
+ "expected": "exact",
+ "mini": true,
+ "compute": "fp32blocks",
+ "a": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded",
+ "b": "/mnt/nfs_share/JiRackDeltaNet_70b/expanded",
+ "layers_run": [
+ 4,
+ 8
+ ],
+ "intermediate": [
+ 26624,
+ 26624
+ ],
+ "prompts": [
+ "The capital of France is",
+ "def fibonacci(n):\n ",
+ "Столица России —"
+ ],
+ "ids": [
+ [
+ 760,
+ 6511,
+ 314,
+ 9338,
+ 369
+ ],
+ [
+ 727,
+ 73111,
+ 1393,
+ 1590,
+ 198,
+ 257
+ ],
+ [
+ 212396,
+ 168314,
+ 170312,
+ 1892
+ ]
+ ],
+ "rows": [
+ {
+ "T": 5,
+ "identical": true,
+ "max_abs": 0.0,
+ "mean_abs": 0.0,
+ "max_kl": 0.0,
+ "top1_agree": 5
+ },
+ {
+ "T": 6,
+ "identical": true,
+ "max_abs": 0.0,
+ "mean_abs": 0.0,
+ "max_kl": 0.0,
+ "top1_agree": 6
+ },
+ {
+ "T": 4,
+ "identical": true,
+ "max_abs": 0.0,
+ "mean_abs": 0.0,
+ "max_kl": 0.0,
+ "top1_agree": 4
+ }
+ ],
+ "a_info": {
+ "cached": "/mnt/nfs_share/JiRackDeltaNet_36b/expanded/gate_logits_fp32blocks_L4.npz"
+ },
+ "b_info": {
+ "loaded": 124,
+ "skipped": 1590,
+ "unexpected": [],
+ "missing_buffers": [],
+ "seconds": 138
+ },
+ "torch": "2.13.0+cu130"
+}
\ No newline at end of file
diff --git a/sft_final_fp8/gguf.txt b/sft_final_fp8/gguf.txt
new file mode 100644
index 0000000000000000000000000000000000000000..79bf8f0314f527cf5c6c5f85426a44eb306fe28d
--- /dev/null
+++ b/sft_final_fp8/gguf.txt
@@ -0,0 +1,35 @@
+# 1. Конвертация в bf16 (как договорились)
+python convert_hf_to_gguf.py /mnt/nfs_share/Qwen3_8 \
+ --outfile /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.gguf --outtype bf16
+
+# 2. Сборка llama-quantize (один раз)
+cd /mnt/nfs_share/llama.cpp
+cmake -B build
+cmake --build build -j
+
+# 3. Квантование в Q4_K_M
+./build/bin/llama-quantize \
+ /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.gguf \
+ /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.Q4_K_M.gguf \
+ Q4_K_M
+
+
+
+cd /mnt/nfs_share/llama.cpp
+cmake -B build -DGGML_CUDA=ON # для NVIDIA
+make -C build
+
+
+/mnt/nfs_share/llama.cpp/build/bin/llama-cli \
+ -m /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.Q4_K_M.gguf \
+ -n 512 \
+ -c 4096 \
+ -p "Explain quantum computing in simple terms:"
+
+
+
+
+
+/mnt/nfs_share/llama.cpp/build/bin/llama-server \
+ -m /mnt/nfs_share/Qwen3_8/JiRackDeltaNet_27b.Q4_K_M.gguf \
+ --port 8000
diff --git a/sft_final_fp8/jirack_sft_state.json b/sft_final_fp8/jirack_sft_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..3ccb679b502005f458f3dea7722e9fc187a51eee
--- /dev/null
+++ b/sft_final_fp8/jirack_sft_state.json
@@ -0,0 +1,9 @@
+{
+ "global_step": 374,
+ "lr": 1e-05,
+ "max_steps": 374,
+ "shards": "0",
+ "tok_micro": 8192,
+ "accum": 2,
+ "init_dir": "/home/ubuntu/deltanet70b"
+}
\ No newline at end of file
diff --git a/sft_final_fp8/model-00001-of-00027.safetensors b/sft_final_fp8/model-00001-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..03b5f121062e9daa9e1b8790ca7d9bbad1f3e3d9
--- /dev/null
+++ b/sft_final_fp8/model-00001-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64722ca80a4e982fa7c0b4b98a39724aa1e9f3d7dbb2e2e19e45232e71298c33
+size 3846592104
diff --git a/sft_final_fp8/model-00002-of-00027.safetensors b/sft_final_fp8/model-00002-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ea9aa183d1bcf93f6e8cba4011ee8f4f2c74b38d
--- /dev/null
+++ b/sft_final_fp8/model-00002-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab7f457ba8d979d45d419316a07aa3b4d119afee6dd292b57a030636a26ccd2e
+size 2604414968
diff --git a/sft_final_fp8/model-00003-of-00027.safetensors b/sft_final_fp8/model-00003-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c2e02473dddcdd5f443cb9a6d068517145d93f06
--- /dev/null
+++ b/sft_final_fp8/model-00003-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64e948484e0093b958f0d52e3f6851284001a0938d41c16238789cc21c34fc89
+size 2615967968
diff --git a/sft_final_fp8/model-00004-of-00027.safetensors b/sft_final_fp8/model-00004-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fb488d6ae106236c5eef1de3542156a44f785287
--- /dev/null
+++ b/sft_final_fp8/model-00004-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fe90d3c849dcefd3539a94a2c5ad96b0c0f668886bbaf8908c43cb1298fdd92
+size 2615968008
diff --git a/sft_final_fp8/model-00005-of-00027.safetensors b/sft_final_fp8/model-00005-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..d48bdf49a608425a003676d273b5ef256a0b8576
--- /dev/null
+++ b/sft_final_fp8/model-00005-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d74a18d2436470e5290dc40a466c4a316341f0772da91ef5f2f8fa6b29a1cef2
+size 2615968016
diff --git a/sft_final_fp8/model-00006-of-00027.safetensors b/sft_final_fp8/model-00006-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..75f5727f83160d0d3728621871c2ae929aa19f51
--- /dev/null
+++ b/sft_final_fp8/model-00006-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57e4576b92c0492bbba6d1366dd879390bfddbfff06e88e38981af9b619d15d2
+size 2604415064
diff --git a/sft_final_fp8/model-00007-of-00027.safetensors b/sft_final_fp8/model-00007-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ff6e04af88b5a5ae5c25cfa777cb8a56b0e233bc
--- /dev/null
+++ b/sft_final_fp8/model-00007-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb4b853c0d419455f3c55a15dd13fa7a004f7ff9f65e74a663bdbdb275e22e0d
+size 2615968008
diff --git a/sft_final_fp8/model-00008-of-00027.safetensors b/sft_final_fp8/model-00008-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c1a705199018cc039b9fd2972ab566c2605699c1
--- /dev/null
+++ b/sft_final_fp8/model-00008-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc9e2732bd5a5772026a9e5f325de8c9447a4e041045bf90f76e06fc2bf1aea7
+size 2615968008
diff --git a/sft_final_fp8/model-00009-of-00027.safetensors b/sft_final_fp8/model-00009-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..213dc677edc86cd00cb887fb1435f2a65f0efd14
--- /dev/null
+++ b/sft_final_fp8/model-00009-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:968180225a6dfbb6defec5c0f0873d35ba4fb820e76aca23e656ddcbd0f81d2d
+size 2615968016
diff --git a/sft_final_fp8/model-00010-of-00027.safetensors b/sft_final_fp8/model-00010-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8c5d3ff94ab7bc5befd40b95201283a4a488fca0
--- /dev/null
+++ b/sft_final_fp8/model-00010-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a29e70457b7b6b83f17f524bdf727688adf1f9bd862b2e01843c3008a07a683
+size 2604415064
diff --git a/sft_final_fp8/model-00011-of-00027.safetensors b/sft_final_fp8/model-00011-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..b6cf350db162b343d6843a4d291134f0c838b301
--- /dev/null
+++ b/sft_final_fp8/model-00011-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a373e0df147b058f1d97771dc9481843209545b61f13f56e6dd6b8f78ac3972
+size 2615968008
diff --git a/sft_final_fp8/model-00012-of-00027.safetensors b/sft_final_fp8/model-00012-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c7842c52d21c97f8805b3f7cc36e564a0682301a
--- /dev/null
+++ b/sft_final_fp8/model-00012-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:269e5de6bfb6b9b533f3e000863c8f5461cdd83c0ed4993af7bfc8e260f62bfb
+size 2615968008
diff --git a/sft_final_fp8/model-00013-of-00027.safetensors b/sft_final_fp8/model-00013-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fe3364c96489186b143f83636e467e168d486fde
--- /dev/null
+++ b/sft_final_fp8/model-00013-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9898496c99f4b076f3324e35f8429afeb6777ed67ee64bc71713fb6f1176b33
+size 2615968016
diff --git a/sft_final_fp8/model-00014-of-00027.safetensors b/sft_final_fp8/model-00014-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..4e3f1f31051f5e1c3772b2c267226261a37885b4
--- /dev/null
+++ b/sft_final_fp8/model-00014-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24f55a800caf7972603adeaa31be3a04427eaf4ff1fd4230e0b1ff6b190f2704
+size 2604415064
diff --git a/sft_final_fp8/model-00015-of-00027.safetensors b/sft_final_fp8/model-00015-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..8c3423de94061bf7dbeab415d18c1fa4e621bb6f
--- /dev/null
+++ b/sft_final_fp8/model-00015-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcfd305aa5c14e66097ca804093342ec031118c45b05dac2ba2a6013bad4b326
+size 2615968008
diff --git a/sft_final_fp8/model-00016-of-00027.safetensors b/sft_final_fp8/model-00016-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9014782277dd3c8b3b6f5e3726ff8babb914744e
--- /dev/null
+++ b/sft_final_fp8/model-00016-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3625abb6433611716c36f6cfc3d0f02f113c7a23bcf6de70587feeec84994ab
+size 2615968008
diff --git a/sft_final_fp8/model-00017-of-00027.safetensors b/sft_final_fp8/model-00017-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..ff20ba28b7fd11b249c225bda9d01c817358e59a
--- /dev/null
+++ b/sft_final_fp8/model-00017-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ef7bc77209ba52291e8d14da96de17f9aabe3674a9fa6a8194de50e7e05b581
+size 2615968016
diff --git a/sft_final_fp8/model-00018-of-00027.safetensors b/sft_final_fp8/model-00018-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7e58004166fe2ba43469e2d97f232bc068834cb4
--- /dev/null
+++ b/sft_final_fp8/model-00018-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74c0b5a5b26e143790548c91ae565050307205d9aa403793aba7d49f8f432e75
+size 2604415064
diff --git a/sft_final_fp8/model-00019-of-00027.safetensors b/sft_final_fp8/model-00019-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..980f29e639c90e4afb9e36403540902aea09758f
--- /dev/null
+++ b/sft_final_fp8/model-00019-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8eec7b35b1772de104288b627db56b6eb0e33025fdefecd20c635820e78a5dd
+size 2615968008
diff --git a/sft_final_fp8/model-00020-of-00027.safetensors b/sft_final_fp8/model-00020-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..c5b996b656c23427a2a58404768a5ac75157ce1c
--- /dev/null
+++ b/sft_final_fp8/model-00020-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77cdd21052a22f70c02e02bb0a1869d686c95d488bd42fd4ac6175325f4bd59d
+size 2615968008
diff --git a/sft_final_fp8/model-00021-of-00027.safetensors b/sft_final_fp8/model-00021-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..9ce875caf3d7686fdc892ee7fa41ff9bdef8692a
--- /dev/null
+++ b/sft_final_fp8/model-00021-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71a57e113c9b52eccf06c8b573740133c3a1dc17e59bf0a897ab451993aa80ad
+size 2615968064
diff --git a/sft_final_fp8/model-00022-of-00027.safetensors b/sft_final_fp8/model-00022-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..316c7809f0930379c54530f2b391dc006e5b3421
--- /dev/null
+++ b/sft_final_fp8/model-00022-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed8178653d4aa2c4098285dcfee06a904fbc0a497f904aa064fffd702ca1bc06
+size 2604415160
diff --git a/sft_final_fp8/model-00023-of-00027.safetensors b/sft_final_fp8/model-00023-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..5a0315add9f30febd5412dcb125b33af6d9b3691
--- /dev/null
+++ b/sft_final_fp8/model-00023-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2532a6f9735df117361324a82eb9c21a749f87fc3dc15f798ac59ea9ef038616
+size 2615968112
diff --git a/sft_final_fp8/model-00024-of-00027.safetensors b/sft_final_fp8/model-00024-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..0b5e31aec67459ae2d1e2196f9621872e8f0939a
--- /dev/null
+++ b/sft_final_fp8/model-00024-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72ec56fc78c3cf4157fccb4a166f8b9b8fad9a5149b17161db394e3cc9dd6889
+size 2615968112
diff --git a/sft_final_fp8/model-00025-of-00027.safetensors b/sft_final_fp8/model-00025-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1996e367cd651dc1f1fc31ceb013b9e432396383
--- /dev/null
+++ b/sft_final_fp8/model-00025-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:046ad67da8a252b4894fd37d74e6f39832a085fcf8ff3a6e67a81298e83f044b
+size 2615968112
diff --git a/sft_final_fp8/model-00026-of-00027.safetensors b/sft_final_fp8/model-00026-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..627571cbfd9d34d7bb8bb04ca89f3ebb85aa2c5e
--- /dev/null
+++ b/sft_final_fp8/model-00026-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d61324fc5639cbbcc87249c535f176c45181a6171da964199d51f6978e731e86
+size 2604415160
diff --git a/sft_final_fp8/model-00027-of-00027.safetensors b/sft_final_fp8/model-00027-of-00027.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..7606678ea340a5203ae44138d254c30a223ad7ed
--- /dev/null
+++ b/sft_final_fp8/model-00027-of-00027.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90b64f5d9b1fa9c1fd73947b3b069a4480f41c15064c38739f70372dbfdf67f7
+size 3948040696
diff --git a/sft_final_fp8/model.safetensors.index.json b/sft_final_fp8/model.safetensors.index.json
new file mode 100644
index 0000000000000000000000000000000000000000..9fd158ca698af5a5c435e072ce3b0dfc34f4400a
--- /dev/null
+++ b/sft_final_fp8/model.safetensors.index.json
@@ -0,0 +1,2521 @@
+{
+ "metadata": {
+ "total_size": 73112673280
+ },
+ "weight_map": {
+ "model.embed_tokens.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.A_log": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.conv1d.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.dt_bias": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.in_proj_a.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.in_proj_b.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.in_proj_qkv.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.in_proj_qkv.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.in_proj_z.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.in_proj_z.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.norm.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.out_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.linear_attn.out_proj.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.0.mlp.down_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.mlp.down_proj.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.mlp.gate_proj.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.0.mlp.up_proj.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.input_layernorm.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.A_log": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.conv1d.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.dt_bias": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.in_proj_a.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.in_proj_b.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.in_proj_qkv.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.in_proj_qkv.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.in_proj_z.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.in_proj_z.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.norm.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.out_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.linear_attn.out_proj.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.1.mlp.down_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.mlp.down_proj.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.mlp.gate_proj.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.1.mlp.up_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.1.mlp.up_proj.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.A_log": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.conv1d.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.dt_bias": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.in_proj_a.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.in_proj_b.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.in_proj_qkv.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.in_proj_qkv.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.in_proj_z.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.in_proj_z.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.norm.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.out_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.linear_attn.out_proj.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00027.safetensors",
+ "model.layers.2.mlp.gate_proj.weight_scale_inv": "model-00001-of-00027.safetensors",
+ "model.layers.2.input_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.2.mlp.down_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.2.mlp.down_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.2.mlp.up_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.2.mlp.up_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.2.post_attention_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.input_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.mlp.down_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.mlp.down_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.mlp.gate_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.3.mlp.up_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.mlp.up_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.k_norm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.k_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.o_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.q_norm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.q_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.3.self_attn.v_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.4.input_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.A_log": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.conv1d.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.dt_bias": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.in_proj_a.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.in_proj_b.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.in_proj_qkv.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.in_proj_qkv.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.in_proj_z.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.in_proj_z.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.norm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.out_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.linear_attn.out_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.4.mlp.down_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.mlp.down_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.mlp.gate_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.4.mlp.up_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.4.mlp.up_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.input_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.A_log": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.conv1d.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.dt_bias": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.in_proj_a.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.in_proj_b.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.in_proj_qkv.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.in_proj_qkv.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.in_proj_z.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.in_proj_z.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.norm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.out_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.linear_attn.out_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.5.mlp.down_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.mlp.down_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.mlp.gate_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.5.mlp.up_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.5.mlp.up_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.input_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.A_log": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.conv1d.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.dt_bias": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.in_proj_a.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.in_proj_b.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.in_proj_qkv.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.in_proj_qkv.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.in_proj_z.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.in_proj_z.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.norm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.out_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.linear_attn.out_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.6.mlp.down_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.mlp.down_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.mlp.gate_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.6.mlp.up_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.6.mlp.up_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.mlp.gate_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.k_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.o_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.q_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00027.safetensors",
+ "model.layers.7.self_attn.v_proj.weight_scale_inv": "model-00002-of-00027.safetensors",
+ "model.layers.10.input_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.A_log": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.conv1d.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.dt_bias": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.in_proj_a.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.in_proj_b.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.in_proj_qkv.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.in_proj_qkv.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.in_proj_z.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.in_proj_z.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.norm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.out_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.linear_attn.out_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.10.mlp.down_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.mlp.down_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.mlp.gate_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.10.mlp.up_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.10.mlp.up_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.input_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.mlp.down_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.mlp.down_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.mlp.gate_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.11.mlp.up_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.mlp.up_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.k_norm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.k_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.o_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.q_norm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.q_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.11.self_attn.v_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.A_log": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.conv1d.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.dt_bias": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.in_proj_a.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.in_proj_b.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.in_proj_qkv.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.in_proj_qkv.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.in_proj_z.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.in_proj_z.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.norm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.out_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.linear_attn.out_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.mlp.gate_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.7.input_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.7.mlp.down_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.7.mlp.down_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.7.mlp.up_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.7.mlp.up_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.7.post_attention_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.input_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.A_log": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.conv1d.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.dt_bias": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.in_proj_a.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.in_proj_b.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.in_proj_qkv.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.in_proj_qkv.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.in_proj_z.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.in_proj_z.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.norm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.out_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.linear_attn.out_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.8.mlp.down_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.mlp.down_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.8.mlp.gate_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.mlp.gate_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.8.mlp.up_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.8.mlp.up_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.input_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.A_log": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.conv1d.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.dt_bias": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.in_proj_a.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.in_proj_b.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.in_proj_qkv.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.in_proj_qkv.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.in_proj_z.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.in_proj_z.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.norm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.out_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.linear_attn.out_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.9.mlp.down_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.mlp.down_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.mlp.gate_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.9.mlp.up_proj.weight": "model-00003-of-00027.safetensors",
+ "model.layers.9.mlp.up_proj.weight_scale_inv": "model-00003-of-00027.safetensors",
+ "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00027.safetensors",
+ "model.layers.12.input_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.12.mlp.down_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.12.mlp.down_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.12.mlp.up_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.12.mlp.up_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.12.post_attention_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.input_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.A_log": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.conv1d.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.dt_bias": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.in_proj_a.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.in_proj_b.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.in_proj_qkv.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.in_proj_qkv.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.in_proj_z.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.in_proj_z.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.norm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.out_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.linear_attn.out_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.13.mlp.down_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.mlp.down_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.13.mlp.gate_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.mlp.gate_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.13.mlp.up_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.13.mlp.up_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.13.post_attention_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.input_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.A_log": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.conv1d.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.dt_bias": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.in_proj_a.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.in_proj_b.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.in_proj_qkv.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.in_proj_qkv.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.in_proj_z.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.in_proj_z.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.norm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.out_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.linear_attn.out_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.14.mlp.down_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.mlp.down_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.14.mlp.gate_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.mlp.gate_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.14.mlp.up_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.14.mlp.up_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.input_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.mlp.down_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.mlp.down_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.mlp.gate_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.15.mlp.up_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.mlp.up_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.k_norm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.k_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.o_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.q_norm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.q_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.15.self_attn.v_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.16.input_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.A_log": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.conv1d.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.dt_bias": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.in_proj_a.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.in_proj_b.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.in_proj_qkv.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.in_proj_qkv.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.in_proj_z.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.in_proj_z.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.norm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.out_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.linear_attn.out_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.16.mlp.down_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.mlp.down_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.mlp.gate_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.16.mlp.up_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.16.mlp.up_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.A_log": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.conv1d.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.dt_bias": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.in_proj_a.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.in_proj_b.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.in_proj_qkv.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.in_proj_qkv.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.in_proj_z.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.in_proj_z.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.norm.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.out_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.linear_attn.out_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00027.safetensors",
+ "model.layers.17.mlp.gate_proj.weight_scale_inv": "model-00004-of-00027.safetensors",
+ "model.layers.17.input_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.17.mlp.down_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.17.mlp.down_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.17.mlp.up_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.17.mlp.up_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.17.post_attention_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.input_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.A_log": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.conv1d.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.dt_bias": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.in_proj_a.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.in_proj_b.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.in_proj_qkv.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.in_proj_qkv.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.in_proj_z.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.in_proj_z.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.norm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.out_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.linear_attn.out_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.18.mlp.down_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.mlp.down_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.18.mlp.gate_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.mlp.gate_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.18.mlp.up_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.18.mlp.up_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.18.post_attention_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.input_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.mlp.down_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.mlp.down_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.19.mlp.gate_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.mlp.gate_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.19.mlp.up_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.mlp.up_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.19.post_attention_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.k_norm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.k_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.k_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.o_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.o_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.q_norm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.q_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.q_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.v_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.19.self_attn.v_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.20.input_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.A_log": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.conv1d.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.dt_bias": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.in_proj_a.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.in_proj_b.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.in_proj_qkv.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.in_proj_qkv.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.in_proj_z.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.in_proj_z.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.norm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.out_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.linear_attn.out_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.20.mlp.down_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.mlp.down_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.mlp.gate_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.20.mlp.up_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.20.mlp.up_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.input_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.A_log": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.conv1d.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.dt_bias": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.in_proj_a.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.in_proj_b.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.in_proj_qkv.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.in_proj_qkv.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.in_proj_z.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.in_proj_z.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.norm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.out_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.linear_attn.out_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.21.mlp.down_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.mlp.down_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.mlp.gate_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.21.mlp.up_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.21.mlp.up_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.A_log": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.conv1d.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.dt_bias": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.in_proj_a.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.in_proj_b.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.in_proj_qkv.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.in_proj_qkv.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.in_proj_z.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.in_proj_z.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.norm.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.out_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.linear_attn.out_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00027.safetensors",
+ "model.layers.22.mlp.gate_proj.weight_scale_inv": "model-00005-of-00027.safetensors",
+ "model.layers.22.input_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.22.mlp.down_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.22.mlp.down_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.22.mlp.up_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.22.mlp.up_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.22.post_attention_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.input_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.mlp.down_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.mlp.down_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.23.mlp.gate_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.mlp.gate_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.23.mlp.up_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.mlp.up_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.23.post_attention_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.k_norm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.k_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.k_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.o_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.o_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.q_norm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.q_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.q_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.v_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.23.self_attn.v_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.24.input_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.A_log": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.conv1d.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.dt_bias": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.in_proj_a.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.in_proj_b.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.in_proj_qkv.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.in_proj_qkv.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.in_proj_z.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.in_proj_z.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.norm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.out_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.linear_attn.out_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.24.mlp.down_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.mlp.down_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.24.mlp.gate_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.mlp.gate_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.24.mlp.up_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.24.mlp.up_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.24.post_attention_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.input_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.A_log": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.conv1d.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.dt_bias": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.in_proj_a.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.in_proj_b.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.in_proj_qkv.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.in_proj_qkv.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.in_proj_z.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.in_proj_z.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.norm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.out_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.linear_attn.out_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.25.mlp.down_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.mlp.down_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.25.mlp.gate_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.mlp.gate_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.25.mlp.up_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.25.mlp.up_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.input_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.A_log": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.conv1d.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.dt_bias": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.in_proj_a.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.in_proj_b.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.in_proj_qkv.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.in_proj_qkv.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.in_proj_z.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.in_proj_z.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.norm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.out_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.linear_attn.out_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.26.mlp.down_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.mlp.down_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.mlp.gate_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.26.mlp.up_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.26.mlp.up_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.mlp.gate_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.k_norm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.k_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.o_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.q_norm.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.q_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00027.safetensors",
+ "model.layers.27.self_attn.v_proj.weight_scale_inv": "model-00006-of-00027.safetensors",
+ "model.layers.27.input_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.27.mlp.down_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.27.mlp.down_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.27.mlp.up_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.27.mlp.up_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.27.post_attention_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.input_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.A_log": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.conv1d.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.dt_bias": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.in_proj_a.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.in_proj_b.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.in_proj_qkv.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.in_proj_qkv.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.in_proj_z.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.in_proj_z.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.norm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.out_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.linear_attn.out_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.28.mlp.down_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.mlp.down_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.28.mlp.gate_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.mlp.gate_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.28.mlp.up_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.28.mlp.up_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.28.post_attention_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.input_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.A_log": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.conv1d.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.dt_bias": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.in_proj_a.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.in_proj_b.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.in_proj_qkv.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.in_proj_qkv.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.in_proj_z.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.in_proj_z.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.norm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.out_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.linear_attn.out_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.29.mlp.down_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.mlp.down_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.29.mlp.gate_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.mlp.gate_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.29.mlp.up_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.29.mlp.up_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.29.post_attention_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.input_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.A_log": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.conv1d.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.dt_bias": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.in_proj_a.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.in_proj_b.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.in_proj_qkv.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.in_proj_qkv.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.in_proj_z.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.in_proj_z.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.norm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.out_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.linear_attn.out_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.30.mlp.down_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.mlp.down_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.30.mlp.gate_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.mlp.gate_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.30.mlp.up_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.30.mlp.up_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.30.post_attention_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.input_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.mlp.down_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.mlp.down_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.31.mlp.gate_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.mlp.gate_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.31.mlp.up_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.mlp.up_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.k_norm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.k_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.k_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.o_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.o_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.q_norm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.q_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.q_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.v_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.31.self_attn.v_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.A_log": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.conv1d.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.dt_bias": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.in_proj_a.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.in_proj_b.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.in_proj_qkv.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.in_proj_qkv.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.in_proj_z.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.in_proj_z.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.norm.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.out_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.linear_attn.out_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.32.mlp.gate_proj.weight": "model-00007-of-00027.safetensors",
+ "model.layers.32.mlp.gate_proj.weight_scale_inv": "model-00007-of-00027.safetensors",
+ "model.layers.32.input_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.32.mlp.down_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.32.mlp.down_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.32.mlp.up_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.32.mlp.up_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.32.post_attention_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.input_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.A_log": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.conv1d.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.dt_bias": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.in_proj_a.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.in_proj_b.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.in_proj_qkv.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.in_proj_qkv.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.in_proj_z.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.in_proj_z.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.norm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.out_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.linear_attn.out_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.33.mlp.down_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.mlp.down_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.33.mlp.gate_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.mlp.gate_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.33.mlp.up_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.33.mlp.up_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.33.post_attention_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.input_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.A_log": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.conv1d.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.dt_bias": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.in_proj_a.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.in_proj_b.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.in_proj_qkv.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.in_proj_qkv.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.in_proj_z.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.in_proj_z.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.norm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.out_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.linear_attn.out_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.34.mlp.down_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.mlp.down_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.34.mlp.gate_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.mlp.gate_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.34.mlp.up_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.34.mlp.up_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.34.post_attention_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.input_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.mlp.down_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.mlp.down_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.35.mlp.gate_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.mlp.gate_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.35.mlp.up_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.mlp.up_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.35.post_attention_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.k_norm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.k_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.k_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.o_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.o_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.q_norm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.q_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.q_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.v_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.35.self_attn.v_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.36.input_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.A_log": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.conv1d.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.dt_bias": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.in_proj_a.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.in_proj_b.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.in_proj_qkv.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.in_proj_qkv.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.in_proj_z.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.in_proj_z.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.norm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.out_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.linear_attn.out_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.36.mlp.down_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.mlp.down_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.36.mlp.gate_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.mlp.gate_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.36.mlp.up_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.36.mlp.up_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.36.post_attention_layernorm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.A_log": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.conv1d.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.dt_bias": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.in_proj_a.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.in_proj_b.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.in_proj_qkv.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.in_proj_qkv.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.in_proj_z.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.in_proj_z.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.norm.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.out_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.linear_attn.out_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.37.mlp.gate_proj.weight": "model-00008-of-00027.safetensors",
+ "model.layers.37.mlp.gate_proj.weight_scale_inv": "model-00008-of-00027.safetensors",
+ "model.layers.37.input_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.37.mlp.down_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.37.mlp.down_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.37.mlp.up_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.37.mlp.up_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.37.post_attention_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.input_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.A_log": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.conv1d.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.dt_bias": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.in_proj_a.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.in_proj_b.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.in_proj_qkv.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.in_proj_qkv.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.in_proj_z.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.in_proj_z.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.norm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.out_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.linear_attn.out_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.38.mlp.down_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.mlp.down_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.38.mlp.gate_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.mlp.gate_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.38.mlp.up_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.38.mlp.up_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.38.post_attention_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.input_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.mlp.down_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.mlp.down_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.39.mlp.gate_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.mlp.gate_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.39.mlp.up_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.mlp.up_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.39.post_attention_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.k_norm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.k_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.k_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.o_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.o_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.q_norm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.q_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.q_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.v_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.39.self_attn.v_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.40.input_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.A_log": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.conv1d.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.dt_bias": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.in_proj_a.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.in_proj_b.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.in_proj_qkv.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.in_proj_qkv.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.in_proj_z.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.in_proj_z.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.norm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.out_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.linear_attn.out_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.40.mlp.down_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.mlp.down_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.40.mlp.gate_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.mlp.gate_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.40.mlp.up_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.40.mlp.up_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.40.post_attention_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.input_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.A_log": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.conv1d.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.dt_bias": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.in_proj_a.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.in_proj_b.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.in_proj_qkv.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.in_proj_qkv.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.in_proj_z.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.in_proj_z.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.norm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.out_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.linear_attn.out_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.41.mlp.down_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.mlp.down_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.41.mlp.gate_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.mlp.gate_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.41.mlp.up_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.41.mlp.up_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.41.post_attention_layernorm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.A_log": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.conv1d.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.dt_bias": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.in_proj_a.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.in_proj_b.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.in_proj_qkv.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.in_proj_qkv.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.in_proj_z.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.in_proj_z.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.norm.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.out_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.linear_attn.out_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.42.mlp.gate_proj.weight": "model-00009-of-00027.safetensors",
+ "model.layers.42.mlp.gate_proj.weight_scale_inv": "model-00009-of-00027.safetensors",
+ "model.layers.42.input_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.42.mlp.down_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.42.mlp.down_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.42.mlp.up_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.42.mlp.up_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.42.post_attention_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.input_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.mlp.down_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.mlp.down_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.43.mlp.gate_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.mlp.gate_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.43.mlp.up_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.mlp.up_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.43.post_attention_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.k_norm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.k_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.k_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.o_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.o_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.q_norm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.q_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.q_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.v_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.43.self_attn.v_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.44.input_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.A_log": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.conv1d.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.dt_bias": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.in_proj_a.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.in_proj_b.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.in_proj_qkv.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.in_proj_qkv.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.in_proj_z.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.in_proj_z.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.norm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.out_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.linear_attn.out_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.44.mlp.down_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.mlp.down_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.44.mlp.gate_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.mlp.gate_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.44.mlp.up_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.44.mlp.up_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.44.post_attention_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.input_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.A_log": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.conv1d.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.dt_bias": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.in_proj_a.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.in_proj_b.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.in_proj_qkv.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.in_proj_qkv.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.in_proj_z.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.in_proj_z.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.norm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.out_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.linear_attn.out_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.45.mlp.down_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.mlp.down_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.45.mlp.gate_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.mlp.gate_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.45.mlp.up_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.45.mlp.up_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.45.post_attention_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.input_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.A_log": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.conv1d.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.dt_bias": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.in_proj_a.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.in_proj_b.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.in_proj_qkv.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.in_proj_qkv.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.in_proj_z.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.in_proj_z.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.norm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.out_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.linear_attn.out_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.46.mlp.down_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.mlp.down_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.46.mlp.gate_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.mlp.gate_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.46.mlp.up_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.46.mlp.up_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.46.post_attention_layernorm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.mlp.gate_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.mlp.gate_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.k_norm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.k_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.k_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.o_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.o_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.q_norm.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.q_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.q_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.v_proj.weight": "model-00010-of-00027.safetensors",
+ "model.layers.47.self_attn.v_proj.weight_scale_inv": "model-00010-of-00027.safetensors",
+ "model.layers.47.input_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.47.mlp.down_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.47.mlp.down_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.47.mlp.up_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.47.mlp.up_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.47.post_attention_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.input_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.A_log": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.conv1d.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.dt_bias": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.in_proj_a.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.in_proj_b.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.in_proj_qkv.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.in_proj_qkv.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.in_proj_z.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.in_proj_z.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.norm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.out_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.linear_attn.out_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.48.mlp.down_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.mlp.down_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.48.mlp.gate_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.mlp.gate_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.48.mlp.up_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.48.mlp.up_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.48.post_attention_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.input_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.A_log": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.conv1d.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.dt_bias": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.in_proj_a.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.in_proj_b.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.in_proj_qkv.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.in_proj_qkv.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.in_proj_z.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.in_proj_z.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.norm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.out_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.linear_attn.out_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.49.mlp.down_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.mlp.down_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.49.mlp.gate_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.mlp.gate_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.49.mlp.up_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.49.mlp.up_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.49.post_attention_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.input_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.A_log": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.conv1d.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.dt_bias": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.in_proj_a.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.in_proj_b.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.in_proj_qkv.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.in_proj_qkv.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.in_proj_z.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.in_proj_z.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.norm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.out_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.linear_attn.out_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.50.mlp.down_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.mlp.down_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.50.mlp.gate_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.mlp.gate_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.50.mlp.up_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.50.mlp.up_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.50.post_attention_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.input_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.mlp.down_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.mlp.down_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.51.mlp.gate_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.mlp.gate_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.51.mlp.up_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.mlp.up_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.51.post_attention_layernorm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.k_norm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.k_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.k_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.o_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.o_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.q_norm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.q_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.q_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.v_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.51.self_attn.v_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.A_log": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.conv1d.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.dt_bias": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.in_proj_a.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.in_proj_b.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.in_proj_qkv.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.in_proj_qkv.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.in_proj_z.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.in_proj_z.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.norm.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.out_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.linear_attn.out_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.52.mlp.gate_proj.weight": "model-00011-of-00027.safetensors",
+ "model.layers.52.mlp.gate_proj.weight_scale_inv": "model-00011-of-00027.safetensors",
+ "model.layers.52.input_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.52.mlp.down_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.52.mlp.down_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.52.mlp.up_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.52.mlp.up_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.52.post_attention_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.input_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.A_log": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.conv1d.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.dt_bias": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.in_proj_a.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.in_proj_b.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.in_proj_qkv.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.in_proj_qkv.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.in_proj_z.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.in_proj_z.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.norm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.out_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.linear_attn.out_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.53.mlp.down_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.mlp.down_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.53.mlp.gate_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.mlp.gate_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.53.mlp.up_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.53.mlp.up_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.53.post_attention_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.input_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.A_log": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.conv1d.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.dt_bias": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.in_proj_a.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.in_proj_b.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.in_proj_qkv.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.in_proj_qkv.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.in_proj_z.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.in_proj_z.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.norm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.out_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.linear_attn.out_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.54.mlp.down_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.mlp.down_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.54.mlp.gate_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.mlp.gate_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.54.mlp.up_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.54.mlp.up_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.54.post_attention_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.input_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.mlp.down_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.mlp.down_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.55.mlp.gate_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.mlp.gate_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.55.mlp.up_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.mlp.up_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.55.post_attention_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.k_norm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.k_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.k_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.o_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.o_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.q_norm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.q_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.q_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.v_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.55.self_attn.v_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.56.input_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.A_log": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.conv1d.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.dt_bias": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.in_proj_a.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.in_proj_b.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.in_proj_qkv.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.in_proj_qkv.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.in_proj_z.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.in_proj_z.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.norm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.out_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.linear_attn.out_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.56.mlp.down_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.mlp.down_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.56.mlp.gate_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.mlp.gate_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.56.mlp.up_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.56.mlp.up_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.56.post_attention_layernorm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.A_log": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.conv1d.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.dt_bias": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.in_proj_a.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.in_proj_b.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.in_proj_qkv.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.in_proj_qkv.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.in_proj_z.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.in_proj_z.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.norm.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.out_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.linear_attn.out_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.57.mlp.gate_proj.weight": "model-00012-of-00027.safetensors",
+ "model.layers.57.mlp.gate_proj.weight_scale_inv": "model-00012-of-00027.safetensors",
+ "model.layers.57.input_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.57.mlp.down_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.57.mlp.down_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.57.mlp.up_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.57.mlp.up_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.57.post_attention_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.input_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.A_log": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.conv1d.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.dt_bias": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.in_proj_a.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.in_proj_b.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.in_proj_qkv.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.in_proj_qkv.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.in_proj_z.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.in_proj_z.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.norm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.out_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.linear_attn.out_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.58.mlp.down_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.mlp.down_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.58.mlp.gate_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.mlp.gate_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.58.mlp.up_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.58.mlp.up_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.58.post_attention_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.input_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.mlp.down_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.mlp.down_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.59.mlp.gate_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.mlp.gate_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.59.mlp.up_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.mlp.up_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.59.post_attention_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.k_norm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.k_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.k_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.o_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.o_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.q_norm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.q_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.q_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.v_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.59.self_attn.v_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.60.input_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.A_log": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.conv1d.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.dt_bias": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.in_proj_a.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.in_proj_b.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.in_proj_qkv.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.in_proj_qkv.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.in_proj_z.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.in_proj_z.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.norm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.out_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.linear_attn.out_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.60.mlp.down_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.mlp.down_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.60.mlp.gate_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.mlp.gate_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.60.mlp.up_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.60.mlp.up_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.60.post_attention_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.input_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.A_log": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.conv1d.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.dt_bias": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.in_proj_a.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.in_proj_b.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.in_proj_qkv.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.in_proj_qkv.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.in_proj_z.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.in_proj_z.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.norm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.out_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.linear_attn.out_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.61.mlp.down_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.mlp.down_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.61.mlp.gate_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.mlp.gate_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.61.mlp.up_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.61.mlp.up_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.61.post_attention_layernorm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.A_log": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.conv1d.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.dt_bias": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.in_proj_a.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.in_proj_b.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.in_proj_qkv.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.in_proj_qkv.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.in_proj_z.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.in_proj_z.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.norm.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.out_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.linear_attn.out_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.62.mlp.gate_proj.weight": "model-00013-of-00027.safetensors",
+ "model.layers.62.mlp.gate_proj.weight_scale_inv": "model-00013-of-00027.safetensors",
+ "model.layers.62.input_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.62.mlp.down_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.62.mlp.down_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.62.mlp.up_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.62.mlp.up_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.62.post_attention_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.input_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.mlp.down_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.mlp.down_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.63.mlp.gate_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.mlp.gate_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.63.mlp.up_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.mlp.up_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.63.post_attention_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.k_norm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.k_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.k_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.o_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.o_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.q_norm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.q_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.q_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.v_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.63.self_attn.v_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.64.input_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.A_log": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.conv1d.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.dt_bias": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.in_proj_a.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.in_proj_b.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.in_proj_qkv.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.in_proj_qkv.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.in_proj_z.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.in_proj_z.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.norm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.out_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.linear_attn.out_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.64.mlp.down_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.mlp.down_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.64.mlp.gate_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.mlp.gate_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.64.mlp.up_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.64.mlp.up_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.64.post_attention_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.input_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.A_log": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.conv1d.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.dt_bias": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.in_proj_a.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.in_proj_b.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.in_proj_qkv.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.in_proj_qkv.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.in_proj_z.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.in_proj_z.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.norm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.out_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.linear_attn.out_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.65.mlp.down_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.mlp.down_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.65.mlp.gate_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.mlp.gate_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.65.mlp.up_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.65.mlp.up_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.65.post_attention_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.input_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.A_log": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.conv1d.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.dt_bias": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.in_proj_a.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.in_proj_b.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.in_proj_qkv.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.in_proj_qkv.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.in_proj_z.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.in_proj_z.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.norm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.out_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.linear_attn.out_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.66.mlp.down_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.mlp.down_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.66.mlp.gate_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.mlp.gate_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.66.mlp.up_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.66.mlp.up_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.66.post_attention_layernorm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.mlp.gate_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.mlp.gate_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.k_norm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.k_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.k_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.o_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.o_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.q_norm.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.q_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.q_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.v_proj.weight": "model-00014-of-00027.safetensors",
+ "model.layers.67.self_attn.v_proj.weight_scale_inv": "model-00014-of-00027.safetensors",
+ "model.layers.67.input_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.67.mlp.down_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.67.mlp.down_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.67.mlp.up_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.67.mlp.up_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.67.post_attention_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.input_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.A_log": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.conv1d.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.dt_bias": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.in_proj_a.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.in_proj_b.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.in_proj_qkv.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.in_proj_qkv.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.in_proj_z.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.in_proj_z.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.norm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.out_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.linear_attn.out_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.68.mlp.down_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.mlp.down_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.68.mlp.gate_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.mlp.gate_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.68.mlp.up_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.68.mlp.up_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.68.post_attention_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.input_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.A_log": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.conv1d.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.dt_bias": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.in_proj_a.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.in_proj_b.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.in_proj_qkv.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.in_proj_qkv.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.in_proj_z.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.in_proj_z.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.norm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.out_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.linear_attn.out_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.69.mlp.down_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.mlp.down_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.69.mlp.gate_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.mlp.gate_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.69.mlp.up_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.69.mlp.up_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.69.post_attention_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.input_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.A_log": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.conv1d.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.dt_bias": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.in_proj_a.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.in_proj_b.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.in_proj_qkv.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.in_proj_qkv.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.in_proj_z.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.in_proj_z.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.norm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.out_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.linear_attn.out_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.70.mlp.down_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.mlp.down_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.70.mlp.gate_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.mlp.gate_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.70.mlp.up_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.70.mlp.up_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.70.post_attention_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.input_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.mlp.down_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.mlp.down_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.71.mlp.gate_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.mlp.gate_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.71.mlp.up_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.mlp.up_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.71.post_attention_layernorm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.k_norm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.k_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.k_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.o_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.o_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.q_norm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.q_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.q_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.v_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.71.self_attn.v_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.A_log": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.conv1d.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.dt_bias": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.in_proj_a.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.in_proj_b.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.in_proj_qkv.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.in_proj_qkv.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.in_proj_z.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.in_proj_z.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.norm.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.out_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.linear_attn.out_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.72.mlp.gate_proj.weight": "model-00015-of-00027.safetensors",
+ "model.layers.72.mlp.gate_proj.weight_scale_inv": "model-00015-of-00027.safetensors",
+ "model.layers.72.input_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.72.mlp.down_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.72.mlp.down_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.72.mlp.up_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.72.mlp.up_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.72.post_attention_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.input_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.A_log": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.conv1d.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.dt_bias": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.in_proj_a.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.in_proj_b.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.in_proj_qkv.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.in_proj_qkv.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.in_proj_z.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.in_proj_z.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.norm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.out_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.linear_attn.out_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.73.mlp.down_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.mlp.down_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.73.mlp.gate_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.mlp.gate_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.73.mlp.up_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.73.mlp.up_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.73.post_attention_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.input_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.A_log": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.conv1d.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.dt_bias": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.in_proj_a.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.in_proj_b.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.in_proj_qkv.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.in_proj_qkv.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.in_proj_z.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.in_proj_z.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.norm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.out_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.linear_attn.out_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.74.mlp.down_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.mlp.down_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.74.mlp.gate_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.mlp.gate_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.74.mlp.up_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.74.mlp.up_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.74.post_attention_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.input_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.mlp.down_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.mlp.down_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.75.mlp.gate_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.mlp.gate_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.75.mlp.up_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.mlp.up_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.75.post_attention_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.k_norm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.k_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.k_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.o_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.o_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.q_norm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.q_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.q_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.v_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.75.self_attn.v_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.76.input_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.A_log": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.conv1d.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.dt_bias": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.in_proj_a.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.in_proj_b.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.in_proj_qkv.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.in_proj_qkv.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.in_proj_z.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.in_proj_z.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.norm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.out_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.linear_attn.out_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.76.mlp.down_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.mlp.down_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.76.mlp.gate_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.mlp.gate_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.76.mlp.up_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.76.mlp.up_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.76.post_attention_layernorm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.A_log": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.conv1d.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.dt_bias": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.in_proj_a.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.in_proj_b.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.in_proj_qkv.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.in_proj_qkv.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.in_proj_z.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.in_proj_z.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.norm.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.out_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.linear_attn.out_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.77.mlp.gate_proj.weight": "model-00016-of-00027.safetensors",
+ "model.layers.77.mlp.gate_proj.weight_scale_inv": "model-00016-of-00027.safetensors",
+ "model.layers.77.input_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.77.mlp.down_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.77.mlp.down_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.77.mlp.up_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.77.mlp.up_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.77.post_attention_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.input_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.A_log": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.conv1d.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.dt_bias": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.in_proj_a.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.in_proj_b.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.in_proj_qkv.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.in_proj_qkv.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.in_proj_z.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.in_proj_z.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.norm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.out_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.linear_attn.out_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.78.mlp.down_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.mlp.down_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.78.mlp.gate_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.mlp.gate_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.78.mlp.up_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.78.mlp.up_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.78.post_attention_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.input_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.mlp.down_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.mlp.down_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.79.mlp.gate_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.mlp.gate_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.79.mlp.up_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.mlp.up_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.79.post_attention_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.k_norm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.k_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.k_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.o_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.o_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.q_norm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.q_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.q_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.v_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.79.self_attn.v_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.80.input_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.A_log": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.conv1d.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.dt_bias": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.in_proj_a.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.in_proj_b.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.in_proj_qkv.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.in_proj_qkv.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.in_proj_z.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.in_proj_z.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.norm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.out_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.linear_attn.out_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.80.mlp.down_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.mlp.down_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.80.mlp.gate_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.mlp.gate_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.80.mlp.up_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.80.mlp.up_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.80.post_attention_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.input_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.A_log": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.conv1d.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.dt_bias": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.in_proj_a.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.in_proj_b.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.in_proj_qkv.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.in_proj_qkv.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.in_proj_z.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.in_proj_z.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.norm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.out_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.linear_attn.out_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.81.mlp.down_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.mlp.down_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.81.mlp.gate_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.mlp.gate_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.81.mlp.up_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.81.mlp.up_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.81.post_attention_layernorm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.A_log": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.conv1d.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.dt_bias": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.in_proj_a.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.in_proj_b.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.in_proj_qkv.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.in_proj_qkv.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.in_proj_z.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.in_proj_z.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.norm.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.out_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.linear_attn.out_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.82.mlp.gate_proj.weight": "model-00017-of-00027.safetensors",
+ "model.layers.82.mlp.gate_proj.weight_scale_inv": "model-00017-of-00027.safetensors",
+ "model.layers.82.input_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.82.mlp.down_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.82.mlp.down_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.82.mlp.up_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.82.mlp.up_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.82.post_attention_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.input_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.mlp.down_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.mlp.down_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.83.mlp.gate_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.mlp.gate_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.83.mlp.up_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.mlp.up_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.83.post_attention_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.k_norm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.k_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.k_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.o_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.o_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.q_norm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.q_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.q_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.v_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.83.self_attn.v_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.84.input_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.A_log": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.conv1d.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.dt_bias": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.in_proj_a.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.in_proj_b.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.in_proj_qkv.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.in_proj_qkv.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.in_proj_z.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.in_proj_z.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.norm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.out_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.linear_attn.out_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.84.mlp.down_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.mlp.down_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.84.mlp.gate_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.mlp.gate_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.84.mlp.up_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.84.mlp.up_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.84.post_attention_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.input_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.A_log": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.conv1d.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.dt_bias": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.in_proj_a.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.in_proj_b.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.in_proj_qkv.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.in_proj_qkv.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.in_proj_z.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.in_proj_z.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.norm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.out_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.linear_attn.out_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.85.mlp.down_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.mlp.down_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.85.mlp.gate_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.mlp.gate_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.85.mlp.up_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.85.mlp.up_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.85.post_attention_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.input_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.A_log": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.conv1d.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.dt_bias": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.in_proj_a.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.in_proj_b.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.in_proj_qkv.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.in_proj_qkv.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.in_proj_z.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.in_proj_z.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.norm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.out_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.linear_attn.out_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.86.mlp.down_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.mlp.down_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.86.mlp.gate_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.mlp.gate_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.86.mlp.up_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.86.mlp.up_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.86.post_attention_layernorm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.mlp.gate_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.mlp.gate_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.k_norm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.k_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.k_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.o_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.o_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.q_norm.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.q_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.q_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.v_proj.weight": "model-00018-of-00027.safetensors",
+ "model.layers.87.self_attn.v_proj.weight_scale_inv": "model-00018-of-00027.safetensors",
+ "model.layers.87.input_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.87.mlp.down_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.87.mlp.down_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.87.mlp.up_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.87.mlp.up_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.87.post_attention_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.input_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.A_log": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.conv1d.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.dt_bias": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.in_proj_a.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.in_proj_b.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.in_proj_qkv.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.in_proj_qkv.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.in_proj_z.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.in_proj_z.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.norm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.out_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.linear_attn.out_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.88.mlp.down_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.mlp.down_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.88.mlp.gate_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.mlp.gate_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.88.mlp.up_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.88.mlp.up_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.88.post_attention_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.input_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.A_log": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.conv1d.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.dt_bias": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.in_proj_a.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.in_proj_b.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.in_proj_qkv.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.in_proj_qkv.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.in_proj_z.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.in_proj_z.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.norm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.out_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.linear_attn.out_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.89.mlp.down_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.mlp.down_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.89.mlp.gate_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.mlp.gate_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.89.mlp.up_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.89.mlp.up_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.89.post_attention_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.input_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.A_log": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.conv1d.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.dt_bias": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.in_proj_a.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.in_proj_b.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.in_proj_qkv.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.in_proj_qkv.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.in_proj_z.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.in_proj_z.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.norm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.out_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.linear_attn.out_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.90.mlp.down_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.mlp.down_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.90.mlp.gate_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.mlp.gate_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.90.mlp.up_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.90.mlp.up_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.90.post_attention_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.input_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.mlp.down_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.mlp.down_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.91.mlp.gate_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.mlp.gate_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.91.mlp.up_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.mlp.up_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.91.post_attention_layernorm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.k_norm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.k_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.k_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.o_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.o_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.q_norm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.q_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.q_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.v_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.91.self_attn.v_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.A_log": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.conv1d.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.dt_bias": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.in_proj_a.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.in_proj_b.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.in_proj_qkv.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.in_proj_qkv.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.in_proj_z.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.in_proj_z.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.norm.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.out_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.linear_attn.out_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.92.mlp.gate_proj.weight": "model-00019-of-00027.safetensors",
+ "model.layers.92.mlp.gate_proj.weight_scale_inv": "model-00019-of-00027.safetensors",
+ "model.layers.92.input_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.92.mlp.down_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.92.mlp.down_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.92.mlp.up_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.92.mlp.up_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.92.post_attention_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.input_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.A_log": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.conv1d.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.dt_bias": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.in_proj_a.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.in_proj_b.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.in_proj_qkv.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.in_proj_qkv.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.in_proj_z.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.in_proj_z.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.norm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.out_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.linear_attn.out_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.93.mlp.down_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.mlp.down_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.93.mlp.gate_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.mlp.gate_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.93.mlp.up_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.93.mlp.up_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.93.post_attention_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.input_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.A_log": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.conv1d.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.dt_bias": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.in_proj_a.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.in_proj_b.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.in_proj_qkv.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.in_proj_qkv.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.in_proj_z.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.in_proj_z.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.norm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.out_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.linear_attn.out_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.94.mlp.down_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.mlp.down_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.94.mlp.gate_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.mlp.gate_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.94.mlp.up_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.94.mlp.up_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.94.post_attention_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.input_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.mlp.down_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.mlp.down_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.95.mlp.gate_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.mlp.gate_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.95.mlp.up_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.mlp.up_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.95.post_attention_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.k_norm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.k_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.k_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.o_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.o_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.q_norm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.q_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.q_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.v_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.95.self_attn.v_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.96.input_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.A_log": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.conv1d.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.dt_bias": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.in_proj_a.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.in_proj_b.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.in_proj_qkv.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.in_proj_qkv.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.in_proj_z.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.in_proj_z.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.norm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.out_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.linear_attn.out_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.96.mlp.down_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.mlp.down_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.96.mlp.gate_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.mlp.gate_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.96.mlp.up_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.96.mlp.up_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.96.post_attention_layernorm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.A_log": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.conv1d.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.dt_bias": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.in_proj_a.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.in_proj_b.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.in_proj_qkv.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.in_proj_qkv.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.in_proj_z.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.in_proj_z.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.norm.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.out_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.linear_attn.out_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.97.mlp.gate_proj.weight": "model-00020-of-00027.safetensors",
+ "model.layers.97.mlp.gate_proj.weight_scale_inv": "model-00020-of-00027.safetensors",
+ "model.layers.100.input_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.A_log": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.conv1d.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.dt_bias": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.in_proj_a.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.in_proj_b.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.in_proj_qkv.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.in_proj_qkv.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.in_proj_z.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.in_proj_z.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.norm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.out_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.linear_attn.out_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.100.mlp.down_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.mlp.down_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.100.mlp.gate_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.mlp.gate_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.100.mlp.up_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.100.mlp.up_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.100.post_attention_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.input_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.A_log": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.conv1d.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.dt_bias": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.in_proj_a.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.in_proj_b.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.in_proj_qkv.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.in_proj_qkv.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.in_proj_z.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.in_proj_z.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.norm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.out_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.linear_attn.out_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.101.mlp.down_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.mlp.down_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.101.mlp.gate_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.mlp.gate_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.101.mlp.up_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.101.mlp.up_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.101.post_attention_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.A_log": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.conv1d.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.dt_bias": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.in_proj_a.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.in_proj_b.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.in_proj_qkv.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.in_proj_qkv.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.in_proj_z.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.in_proj_z.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.norm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.out_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.linear_attn.out_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.102.mlp.gate_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.102.mlp.gate_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.97.input_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.97.mlp.down_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.97.mlp.down_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.97.mlp.up_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.97.mlp.up_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.97.post_attention_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.input_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.A_log": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.conv1d.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.dt_bias": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.in_proj_a.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.in_proj_b.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.in_proj_qkv.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.in_proj_qkv.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.in_proj_z.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.in_proj_z.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.norm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.out_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.linear_attn.out_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.98.mlp.down_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.mlp.down_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.98.mlp.gate_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.mlp.gate_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.98.mlp.up_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.98.mlp.up_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.98.post_attention_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.input_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.mlp.down_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.mlp.down_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.99.mlp.gate_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.mlp.gate_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.99.mlp.up_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.mlp.up_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.99.post_attention_layernorm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.k_norm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.k_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.k_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.o_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.o_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.q_norm.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.q_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.q_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.v_proj.weight": "model-00021-of-00027.safetensors",
+ "model.layers.99.self_attn.v_proj.weight_scale_inv": "model-00021-of-00027.safetensors",
+ "model.layers.102.input_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.102.mlp.down_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.102.mlp.down_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.102.mlp.up_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.102.mlp.up_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.102.post_attention_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.input_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.mlp.down_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.mlp.down_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.103.mlp.gate_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.mlp.gate_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.103.mlp.up_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.mlp.up_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.103.post_attention_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.k_norm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.k_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.k_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.o_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.o_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.q_norm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.q_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.q_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.v_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.103.self_attn.v_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.104.input_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.A_log": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.conv1d.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.dt_bias": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.in_proj_a.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.in_proj_b.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.in_proj_qkv.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.in_proj_qkv.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.in_proj_z.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.in_proj_z.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.norm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.out_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.linear_attn.out_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.104.mlp.down_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.mlp.down_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.104.mlp.gate_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.mlp.gate_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.104.mlp.up_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.104.mlp.up_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.104.post_attention_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.input_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.A_log": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.conv1d.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.dt_bias": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.in_proj_a.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.in_proj_b.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.in_proj_qkv.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.in_proj_qkv.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.in_proj_z.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.in_proj_z.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.norm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.out_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.linear_attn.out_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.105.mlp.down_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.mlp.down_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.105.mlp.gate_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.mlp.gate_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.105.mlp.up_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.105.mlp.up_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.105.post_attention_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.input_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.A_log": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.conv1d.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.dt_bias": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.in_proj_a.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.in_proj_b.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.in_proj_qkv.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.in_proj_qkv.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.in_proj_z.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.in_proj_z.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.norm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.out_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.linear_attn.out_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.106.mlp.down_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.mlp.down_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.106.mlp.gate_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.mlp.gate_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.106.mlp.up_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.106.mlp.up_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.106.post_attention_layernorm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.mlp.gate_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.mlp.gate_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.k_norm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.k_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.k_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.o_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.o_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.q_norm.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.q_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.q_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.v_proj.weight": "model-00022-of-00027.safetensors",
+ "model.layers.107.self_attn.v_proj.weight_scale_inv": "model-00022-of-00027.safetensors",
+ "model.layers.107.input_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.107.mlp.down_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.107.mlp.down_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.107.mlp.up_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.107.mlp.up_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.107.post_attention_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.input_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.A_log": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.conv1d.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.dt_bias": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.in_proj_a.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.in_proj_b.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.in_proj_qkv.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.in_proj_qkv.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.in_proj_z.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.in_proj_z.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.norm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.out_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.linear_attn.out_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.108.mlp.down_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.mlp.down_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.108.mlp.gate_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.mlp.gate_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.108.mlp.up_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.108.mlp.up_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.108.post_attention_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.input_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.A_log": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.conv1d.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.dt_bias": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.in_proj_a.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.in_proj_b.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.in_proj_qkv.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.in_proj_qkv.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.in_proj_z.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.in_proj_z.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.norm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.out_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.linear_attn.out_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.109.mlp.down_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.mlp.down_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.109.mlp.gate_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.mlp.gate_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.109.mlp.up_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.109.mlp.up_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.109.post_attention_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.input_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.A_log": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.conv1d.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.dt_bias": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.in_proj_a.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.in_proj_b.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.in_proj_qkv.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.in_proj_qkv.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.in_proj_z.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.in_proj_z.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.norm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.out_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.linear_attn.out_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.110.mlp.down_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.mlp.down_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.110.mlp.gate_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.mlp.gate_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.110.mlp.up_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.110.mlp.up_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.110.post_attention_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.input_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.mlp.down_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.mlp.down_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.111.mlp.gate_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.mlp.gate_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.111.mlp.up_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.mlp.up_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.111.post_attention_layernorm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.k_norm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.k_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.k_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.o_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.o_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.q_norm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.q_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.q_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.v_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.111.self_attn.v_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.A_log": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.conv1d.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.dt_bias": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.in_proj_a.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.in_proj_b.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.in_proj_qkv.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.in_proj_qkv.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.in_proj_z.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.in_proj_z.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.norm.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.out_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.linear_attn.out_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.112.mlp.gate_proj.weight": "model-00023-of-00027.safetensors",
+ "model.layers.112.mlp.gate_proj.weight_scale_inv": "model-00023-of-00027.safetensors",
+ "model.layers.112.input_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.112.mlp.down_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.112.mlp.down_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.112.mlp.up_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.112.mlp.up_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.112.post_attention_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.input_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.A_log": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.conv1d.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.dt_bias": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.in_proj_a.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.in_proj_b.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.in_proj_qkv.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.in_proj_qkv.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.in_proj_z.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.in_proj_z.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.norm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.out_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.linear_attn.out_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.113.mlp.down_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.mlp.down_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.113.mlp.gate_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.mlp.gate_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.113.mlp.up_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.113.mlp.up_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.113.post_attention_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.input_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.A_log": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.conv1d.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.dt_bias": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.in_proj_a.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.in_proj_b.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.in_proj_qkv.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.in_proj_qkv.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.in_proj_z.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.in_proj_z.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.norm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.out_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.linear_attn.out_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.114.mlp.down_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.mlp.down_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.114.mlp.gate_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.mlp.gate_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.114.mlp.up_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.114.mlp.up_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.114.post_attention_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.input_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.mlp.down_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.mlp.down_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.115.mlp.gate_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.mlp.gate_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.115.mlp.up_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.mlp.up_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.115.post_attention_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.k_norm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.k_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.k_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.o_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.o_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.q_norm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.q_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.q_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.v_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.115.self_attn.v_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.116.input_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.A_log": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.conv1d.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.dt_bias": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.in_proj_a.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.in_proj_b.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.in_proj_qkv.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.in_proj_qkv.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.in_proj_z.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.in_proj_z.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.norm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.out_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.linear_attn.out_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.116.mlp.down_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.mlp.down_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.116.mlp.gate_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.mlp.gate_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.116.mlp.up_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.116.mlp.up_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.116.post_attention_layernorm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.A_log": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.conv1d.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.dt_bias": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.in_proj_a.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.in_proj_b.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.in_proj_qkv.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.in_proj_qkv.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.in_proj_z.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.in_proj_z.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.norm.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.out_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.linear_attn.out_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.117.mlp.gate_proj.weight": "model-00024-of-00027.safetensors",
+ "model.layers.117.mlp.gate_proj.weight_scale_inv": "model-00024-of-00027.safetensors",
+ "model.layers.117.input_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.117.mlp.down_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.117.mlp.down_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.117.mlp.up_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.117.mlp.up_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.117.post_attention_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.input_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.A_log": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.conv1d.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.dt_bias": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.in_proj_a.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.in_proj_b.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.in_proj_qkv.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.in_proj_qkv.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.in_proj_z.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.in_proj_z.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.norm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.out_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.linear_attn.out_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.118.mlp.down_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.mlp.down_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.118.mlp.gate_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.mlp.gate_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.118.mlp.up_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.118.mlp.up_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.118.post_attention_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.input_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.mlp.down_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.mlp.down_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.119.mlp.gate_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.mlp.gate_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.119.mlp.up_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.mlp.up_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.119.post_attention_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.k_norm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.k_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.k_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.o_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.o_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.q_norm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.q_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.q_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.v_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.119.self_attn.v_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.120.input_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.A_log": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.conv1d.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.dt_bias": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.in_proj_a.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.in_proj_b.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.in_proj_qkv.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.in_proj_qkv.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.in_proj_z.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.in_proj_z.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.norm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.out_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.linear_attn.out_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.120.mlp.down_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.mlp.down_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.120.mlp.gate_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.mlp.gate_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.120.mlp.up_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.120.mlp.up_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.120.post_attention_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.input_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.A_log": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.conv1d.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.dt_bias": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.in_proj_a.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.in_proj_b.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.in_proj_qkv.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.in_proj_qkv.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.in_proj_z.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.in_proj_z.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.norm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.out_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.linear_attn.out_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.121.mlp.down_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.mlp.down_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.121.mlp.gate_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.mlp.gate_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.121.mlp.up_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.121.mlp.up_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.121.post_attention_layernorm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.A_log": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.conv1d.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.dt_bias": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.in_proj_a.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.in_proj_b.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.in_proj_qkv.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.in_proj_qkv.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.in_proj_z.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.in_proj_z.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.norm.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.out_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.linear_attn.out_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.122.mlp.gate_proj.weight": "model-00025-of-00027.safetensors",
+ "model.layers.122.mlp.gate_proj.weight_scale_inv": "model-00025-of-00027.safetensors",
+ "model.layers.122.input_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.122.mlp.down_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.122.mlp.down_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.122.mlp.up_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.122.mlp.up_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.122.post_attention_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.input_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.mlp.down_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.mlp.down_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.123.mlp.gate_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.mlp.gate_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.123.mlp.up_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.mlp.up_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.123.post_attention_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.k_norm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.k_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.k_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.o_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.o_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.q_norm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.q_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.q_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.v_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.123.self_attn.v_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.124.input_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.A_log": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.conv1d.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.dt_bias": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.in_proj_a.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.in_proj_b.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.in_proj_qkv.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.in_proj_qkv.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.in_proj_z.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.in_proj_z.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.norm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.out_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.linear_attn.out_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.124.mlp.down_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.mlp.down_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.124.mlp.gate_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.mlp.gate_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.124.mlp.up_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.124.mlp.up_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.124.post_attention_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.input_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.A_log": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.conv1d.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.dt_bias": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.in_proj_a.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.in_proj_b.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.in_proj_qkv.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.in_proj_qkv.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.in_proj_z.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.in_proj_z.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.norm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.out_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.linear_attn.out_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.125.mlp.down_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.mlp.down_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.125.mlp.gate_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.mlp.gate_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.125.mlp.up_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.125.mlp.up_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.125.post_attention_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.input_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.A_log": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.conv1d.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.dt_bias": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.in_proj_a.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.in_proj_b.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.in_proj_qkv.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.in_proj_qkv.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.in_proj_z.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.in_proj_z.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.norm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.out_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.linear_attn.out_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.126.mlp.down_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.mlp.down_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.126.mlp.gate_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.mlp.gate_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.126.mlp.up_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.126.mlp.up_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.126.post_attention_layernorm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.mlp.gate_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.mlp.gate_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.k_norm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.k_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.k_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.o_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.o_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.q_norm.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.q_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.q_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.v_proj.weight": "model-00026-of-00027.safetensors",
+ "model.layers.127.self_attn.v_proj.weight_scale_inv": "model-00026-of-00027.safetensors",
+ "lm_head.weight": "model-00027-of-00027.safetensors",
+ "model.layers.127.input_layernorm.weight": "model-00027-of-00027.safetensors",
+ "model.layers.127.mlp.down_proj.weight": "model-00027-of-00027.safetensors",
+ "model.layers.127.mlp.down_proj.weight_scale_inv": "model-00027-of-00027.safetensors",
+ "model.layers.127.mlp.up_proj.weight": "model-00027-of-00027.safetensors",
+ "model.layers.127.mlp.up_proj.weight_scale_inv": "model-00027-of-00027.safetensors",
+ "model.layers.127.post_attention_layernorm.weight": "model-00027-of-00027.safetensors",
+ "model.norm.weight": "model-00027-of-00027.safetensors",
+ "mtp.fc.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.input_layernorm.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.mlp.down_proj.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.mlp.gate_proj.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.mlp.up_proj.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.post_attention_layernorm.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.self_attn.k_norm.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.self_attn.k_proj.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.self_attn.o_proj.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.self_attn.q_norm.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.self_attn.q_proj.weight": "model-00027-of-00027.safetensors",
+ "mtp.layers.0.self_attn.v_proj.weight": "model-00027-of-00027.safetensors",
+ "mtp.norm.weight": "model-00027-of-00027.safetensors",
+ "mtp.pre_fc_norm_embedding.weight": "model-00027-of-00027.safetensors",
+ "mtp.pre_fc_norm_hidden.weight": "model-00027-of-00027.safetensors"
+ }
+}
\ No newline at end of file
diff --git a/sft_final_fp8/system_prompt.txt b/sft_final_fp8/system_prompt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d8a820acc8ba81380252f0545448846034adcdd2
--- /dev/null
+++ b/sft_final_fp8/system_prompt.txt
@@ -0,0 +1,6 @@
+You are JiRack, an assistant that answers immediately and to the point.
+Never use a reasoning block wrapped in think tags.
+Do not show internal reasoning, drafts, or step-by-step thought process.
+Give the final, ready answer right away, with no preliminary explanation of your reasoning.
+If you feel the need to think through the question, do it silently, internally,
+and output only the final answer to the user.
diff --git a/sft_final_fp8/tokenizer.json b/sft_final_fp8/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..adaf9dba01362c086cfcbbf51444ba709f7e0a2c
--- /dev/null
+++ b/sft_final_fp8/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7befaf450030f2fc58d7ea6b3941216ec6cccd85115bf7c2bd5d127f2632cff1
+size 20010936
diff --git a/sft_final_fp8/tokenizer_config.json b/sft_final_fp8/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..a453e3588da8daccea8f570c35872c97642c2a07
--- /dev/null
+++ b/sft_final_fp8/tokenizer_config.json
@@ -0,0 +1,148 @@
+{
+ "add_prefix_space": false,
+ "audio_bos_token": "<|audio_start|>",
+ "audio_eos_token": "<|audio_end|>",
+ "audio_token": "<|audio_pad|>",
+ "backend": "tokenizers",
+ "bos_token": null,
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "<|im_end|>",
+ "errors": "replace",
+ "extra_special_tokens": [
+ "<|image|>",
+ "<|video|>",
+ "<|sound|>",
+ "<|voice|>",
+ "<|listening|>",
+ "<|vision|>",
+ "<|mood_happy|>",
+ "<|mood_sad|>",
+ "<|mood_angry|>",
+ "<|mood_neutral|>",
+ "",
+ "",
+ "",
+ "<|action_start|>",
+ "<|action_end|>",
+ "<|trajectory_start|>",
+ "<|trajectory_end|>",
+ "<|joint_start|>",
+ "<|joint_end|>",
+ "<|sensor_start|>",
+ "<|sensor_end|>",
+ "<|command_start|>",
+ "<|command_end|>",
+ "<|state_start|>",
+ "<|state_end|>",
+ "<|pose|>",
+ "<|velocity|>",
+ "<|force|>",
+ "<|torque|>",
+ "<|gripper|>",
+ "<|navigation|>",
+ "<|obstacle|>",
+ "<|task_start|>",
+ "<|task_end|>",
+ "<|plan_start|>",
+ "<|plan_end|>",
+ "<|behavior_start|>",
+ "<|behavior_end|>",
+ "<|skill_start|>",
+ "<|skill_end|>",
+ "<|motor|>",
+ "<|servo|>",
+ "<|imu|>",
+ "<|lidar|>",
+ "<|camera|>",
+ "<|depth|>",
+ "<|waypoint|>",
+ "<|path|>",
+ "<|collision|>",
+ "<|grasp|>",
+ "<|release|>",
+ "<|homing|>",
+ "<|emergency_stop|>",
+ "<|calibration|>",
+ "<|manipulation|>",
+ "<|locomotion|>",
+ "<|feedback|>",
+ "<|control_loop|>",
+ "<|language|>",
+ "<|tool_call_start|>",
+ "<|tool_call_end|>",
+ "<|tool_result_start|>",
+ "<|tool_result_end|>",
+ "__SCIENCE__",
+ "__CODING__",
+ "__STOCK_EXCHANGE__",
+ "__MEDICINE__",
+ "__GOVERNMENT__",
+ "__NEWS__",
+ "__GENERAL__",
+ "__MATERIAL_SCIENCE__",
+ "__ELECTRONICS__",
+ "__MICROELECTRONICS__",
+ "__ENGINEERING__",
+ "__ROBOTICS__",
+ "__ENERGY__",
+ "__AUTOMOTIVE__",
+ "__AVIATION__",
+ "__MATH__",
+ "__PYTHON__",
+ "__C__",
+ "__CPP__",
+ "__C_SHARP__",
+ "__JAVA__",
+ "__JAVASCRIPT__",
+ "__TYPESCRIPT__",
+ "__RUST__",
+ "__GO__",
+ "__RUBY__",
+ "__PHP__",
+ "__SWIFT__",
+ "__KOTLIN__",
+ "__BASH__",
+ "__SQL__",
+ "__ASSEMBLY__",
+ "__PHILOSOPHY__",
+ "__LITERATURE__",
+ "__SOCIOLOGY__",
+ "__PSYCHOLOGY__",
+ "__POLITICAL_SCIENCE__",
+ "__CULTURAL_STUDIES__",
+ "__ETHNOGRAPHY__",
+ "__HUMAN_RIGHTS__",
+ "__COMPLIANCE__",
+ "__MILITARY__",
+ "__BANKING__",
+ "__OIL_INDUSTRY__",
+ "__LIGHT_INDUSTRY__",
+ "__NATURE__",
+ "__OCEAN__",
+ "__SPORT__",
+ "__CULINARY__",
+ "__TRAVEL__",
+ "__HOBBY__"
+ ],
+ "image_token": "<|image_pad|>",
+ "is_local": false,
+ "local_files_only": false,
+ "model_max_length": 262144,
+ "model_specific_special_tokens": {
+ "audio_bos_token": "<|audio_start|>",
+ "audio_eos_token": "<|audio_end|>",
+ "audio_token": "<|audio_pad|>",
+ "image_token": "<|image_pad|>",
+ "video_token": "<|video_pad|>",
+ "vision_bos_token": "<|vision_start|>",
+ "vision_eos_token": "<|vision_end|>"
+ },
+ "pad_token": "<|endoftext|>",
+ "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+ "split_special_tokens": false,
+ "tokenizer_class": "Qwen2Tokenizer",
+ "unk_token": null,
+ "video_token": "<|video_pad|>",
+ "vision_bos_token": "<|vision_start|>",
+ "vision_eos_token": "<|vision_end|>"
+}