| { |
| "format": "modilify_mk2_native_mlx_inference_v1", |
| "created_at": "2026-10-06T07:00:46.116333+00:00", |
| "state_schema_version": 25, |
| "training_scheme": "gold_prefix_shared_commit_0_256_committed_ce_calibration_causal_throughput_terminal_sft", |
| "memory_scheme": "dual_timescale_gdn2_trajectory_memory", |
| "memory_architecture": "compact_gdn2_v2", |
| "global_step": 1250, |
| "lora_config": { |
| "r": 16, |
| "alpha": 16, |
| "dropout": 0.05, |
| "target_modules": [ |
| "q_proj", |
| "k_proj", |
| "v_proj", |
| "o_proj", |
| "gate_proj", |
| "up_proj", |
| "down_proj", |
| "proj" |
| ], |
| "expert_r": 8, |
| "expert_alpha": 8 |
| }, |
| "precision_policy": "gdn2_small_fp32_v1", |
| "release_version": "1.0.0", |
| "model_id": "Modilify/Modilify-Mk2-preview-mlx", |
| "base_model": "google/diffusiongemma-26B-A4B-it", |
| "source_manifest_sha256": "450600bb96c5d3d2c85dff82e7de2bede33a20df142f57c1f765b71322bbb827", |
| "trainable_names": [ |
| "latent_deliberation.blocks.0.attn.k_norm.weight", |
| "latent_deliberation.blocks.0.attn.k_proj.weight", |
| "latent_deliberation.blocks.0.attn.o_proj.weight", |
| "latent_deliberation.blocks.0.attn.q_norm.weight", |
| "latent_deliberation.blocks.0.attn.q_proj.weight", |
| "latent_deliberation.blocks.0.attn.v_proj.weight", |
| "latent_deliberation.blocks.0.ff.down.weight", |
| "latent_deliberation.blocks.0.ff.gate.weight", |
| "latent_deliberation.blocks.0.ff.up.weight", |
| "latent_deliberation.blocks.0.ff_norm.weight", |
| "latent_deliberation.blocks.0.norm.weight", |
| "latent_deliberation.blocks.1.attn.k_norm.weight", |
| "latent_deliberation.blocks.1.attn.k_proj.weight", |
| "latent_deliberation.blocks.1.attn.o_proj.weight", |
| "latent_deliberation.blocks.1.attn.q_norm.weight", |
| "latent_deliberation.blocks.1.attn.q_proj.weight", |
| "latent_deliberation.blocks.1.attn.v_proj.weight", |
| "latent_deliberation.blocks.1.ff.down.weight", |
| "latent_deliberation.blocks.1.ff.gate.weight", |
| "latent_deliberation.blocks.1.ff.up.weight", |
| "latent_deliberation.blocks.1.ff_norm.weight", |
| "latent_deliberation.blocks.1.norm.weight", |
| "latent_deliberation.blocks.2.attn.k_norm.weight", |
| "latent_deliberation.blocks.2.attn.k_proj.weight", |
| "latent_deliberation.blocks.2.attn.o_proj.weight", |
| "latent_deliberation.blocks.2.attn.q_norm.weight", |
| "latent_deliberation.blocks.2.attn.q_proj.weight", |
| "latent_deliberation.blocks.2.attn.v_proj.weight", |
| "latent_deliberation.blocks.2.ff.down.weight", |
| "latent_deliberation.blocks.2.ff.gate.weight", |
| "latent_deliberation.blocks.2.ff.up.weight", |
| "latent_deliberation.blocks.2.ff_norm.weight", |
| "latent_deliberation.blocks.2.norm.weight", |
| "latent_deliberation.blocks.3.attn.k_norm.weight", |
| "latent_deliberation.blocks.3.attn.k_proj.weight", |
| "latent_deliberation.blocks.3.attn.o_proj.weight", |
| "latent_deliberation.blocks.3.attn.q_norm.weight", |
| "latent_deliberation.blocks.3.attn.q_proj.weight", |
| "latent_deliberation.blocks.3.attn.v_proj.weight", |
| "latent_deliberation.blocks.3.ff.down.weight", |
| "latent_deliberation.blocks.3.ff.gate.weight", |
| "latent_deliberation.blocks.3.ff.up.weight", |
| "latent_deliberation.blocks.3.ff_norm.weight", |
| "latent_deliberation.blocks.3.norm.weight", |
| "latent_deliberation.experience_in.weight", |
| "latent_deliberation.output_norm.weight", |
| "latent_deliberation.persistent_memory_bus.alpha", |
| "latent_deliberation.reason_embed.weight", |
| "latent_deliberation.trajectory.cell.a_log", |
| "latent_deliberation.trajectory.cell.b_proj.weight", |
| "latent_deliberation.trajectory.cell.dt_bias", |
| "latent_deliberation.trajectory.cell.f_proj.down.weight", |
| "latent_deliberation.trajectory.cell.f_proj.up.weight", |
| "latent_deliberation.trajectory.cell.g_proj.down.weight", |
| "latent_deliberation.trajectory.cell.g_proj.up.weight", |
| "latent_deliberation.trajectory.cell.k_proj.weight", |
| "latent_deliberation.trajectory.cell.o_proj.weight", |
| "latent_deliberation.trajectory.cell.q_proj.weight", |
| "latent_deliberation.trajectory.cell.v_proj.weight", |
| "latent_deliberation.trajectory.cell.w_proj.weight", |
| "latent_deliberation.trajectory.persistent.a_log", |
| "latent_deliberation.trajectory.persistent.b_proj.weight", |
| "latent_deliberation.trajectory.persistent.dt_bias", |
| "latent_deliberation.trajectory.persistent.f_proj.down.weight", |
| "latent_deliberation.trajectory.persistent.f_proj.up.weight", |
| "latent_deliberation.trajectory.persistent.g_proj.down.weight", |
| "latent_deliberation.trajectory.persistent.g_proj.up.weight", |
| "latent_deliberation.trajectory.persistent.k_proj.weight", |
| "latent_deliberation.trajectory.persistent.o_proj.weight", |
| "latent_deliberation.trajectory.persistent.q_proj.weight", |
| "latent_deliberation.trajectory.persistent.v_proj.weight", |
| "latent_deliberation.trajectory.persistent.w_proj.weight", |
| "latent_deliberation.trajectory.probe_embed.weight", |
| "latent_deliberation.trajectory.row.a_log", |
| "latent_deliberation.trajectory.row.b_proj.weight", |
| "latent_deliberation.trajectory.row.dt_bias", |
| "latent_deliberation.trajectory.row.f_proj.down.weight", |
| "latent_deliberation.trajectory.row.f_proj.up.weight", |
| "latent_deliberation.trajectory.row.g_proj.down.weight", |
| "latent_deliberation.trajectory.row.g_proj.up.weight", |
| "latent_deliberation.trajectory.row.k_proj.weight", |
| "latent_deliberation.trajectory.row.o_proj.weight", |
| "latent_deliberation.trajectory.row.q_proj.weight", |
| "latent_deliberation.trajectory.row.v_proj.weight", |
| "latent_deliberation.trajectory.row.w_proj.weight", |
| "latent_deliberation.working_memory_bus.address_norm.weight", |
| "latent_deliberation.working_memory_bus.alpha", |
| "latent_deliberation.working_memory_bus.k_proj.weight", |
| "latent_deliberation.working_memory_bus.memory_norm.weight", |
| "latent_deliberation.working_memory_bus.o_proj.0.weight", |
| "latent_deliberation.working_memory_bus.o_proj.1.weight", |
| "latent_deliberation.working_memory_bus.o_proj.2.weight", |
| "latent_deliberation.working_memory_bus.o_proj.3.weight", |
| "latent_deliberation.working_memory_bus.o_proj.4.weight", |
| "latent_deliberation.working_memory_bus.q_norm.weight", |
| "latent_deliberation.working_memory_bus.q_proj.0.weight", |
| "latent_deliberation.working_memory_bus.q_proj.1.weight", |
| "latent_deliberation.working_memory_bus.q_proj.2.weight", |
| "latent_deliberation.working_memory_bus.q_proj.3.weight", |
| "latent_deliberation.working_memory_bus.q_proj.4.weight", |
| "latent_deliberation.working_memory_bus.rel_bias", |
| "latent_deliberation.working_memory_bus.v_proj.weight", |
| "model.decoder.layers.0.experts.down_proj.lora_a", |
| "model.decoder.layers.0.experts.down_proj.lora_b", |
| "model.decoder.layers.0.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.0.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.0.mlp.down_proj.lora_a", |
| "model.decoder.layers.0.mlp.down_proj.lora_b", |
| "model.decoder.layers.0.mlp.gate_proj.lora_a", |
| "model.decoder.layers.0.mlp.gate_proj.lora_b", |
| "model.decoder.layers.0.mlp.up_proj.lora_a", |
| "model.decoder.layers.0.mlp.up_proj.lora_b", |
| "model.decoder.layers.0.self_attn.k_proj.lora_a", |
| "model.decoder.layers.0.self_attn.k_proj.lora_b", |
| "model.decoder.layers.0.self_attn.o_proj.lora_a", |
| "model.decoder.layers.0.self_attn.o_proj.lora_b", |
| "model.decoder.layers.0.self_attn.q_proj.lora_a", |
| "model.decoder.layers.0.self_attn.q_proj.lora_b", |
| "model.decoder.layers.0.self_attn.v_proj.lora_a", |
| "model.decoder.layers.0.self_attn.v_proj.lora_b", |
| "model.decoder.layers.1.experts.down_proj.lora_a", |
| "model.decoder.layers.1.experts.down_proj.lora_b", |
| "model.decoder.layers.1.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.1.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.1.mlp.down_proj.lora_a", |
| "model.decoder.layers.1.mlp.down_proj.lora_b", |
| "model.decoder.layers.1.mlp.gate_proj.lora_a", |
| "model.decoder.layers.1.mlp.gate_proj.lora_b", |
| "model.decoder.layers.1.mlp.up_proj.lora_a", |
| "model.decoder.layers.1.mlp.up_proj.lora_b", |
| "model.decoder.layers.1.self_attn.k_proj.lora_a", |
| "model.decoder.layers.1.self_attn.k_proj.lora_b", |
| "model.decoder.layers.1.self_attn.o_proj.lora_a", |
| "model.decoder.layers.1.self_attn.o_proj.lora_b", |
| "model.decoder.layers.1.self_attn.q_proj.lora_a", |
| "model.decoder.layers.1.self_attn.q_proj.lora_b", |
| "model.decoder.layers.1.self_attn.v_proj.lora_a", |
| "model.decoder.layers.1.self_attn.v_proj.lora_b", |
| "model.decoder.layers.10.experts.down_proj.lora_a", |
| "model.decoder.layers.10.experts.down_proj.lora_b", |
| "model.decoder.layers.10.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.10.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.10.mlp.down_proj.lora_a", |
| "model.decoder.layers.10.mlp.down_proj.lora_b", |
| "model.decoder.layers.10.mlp.gate_proj.lora_a", |
| "model.decoder.layers.10.mlp.gate_proj.lora_b", |
| "model.decoder.layers.10.mlp.up_proj.lora_a", |
| "model.decoder.layers.10.mlp.up_proj.lora_b", |
| "model.decoder.layers.10.self_attn.k_proj.lora_a", |
| "model.decoder.layers.10.self_attn.k_proj.lora_b", |
| "model.decoder.layers.10.self_attn.o_proj.lora_a", |
| "model.decoder.layers.10.self_attn.o_proj.lora_b", |
| "model.decoder.layers.10.self_attn.q_proj.lora_a", |
| "model.decoder.layers.10.self_attn.q_proj.lora_b", |
| "model.decoder.layers.10.self_attn.v_proj.lora_a", |
| "model.decoder.layers.10.self_attn.v_proj.lora_b", |
| "model.decoder.layers.11.experts.down_proj.lora_a", |
| "model.decoder.layers.11.experts.down_proj.lora_b", |
| "model.decoder.layers.11.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.11.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.11.mlp.down_proj.lora_a", |
| "model.decoder.layers.11.mlp.down_proj.lora_b", |
| "model.decoder.layers.11.mlp.gate_proj.lora_a", |
| "model.decoder.layers.11.mlp.gate_proj.lora_b", |
| "model.decoder.layers.11.mlp.up_proj.lora_a", |
| "model.decoder.layers.11.mlp.up_proj.lora_b", |
| "model.decoder.layers.11.self_attn.k_proj.lora_a", |
| "model.decoder.layers.11.self_attn.k_proj.lora_b", |
| "model.decoder.layers.11.self_attn.o_proj.lora_a", |
| "model.decoder.layers.11.self_attn.o_proj.lora_b", |
| "model.decoder.layers.11.self_attn.q_proj.lora_a", |
| "model.decoder.layers.11.self_attn.q_proj.lora_b", |
| "model.decoder.layers.12.experts.down_proj.lora_a", |
| "model.decoder.layers.12.experts.down_proj.lora_b", |
| "model.decoder.layers.12.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.12.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.12.mlp.down_proj.lora_a", |
| "model.decoder.layers.12.mlp.down_proj.lora_b", |
| "model.decoder.layers.12.mlp.gate_proj.lora_a", |
| "model.decoder.layers.12.mlp.gate_proj.lora_b", |
| "model.decoder.layers.12.mlp.up_proj.lora_a", |
| "model.decoder.layers.12.mlp.up_proj.lora_b", |
| "model.decoder.layers.12.self_attn.k_proj.lora_a", |
| "model.decoder.layers.12.self_attn.k_proj.lora_b", |
| "model.decoder.layers.12.self_attn.o_proj.lora_a", |
| "model.decoder.layers.12.self_attn.o_proj.lora_b", |
| "model.decoder.layers.12.self_attn.q_proj.lora_a", |
| "model.decoder.layers.12.self_attn.q_proj.lora_b", |
| "model.decoder.layers.12.self_attn.v_proj.lora_a", |
| "model.decoder.layers.12.self_attn.v_proj.lora_b", |
| "model.decoder.layers.13.experts.down_proj.lora_a", |
| "model.decoder.layers.13.experts.down_proj.lora_b", |
| "model.decoder.layers.13.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.13.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.13.mlp.down_proj.lora_a", |
| "model.decoder.layers.13.mlp.down_proj.lora_b", |
| "model.decoder.layers.13.mlp.gate_proj.lora_a", |
| "model.decoder.layers.13.mlp.gate_proj.lora_b", |
| "model.decoder.layers.13.mlp.up_proj.lora_a", |
| "model.decoder.layers.13.mlp.up_proj.lora_b", |
| "model.decoder.layers.13.self_attn.k_proj.lora_a", |
| "model.decoder.layers.13.self_attn.k_proj.lora_b", |
| "model.decoder.layers.13.self_attn.o_proj.lora_a", |
| "model.decoder.layers.13.self_attn.o_proj.lora_b", |
| "model.decoder.layers.13.self_attn.q_proj.lora_a", |
| "model.decoder.layers.13.self_attn.q_proj.lora_b", |
| "model.decoder.layers.13.self_attn.v_proj.lora_a", |
| "model.decoder.layers.13.self_attn.v_proj.lora_b", |
| "model.decoder.layers.14.experts.down_proj.lora_a", |
| "model.decoder.layers.14.experts.down_proj.lora_b", |
| "model.decoder.layers.14.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.14.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.14.mlp.down_proj.lora_a", |
| "model.decoder.layers.14.mlp.down_proj.lora_b", |
| "model.decoder.layers.14.mlp.gate_proj.lora_a", |
| "model.decoder.layers.14.mlp.gate_proj.lora_b", |
| "model.decoder.layers.14.mlp.up_proj.lora_a", |
| "model.decoder.layers.14.mlp.up_proj.lora_b", |
| "model.decoder.layers.14.self_attn.k_proj.lora_a", |
| "model.decoder.layers.14.self_attn.k_proj.lora_b", |
| "model.decoder.layers.14.self_attn.o_proj.lora_a", |
| "model.decoder.layers.14.self_attn.o_proj.lora_b", |
| "model.decoder.layers.14.self_attn.q_proj.lora_a", |
| "model.decoder.layers.14.self_attn.q_proj.lora_b", |
| "model.decoder.layers.14.self_attn.v_proj.lora_a", |
| "model.decoder.layers.14.self_attn.v_proj.lora_b", |
| "model.decoder.layers.15.experts.down_proj.lora_a", |
| "model.decoder.layers.15.experts.down_proj.lora_b", |
| "model.decoder.layers.15.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.15.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.15.mlp.down_proj.lora_a", |
| "model.decoder.layers.15.mlp.down_proj.lora_b", |
| "model.decoder.layers.15.mlp.gate_proj.lora_a", |
| "model.decoder.layers.15.mlp.gate_proj.lora_b", |
| "model.decoder.layers.15.mlp.up_proj.lora_a", |
| "model.decoder.layers.15.mlp.up_proj.lora_b", |
| "model.decoder.layers.15.self_attn.k_proj.lora_a", |
| "model.decoder.layers.15.self_attn.k_proj.lora_b", |
| "model.decoder.layers.15.self_attn.o_proj.lora_a", |
| "model.decoder.layers.15.self_attn.o_proj.lora_b", |
| "model.decoder.layers.15.self_attn.q_proj.lora_a", |
| "model.decoder.layers.15.self_attn.q_proj.lora_b", |
| "model.decoder.layers.15.self_attn.v_proj.lora_a", |
| "model.decoder.layers.15.self_attn.v_proj.lora_b", |
| "model.decoder.layers.16.experts.down_proj.lora_a", |
| "model.decoder.layers.16.experts.down_proj.lora_b", |
| "model.decoder.layers.16.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.16.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.16.mlp.down_proj.lora_a", |
| "model.decoder.layers.16.mlp.down_proj.lora_b", |
| "model.decoder.layers.16.mlp.gate_proj.lora_a", |
| "model.decoder.layers.16.mlp.gate_proj.lora_b", |
| "model.decoder.layers.16.mlp.up_proj.lora_a", |
| "model.decoder.layers.16.mlp.up_proj.lora_b", |
| "model.decoder.layers.16.self_attn.k_proj.lora_a", |
| "model.decoder.layers.16.self_attn.k_proj.lora_b", |
| "model.decoder.layers.16.self_attn.o_proj.lora_a", |
| "model.decoder.layers.16.self_attn.o_proj.lora_b", |
| "model.decoder.layers.16.self_attn.q_proj.lora_a", |
| "model.decoder.layers.16.self_attn.q_proj.lora_b", |
| "model.decoder.layers.16.self_attn.v_proj.lora_a", |
| "model.decoder.layers.16.self_attn.v_proj.lora_b", |
| "model.decoder.layers.17.experts.down_proj.lora_a", |
| "model.decoder.layers.17.experts.down_proj.lora_b", |
| "model.decoder.layers.17.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.17.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.17.mlp.down_proj.lora_a", |
| "model.decoder.layers.17.mlp.down_proj.lora_b", |
| "model.decoder.layers.17.mlp.gate_proj.lora_a", |
| "model.decoder.layers.17.mlp.gate_proj.lora_b", |
| "model.decoder.layers.17.mlp.up_proj.lora_a", |
| "model.decoder.layers.17.mlp.up_proj.lora_b", |
| "model.decoder.layers.17.self_attn.k_proj.lora_a", |
| "model.decoder.layers.17.self_attn.k_proj.lora_b", |
| "model.decoder.layers.17.self_attn.o_proj.lora_a", |
| "model.decoder.layers.17.self_attn.o_proj.lora_b", |
| "model.decoder.layers.17.self_attn.q_proj.lora_a", |
| "model.decoder.layers.17.self_attn.q_proj.lora_b", |
| "model.decoder.layers.18.experts.down_proj.lora_a", |
| "model.decoder.layers.18.experts.down_proj.lora_b", |
| "model.decoder.layers.18.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.18.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.18.mlp.down_proj.lora_a", |
| "model.decoder.layers.18.mlp.down_proj.lora_b", |
| "model.decoder.layers.18.mlp.gate_proj.lora_a", |
| "model.decoder.layers.18.mlp.gate_proj.lora_b", |
| "model.decoder.layers.18.mlp.up_proj.lora_a", |
| "model.decoder.layers.18.mlp.up_proj.lora_b", |
| "model.decoder.layers.18.self_attn.k_proj.lora_a", |
| "model.decoder.layers.18.self_attn.k_proj.lora_b", |
| "model.decoder.layers.18.self_attn.o_proj.lora_a", |
| "model.decoder.layers.18.self_attn.o_proj.lora_b", |
| "model.decoder.layers.18.self_attn.q_proj.lora_a", |
| "model.decoder.layers.18.self_attn.q_proj.lora_b", |
| "model.decoder.layers.18.self_attn.v_proj.lora_a", |
| "model.decoder.layers.18.self_attn.v_proj.lora_b", |
| "model.decoder.layers.19.experts.down_proj.lora_a", |
| "model.decoder.layers.19.experts.down_proj.lora_b", |
| "model.decoder.layers.19.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.19.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.19.mlp.down_proj.lora_a", |
| "model.decoder.layers.19.mlp.down_proj.lora_b", |
| "model.decoder.layers.19.mlp.gate_proj.lora_a", |
| "model.decoder.layers.19.mlp.gate_proj.lora_b", |
| "model.decoder.layers.19.mlp.up_proj.lora_a", |
| "model.decoder.layers.19.mlp.up_proj.lora_b", |
| "model.decoder.layers.19.self_attn.k_proj.lora_a", |
| "model.decoder.layers.19.self_attn.k_proj.lora_b", |
| "model.decoder.layers.19.self_attn.o_proj.lora_a", |
| "model.decoder.layers.19.self_attn.o_proj.lora_b", |
| "model.decoder.layers.19.self_attn.q_proj.lora_a", |
| "model.decoder.layers.19.self_attn.q_proj.lora_b", |
| "model.decoder.layers.19.self_attn.v_proj.lora_a", |
| "model.decoder.layers.19.self_attn.v_proj.lora_b", |
| "model.decoder.layers.2.experts.down_proj.lora_a", |
| "model.decoder.layers.2.experts.down_proj.lora_b", |
| "model.decoder.layers.2.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.2.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.2.mlp.down_proj.lora_a", |
| "model.decoder.layers.2.mlp.down_proj.lora_b", |
| "model.decoder.layers.2.mlp.gate_proj.lora_a", |
| "model.decoder.layers.2.mlp.gate_proj.lora_b", |
| "model.decoder.layers.2.mlp.up_proj.lora_a", |
| "model.decoder.layers.2.mlp.up_proj.lora_b", |
| "model.decoder.layers.2.self_attn.k_proj.lora_a", |
| "model.decoder.layers.2.self_attn.k_proj.lora_b", |
| "model.decoder.layers.2.self_attn.o_proj.lora_a", |
| "model.decoder.layers.2.self_attn.o_proj.lora_b", |
| "model.decoder.layers.2.self_attn.q_proj.lora_a", |
| "model.decoder.layers.2.self_attn.q_proj.lora_b", |
| "model.decoder.layers.2.self_attn.v_proj.lora_a", |
| "model.decoder.layers.2.self_attn.v_proj.lora_b", |
| "model.decoder.layers.20.experts.down_proj.lora_a", |
| "model.decoder.layers.20.experts.down_proj.lora_b", |
| "model.decoder.layers.20.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.20.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.20.mlp.down_proj.lora_a", |
| "model.decoder.layers.20.mlp.down_proj.lora_b", |
| "model.decoder.layers.20.mlp.gate_proj.lora_a", |
| "model.decoder.layers.20.mlp.gate_proj.lora_b", |
| "model.decoder.layers.20.mlp.up_proj.lora_a", |
| "model.decoder.layers.20.mlp.up_proj.lora_b", |
| "model.decoder.layers.20.self_attn.k_proj.lora_a", |
| "model.decoder.layers.20.self_attn.k_proj.lora_b", |
| "model.decoder.layers.20.self_attn.o_proj.lora_a", |
| "model.decoder.layers.20.self_attn.o_proj.lora_b", |
| "model.decoder.layers.20.self_attn.q_proj.lora_a", |
| "model.decoder.layers.20.self_attn.q_proj.lora_b", |
| "model.decoder.layers.20.self_attn.v_proj.lora_a", |
| "model.decoder.layers.20.self_attn.v_proj.lora_b", |
| "model.decoder.layers.21.experts.down_proj.lora_a", |
| "model.decoder.layers.21.experts.down_proj.lora_b", |
| "model.decoder.layers.21.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.21.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.21.mlp.down_proj.lora_a", |
| "model.decoder.layers.21.mlp.down_proj.lora_b", |
| "model.decoder.layers.21.mlp.gate_proj.lora_a", |
| "model.decoder.layers.21.mlp.gate_proj.lora_b", |
| "model.decoder.layers.21.mlp.up_proj.lora_a", |
| "model.decoder.layers.21.mlp.up_proj.lora_b", |
| "model.decoder.layers.21.self_attn.k_proj.lora_a", |
| "model.decoder.layers.21.self_attn.k_proj.lora_b", |
| "model.decoder.layers.21.self_attn.o_proj.lora_a", |
| "model.decoder.layers.21.self_attn.o_proj.lora_b", |
| "model.decoder.layers.21.self_attn.q_proj.lora_a", |
| "model.decoder.layers.21.self_attn.q_proj.lora_b", |
| "model.decoder.layers.21.self_attn.v_proj.lora_a", |
| "model.decoder.layers.21.self_attn.v_proj.lora_b", |
| "model.decoder.layers.22.experts.down_proj.lora_a", |
| "model.decoder.layers.22.experts.down_proj.lora_b", |
| "model.decoder.layers.22.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.22.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.22.mlp.down_proj.lora_a", |
| "model.decoder.layers.22.mlp.down_proj.lora_b", |
| "model.decoder.layers.22.mlp.gate_proj.lora_a", |
| "model.decoder.layers.22.mlp.gate_proj.lora_b", |
| "model.decoder.layers.22.mlp.up_proj.lora_a", |
| "model.decoder.layers.22.mlp.up_proj.lora_b", |
| "model.decoder.layers.22.self_attn.k_proj.lora_a", |
| "model.decoder.layers.22.self_attn.k_proj.lora_b", |
| "model.decoder.layers.22.self_attn.o_proj.lora_a", |
| "model.decoder.layers.22.self_attn.o_proj.lora_b", |
| "model.decoder.layers.22.self_attn.q_proj.lora_a", |
| "model.decoder.layers.22.self_attn.q_proj.lora_b", |
| "model.decoder.layers.22.self_attn.v_proj.lora_a", |
| "model.decoder.layers.22.self_attn.v_proj.lora_b", |
| "model.decoder.layers.23.experts.down_proj.lora_a", |
| "model.decoder.layers.23.experts.down_proj.lora_b", |
| "model.decoder.layers.23.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.23.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.23.mlp.down_proj.lora_a", |
| "model.decoder.layers.23.mlp.down_proj.lora_b", |
| "model.decoder.layers.23.mlp.gate_proj.lora_a", |
| "model.decoder.layers.23.mlp.gate_proj.lora_b", |
| "model.decoder.layers.23.mlp.up_proj.lora_a", |
| "model.decoder.layers.23.mlp.up_proj.lora_b", |
| "model.decoder.layers.23.self_attn.k_proj.lora_a", |
| "model.decoder.layers.23.self_attn.k_proj.lora_b", |
| "model.decoder.layers.23.self_attn.o_proj.lora_a", |
| "model.decoder.layers.23.self_attn.o_proj.lora_b", |
| "model.decoder.layers.23.self_attn.q_proj.lora_a", |
| "model.decoder.layers.23.self_attn.q_proj.lora_b", |
| "model.decoder.layers.24.experts.down_proj.lora_a", |
| "model.decoder.layers.24.experts.down_proj.lora_b", |
| "model.decoder.layers.24.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.24.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.24.mlp.down_proj.lora_a", |
| "model.decoder.layers.24.mlp.down_proj.lora_b", |
| "model.decoder.layers.24.mlp.gate_proj.lora_a", |
| "model.decoder.layers.24.mlp.gate_proj.lora_b", |
| "model.decoder.layers.24.mlp.up_proj.lora_a", |
| "model.decoder.layers.24.mlp.up_proj.lora_b", |
| "model.decoder.layers.24.self_attn.k_proj.lora_a", |
| "model.decoder.layers.24.self_attn.k_proj.lora_b", |
| "model.decoder.layers.24.self_attn.o_proj.lora_a", |
| "model.decoder.layers.24.self_attn.o_proj.lora_b", |
| "model.decoder.layers.24.self_attn.q_proj.lora_a", |
| "model.decoder.layers.24.self_attn.q_proj.lora_b", |
| "model.decoder.layers.24.self_attn.v_proj.lora_a", |
| "model.decoder.layers.24.self_attn.v_proj.lora_b", |
| "model.decoder.layers.25.experts.down_proj.lora_a", |
| "model.decoder.layers.25.experts.down_proj.lora_b", |
| "model.decoder.layers.25.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.25.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.25.mlp.down_proj.lora_a", |
| "model.decoder.layers.25.mlp.down_proj.lora_b", |
| "model.decoder.layers.25.mlp.gate_proj.lora_a", |
| "model.decoder.layers.25.mlp.gate_proj.lora_b", |
| "model.decoder.layers.25.mlp.up_proj.lora_a", |
| "model.decoder.layers.25.mlp.up_proj.lora_b", |
| "model.decoder.layers.25.self_attn.k_proj.lora_a", |
| "model.decoder.layers.25.self_attn.k_proj.lora_b", |
| "model.decoder.layers.25.self_attn.o_proj.lora_a", |
| "model.decoder.layers.25.self_attn.o_proj.lora_b", |
| "model.decoder.layers.25.self_attn.q_proj.lora_a", |
| "model.decoder.layers.25.self_attn.q_proj.lora_b", |
| "model.decoder.layers.25.self_attn.v_proj.lora_a", |
| "model.decoder.layers.25.self_attn.v_proj.lora_b", |
| "model.decoder.layers.26.experts.down_proj.lora_a", |
| "model.decoder.layers.26.experts.down_proj.lora_b", |
| "model.decoder.layers.26.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.26.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.26.mlp.down_proj.lora_a", |
| "model.decoder.layers.26.mlp.down_proj.lora_b", |
| "model.decoder.layers.26.mlp.gate_proj.lora_a", |
| "model.decoder.layers.26.mlp.gate_proj.lora_b", |
| "model.decoder.layers.26.mlp.up_proj.lora_a", |
| "model.decoder.layers.26.mlp.up_proj.lora_b", |
| "model.decoder.layers.26.self_attn.k_proj.lora_a", |
| "model.decoder.layers.26.self_attn.k_proj.lora_b", |
| "model.decoder.layers.26.self_attn.o_proj.lora_a", |
| "model.decoder.layers.26.self_attn.o_proj.lora_b", |
| "model.decoder.layers.26.self_attn.q_proj.lora_a", |
| "model.decoder.layers.26.self_attn.q_proj.lora_b", |
| "model.decoder.layers.26.self_attn.v_proj.lora_a", |
| "model.decoder.layers.26.self_attn.v_proj.lora_b", |
| "model.decoder.layers.27.experts.down_proj.lora_a", |
| "model.decoder.layers.27.experts.down_proj.lora_b", |
| "model.decoder.layers.27.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.27.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.27.mlp.down_proj.lora_a", |
| "model.decoder.layers.27.mlp.down_proj.lora_b", |
| "model.decoder.layers.27.mlp.gate_proj.lora_a", |
| "model.decoder.layers.27.mlp.gate_proj.lora_b", |
| "model.decoder.layers.27.mlp.up_proj.lora_a", |
| "model.decoder.layers.27.mlp.up_proj.lora_b", |
| "model.decoder.layers.27.self_attn.k_proj.lora_a", |
| "model.decoder.layers.27.self_attn.k_proj.lora_b", |
| "model.decoder.layers.27.self_attn.o_proj.lora_a", |
| "model.decoder.layers.27.self_attn.o_proj.lora_b", |
| "model.decoder.layers.27.self_attn.q_proj.lora_a", |
| "model.decoder.layers.27.self_attn.q_proj.lora_b", |
| "model.decoder.layers.27.self_attn.v_proj.lora_a", |
| "model.decoder.layers.27.self_attn.v_proj.lora_b", |
| "model.decoder.layers.28.experts.down_proj.lora_a", |
| "model.decoder.layers.28.experts.down_proj.lora_b", |
| "model.decoder.layers.28.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.28.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.28.mlp.down_proj.lora_a", |
| "model.decoder.layers.28.mlp.down_proj.lora_b", |
| "model.decoder.layers.28.mlp.gate_proj.lora_a", |
| "model.decoder.layers.28.mlp.gate_proj.lora_b", |
| "model.decoder.layers.28.mlp.up_proj.lora_a", |
| "model.decoder.layers.28.mlp.up_proj.lora_b", |
| "model.decoder.layers.28.self_attn.k_proj.lora_a", |
| "model.decoder.layers.28.self_attn.k_proj.lora_b", |
| "model.decoder.layers.28.self_attn.o_proj.lora_a", |
| "model.decoder.layers.28.self_attn.o_proj.lora_b", |
| "model.decoder.layers.28.self_attn.q_proj.lora_a", |
| "model.decoder.layers.28.self_attn.q_proj.lora_b", |
| "model.decoder.layers.28.self_attn.v_proj.lora_a", |
| "model.decoder.layers.28.self_attn.v_proj.lora_b", |
| "model.decoder.layers.29.experts.down_proj.lora_a", |
| "model.decoder.layers.29.experts.down_proj.lora_b", |
| "model.decoder.layers.29.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.29.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.29.mlp.down_proj.lora_a", |
| "model.decoder.layers.29.mlp.down_proj.lora_b", |
| "model.decoder.layers.29.mlp.gate_proj.lora_a", |
| "model.decoder.layers.29.mlp.gate_proj.lora_b", |
| "model.decoder.layers.29.mlp.up_proj.lora_a", |
| "model.decoder.layers.29.mlp.up_proj.lora_b", |
| "model.decoder.layers.29.self_attn.k_proj.lora_a", |
| "model.decoder.layers.29.self_attn.k_proj.lora_b", |
| "model.decoder.layers.29.self_attn.o_proj.lora_a", |
| "model.decoder.layers.29.self_attn.o_proj.lora_b", |
| "model.decoder.layers.29.self_attn.q_proj.lora_a", |
| "model.decoder.layers.29.self_attn.q_proj.lora_b", |
| "model.decoder.layers.3.experts.down_proj.lora_a", |
| "model.decoder.layers.3.experts.down_proj.lora_b", |
| "model.decoder.layers.3.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.3.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.3.mlp.down_proj.lora_a", |
| "model.decoder.layers.3.mlp.down_proj.lora_b", |
| "model.decoder.layers.3.mlp.gate_proj.lora_a", |
| "model.decoder.layers.3.mlp.gate_proj.lora_b", |
| "model.decoder.layers.3.mlp.up_proj.lora_a", |
| "model.decoder.layers.3.mlp.up_proj.lora_b", |
| "model.decoder.layers.3.self_attn.k_proj.lora_a", |
| "model.decoder.layers.3.self_attn.k_proj.lora_b", |
| "model.decoder.layers.3.self_attn.o_proj.lora_a", |
| "model.decoder.layers.3.self_attn.o_proj.lora_b", |
| "model.decoder.layers.3.self_attn.q_proj.lora_a", |
| "model.decoder.layers.3.self_attn.q_proj.lora_b", |
| "model.decoder.layers.3.self_attn.v_proj.lora_a", |
| "model.decoder.layers.3.self_attn.v_proj.lora_b", |
| "model.decoder.layers.4.experts.down_proj.lora_a", |
| "model.decoder.layers.4.experts.down_proj.lora_b", |
| "model.decoder.layers.4.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.4.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.4.mlp.down_proj.lora_a", |
| "model.decoder.layers.4.mlp.down_proj.lora_b", |
| "model.decoder.layers.4.mlp.gate_proj.lora_a", |
| "model.decoder.layers.4.mlp.gate_proj.lora_b", |
| "model.decoder.layers.4.mlp.up_proj.lora_a", |
| "model.decoder.layers.4.mlp.up_proj.lora_b", |
| "model.decoder.layers.4.self_attn.k_proj.lora_a", |
| "model.decoder.layers.4.self_attn.k_proj.lora_b", |
| "model.decoder.layers.4.self_attn.o_proj.lora_a", |
| "model.decoder.layers.4.self_attn.o_proj.lora_b", |
| "model.decoder.layers.4.self_attn.q_proj.lora_a", |
| "model.decoder.layers.4.self_attn.q_proj.lora_b", |
| "model.decoder.layers.4.self_attn.v_proj.lora_a", |
| "model.decoder.layers.4.self_attn.v_proj.lora_b", |
| "model.decoder.layers.5.experts.down_proj.lora_a", |
| "model.decoder.layers.5.experts.down_proj.lora_b", |
| "model.decoder.layers.5.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.5.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.5.mlp.down_proj.lora_a", |
| "model.decoder.layers.5.mlp.down_proj.lora_b", |
| "model.decoder.layers.5.mlp.gate_proj.lora_a", |
| "model.decoder.layers.5.mlp.gate_proj.lora_b", |
| "model.decoder.layers.5.mlp.up_proj.lora_a", |
| "model.decoder.layers.5.mlp.up_proj.lora_b", |
| "model.decoder.layers.5.self_attn.k_proj.lora_a", |
| "model.decoder.layers.5.self_attn.k_proj.lora_b", |
| "model.decoder.layers.5.self_attn.o_proj.lora_a", |
| "model.decoder.layers.5.self_attn.o_proj.lora_b", |
| "model.decoder.layers.5.self_attn.q_proj.lora_a", |
| "model.decoder.layers.5.self_attn.q_proj.lora_b", |
| "model.decoder.layers.6.experts.down_proj.lora_a", |
| "model.decoder.layers.6.experts.down_proj.lora_b", |
| "model.decoder.layers.6.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.6.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.6.mlp.down_proj.lora_a", |
| "model.decoder.layers.6.mlp.down_proj.lora_b", |
| "model.decoder.layers.6.mlp.gate_proj.lora_a", |
| "model.decoder.layers.6.mlp.gate_proj.lora_b", |
| "model.decoder.layers.6.mlp.up_proj.lora_a", |
| "model.decoder.layers.6.mlp.up_proj.lora_b", |
| "model.decoder.layers.6.self_attn.k_proj.lora_a", |
| "model.decoder.layers.6.self_attn.k_proj.lora_b", |
| "model.decoder.layers.6.self_attn.o_proj.lora_a", |
| "model.decoder.layers.6.self_attn.o_proj.lora_b", |
| "model.decoder.layers.6.self_attn.q_proj.lora_a", |
| "model.decoder.layers.6.self_attn.q_proj.lora_b", |
| "model.decoder.layers.6.self_attn.v_proj.lora_a", |
| "model.decoder.layers.6.self_attn.v_proj.lora_b", |
| "model.decoder.layers.7.experts.down_proj.lora_a", |
| "model.decoder.layers.7.experts.down_proj.lora_b", |
| "model.decoder.layers.7.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.7.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.7.mlp.down_proj.lora_a", |
| "model.decoder.layers.7.mlp.down_proj.lora_b", |
| "model.decoder.layers.7.mlp.gate_proj.lora_a", |
| "model.decoder.layers.7.mlp.gate_proj.lora_b", |
| "model.decoder.layers.7.mlp.up_proj.lora_a", |
| "model.decoder.layers.7.mlp.up_proj.lora_b", |
| "model.decoder.layers.7.self_attn.k_proj.lora_a", |
| "model.decoder.layers.7.self_attn.k_proj.lora_b", |
| "model.decoder.layers.7.self_attn.o_proj.lora_a", |
| "model.decoder.layers.7.self_attn.o_proj.lora_b", |
| "model.decoder.layers.7.self_attn.q_proj.lora_a", |
| "model.decoder.layers.7.self_attn.q_proj.lora_b", |
| "model.decoder.layers.7.self_attn.v_proj.lora_a", |
| "model.decoder.layers.7.self_attn.v_proj.lora_b", |
| "model.decoder.layers.8.experts.down_proj.lora_a", |
| "model.decoder.layers.8.experts.down_proj.lora_b", |
| "model.decoder.layers.8.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.8.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.8.mlp.down_proj.lora_a", |
| "model.decoder.layers.8.mlp.down_proj.lora_b", |
| "model.decoder.layers.8.mlp.gate_proj.lora_a", |
| "model.decoder.layers.8.mlp.gate_proj.lora_b", |
| "model.decoder.layers.8.mlp.up_proj.lora_a", |
| "model.decoder.layers.8.mlp.up_proj.lora_b", |
| "model.decoder.layers.8.self_attn.k_proj.lora_a", |
| "model.decoder.layers.8.self_attn.k_proj.lora_b", |
| "model.decoder.layers.8.self_attn.o_proj.lora_a", |
| "model.decoder.layers.8.self_attn.o_proj.lora_b", |
| "model.decoder.layers.8.self_attn.q_proj.lora_a", |
| "model.decoder.layers.8.self_attn.q_proj.lora_b", |
| "model.decoder.layers.8.self_attn.v_proj.lora_a", |
| "model.decoder.layers.8.self_attn.v_proj.lora_b", |
| "model.decoder.layers.9.experts.down_proj.lora_a", |
| "model.decoder.layers.9.experts.down_proj.lora_b", |
| "model.decoder.layers.9.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.9.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.9.mlp.down_proj.lora_a", |
| "model.decoder.layers.9.mlp.down_proj.lora_b", |
| "model.decoder.layers.9.mlp.gate_proj.lora_a", |
| "model.decoder.layers.9.mlp.gate_proj.lora_b", |
| "model.decoder.layers.9.mlp.up_proj.lora_a", |
| "model.decoder.layers.9.mlp.up_proj.lora_b", |
| "model.decoder.layers.9.self_attn.k_proj.lora_a", |
| "model.decoder.layers.9.self_attn.k_proj.lora_b", |
| "model.decoder.layers.9.self_attn.o_proj.lora_a", |
| "model.decoder.layers.9.self_attn.o_proj.lora_b", |
| "model.decoder.layers.9.self_attn.q_proj.lora_a", |
| "model.decoder.layers.9.self_attn.q_proj.lora_b", |
| "model.decoder.layers.9.self_attn.v_proj.lora_a", |
| "model.decoder.layers.9.self_attn.v_proj.lora_b" |
| ], |
| "trainable_tensor_count": 632, |
| "total_tensor_count": 1323, |
| "total_size": 51791503298, |
| "tensors": { |
| "model.decoder.embed_tokens.weight": { |
| "shape": [ |
| 262144, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.0.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.1.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.2.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.3.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.4.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 8192, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 8192 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 8192 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 8192, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.self_attn.q_norm.weight": { |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.self_attn.k_norm.weight": { |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.5.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.6.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.7.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.8.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.9.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.10.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 8192, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 8192 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 8192 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 8192, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.self_attn.q_norm.weight": { |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.self_attn.k_norm.weight": { |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.11.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.12.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.13.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.14.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.15.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.16.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 8192, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 8192 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 8192 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 8192, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.self_attn.q_norm.weight": { |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.self_attn.k_norm.weight": { |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.17.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.18.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.19.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.20.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.21.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.22.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 8192, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 8192 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 8192 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 8192, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.self_attn.q_norm.weight": { |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.self_attn.k_norm.weight": { |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.23.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.24.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.25.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.26.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.27.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 4096, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.self_attn.v_proj.linear.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.self_attn.v_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.self_attn.v_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 4096 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 4096, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.self_attn.q_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.self_attn.k_norm.weight": { |
| "shape": [ |
| 256 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.28.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.self_attn.q_proj.linear.weight": { |
| "shape": [ |
| 8192, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.self_attn.q_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.self_attn.q_proj.lora_b": { |
| "shape": [ |
| 16, |
| 8192 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.self_attn.k_proj.linear.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.self_attn.k_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.self_attn.k_proj.lora_b": { |
| "shape": [ |
| 16, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.self_attn.o_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 8192 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.self_attn.o_proj.lora_a": { |
| "shape": [ |
| 8192, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.self_attn.o_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.self_attn.q_norm.weight": { |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.self_attn.k_norm.weight": { |
| "shape": [ |
| 512 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.mlp.gate_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.mlp.gate_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.mlp.gate_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.mlp.up_proj.linear.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.mlp.up_proj.lora_a": { |
| "shape": [ |
| 2816, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.mlp.up_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.mlp.down_proj.linear.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.mlp.down_proj.lora_a": { |
| "shape": [ |
| 2112, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.mlp.down_proj.lora_b": { |
| "shape": [ |
| 16, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.input_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.post_attention_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.pre_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.post_feedforward_layernorm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.router.proj.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.router.scale": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.router.per_expert_scale": { |
| "shape": [ |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.experts.gate_up_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 1408, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.experts.gate_up_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.experts.gate_up_proj.lora_b": { |
| "shape": [ |
| 128, |
| 1408, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.experts.down_proj.linear.weight": { |
| "shape": [ |
| 128, |
| 2816, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.experts.down_proj.lora_a": { |
| "shape": [ |
| 128, |
| 8, |
| 704 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.experts.down_proj.lora_b": { |
| "shape": [ |
| 128, |
| 2816, |
| 8 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.post_feedforward_layernorm_1.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.post_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.pre_feedforward_layernorm_2.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.layers.29.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.self_conditioning.pre_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.self_conditioning.gate_proj.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.self_conditioning.up_proj.weight": { |
| "shape": [ |
| 2112, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.decoder.self_conditioning.down_proj.weight": { |
| "shape": [ |
| 2816, |
| 2112 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.0.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.1.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.2.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.3.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.4.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.5.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.6.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.7.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.8.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.9.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.10.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.11.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.12.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.13.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.14.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.15.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.16.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.17.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.18.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.19.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.20.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.21.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.22.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.23.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.24.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.25.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.26.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.27.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.28.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "model.encoder.language_model.layers.29.layer_scalar": { |
| "shape": [ |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.cell.q_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.cell.k_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.cell.v_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.cell.f_proj.down.weight": { |
| "shape": [ |
| 64, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.cell.f_proj.up.weight": { |
| "shape": [ |
| 1024, |
| 64 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.cell.b_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.cell.w_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.cell.g_proj.down.weight": { |
| "shape": [ |
| 64, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.cell.g_proj.up.weight": { |
| "shape": [ |
| 1024, |
| 64 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.cell.o_proj.weight": { |
| "shape": [ |
| 2816, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.cell.a_log": { |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.trajectory.cell.dt_bias": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.trajectory.row.q_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.row.k_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.row.v_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.row.f_proj.down.weight": { |
| "shape": [ |
| 64, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.row.f_proj.up.weight": { |
| "shape": [ |
| 1024, |
| 64 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.row.b_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.row.w_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.row.g_proj.down.weight": { |
| "shape": [ |
| 64, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.row.g_proj.up.weight": { |
| "shape": [ |
| 1024, |
| 64 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.row.o_proj.weight": { |
| "shape": [ |
| 2816, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.row.a_log": { |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.trajectory.row.dt_bias": { |
| "shape": [ |
| 16, |
| 64 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.trajectory.persistent.q_proj.weight": { |
| "shape": [ |
| 2048, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.persistent.k_proj.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.persistent.v_proj.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.persistent.f_proj.down.weight": { |
| "shape": [ |
| 128, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.persistent.f_proj.up.weight": { |
| "shape": [ |
| 2048, |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.persistent.b_proj.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.persistent.w_proj.weight": { |
| "shape": [ |
| 2048, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.persistent.g_proj.down.weight": { |
| "shape": [ |
| 128, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.persistent.g_proj.up.weight": { |
| "shape": [ |
| 2048, |
| 128 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.persistent.o_proj.weight": { |
| "shape": [ |
| 2816, |
| 2048 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.trajectory.persistent.a_log": { |
| "shape": [ |
| 16 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.trajectory.persistent.dt_bias": { |
| "shape": [ |
| 16, |
| 128 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.trajectory.probe_embed.weight": { |
| "shape": [ |
| 4, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.0.norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.0.attn.q_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.0.attn.k_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.0.attn.v_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.0.attn.o_proj.weight": { |
| "shape": [ |
| 2816, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.0.attn.q_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.0.attn.k_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.0.ff_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.0.ff.gate.weight": { |
| "shape": [ |
| 7168, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.0.ff.up.weight": { |
| "shape": [ |
| 7168, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.0.ff.down.weight": { |
| "shape": [ |
| 2816, |
| 7168 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.1.norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.1.attn.q_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.1.attn.k_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.1.attn.v_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.1.attn.o_proj.weight": { |
| "shape": [ |
| 2816, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.1.attn.q_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.1.attn.k_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.1.ff_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.1.ff.gate.weight": { |
| "shape": [ |
| 7168, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.1.ff.up.weight": { |
| "shape": [ |
| 7168, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.1.ff.down.weight": { |
| "shape": [ |
| 2816, |
| 7168 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.2.norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.2.attn.q_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.2.attn.k_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.2.attn.v_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.2.attn.o_proj.weight": { |
| "shape": [ |
| 2816, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.2.attn.q_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.2.attn.k_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.2.ff_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.2.ff.gate.weight": { |
| "shape": [ |
| 7168, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.2.ff.up.weight": { |
| "shape": [ |
| 7168, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.2.ff.down.weight": { |
| "shape": [ |
| 2816, |
| 7168 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.3.norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.3.attn.q_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.3.attn.k_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.3.attn.v_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.3.attn.o_proj.weight": { |
| "shape": [ |
| 2816, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.3.attn.q_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.3.attn.k_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.3.ff_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.blocks.3.ff.gate.weight": { |
| "shape": [ |
| 7168, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.3.ff.up.weight": { |
| "shape": [ |
| 7168, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.blocks.3.ff.down.weight": { |
| "shape": [ |
| 2816, |
| 7168 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.output_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.working_memory_bus.memory_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.working_memory_bus.address_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.working_memory_bus.k_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.working_memory_bus.v_proj.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.working_memory_bus.q_norm.weight": { |
| "shape": [ |
| 2816 |
| ], |
| "dtype": "float32" |
| }, |
| "latent_deliberation.working_memory_bus.q_proj.0.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.working_memory_bus.q_proj.1.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.working_memory_bus.q_proj.2.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.working_memory_bus.q_proj.3.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.working_memory_bus.q_proj.4.weight": { |
| "shape": [ |
| 1024, |
| 2816 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.working_memory_bus.o_proj.0.weight": { |
| "shape": [ |
| 2816, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.working_memory_bus.o_proj.1.weight": { |
| "shape": [ |
| 2816, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.working_memory_bus.o_proj.2.weight": { |
| "shape": [ |
| 2816, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.working_memory_bus.o_proj.3.weight": { |
| "shape": [ |
| 2816, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.working_memory_bus.o_proj.4.weight": { |
| "shape": [ |
| 2816, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.working_memory_bus.alpha": { |
| "shape": [ |
| 5, |
| 16 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.working_memory_bus.rel_bias": { |
| "shape": [ |
| 16, |
| 511 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.persistent_memory_bus.alpha": { |
| "shape": [ |
| 5, |
| 1 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.experience_in.weight": { |
| "shape": [ |
| 1024, |
| 8448 |
| ], |
| "dtype": "bfloat16" |
| }, |
| "latent_deliberation.reason_embed.weight": { |
| "shape": [ |
| 6, |
| 1024 |
| ], |
| "dtype": "bfloat16" |
| } |
| }, |
| "shards": [ |
| { |
| "file": "model-00001.safetensors", |
| "bytes": 776579654, |
| "sha256": "374380ce2183b488367e840c546c1f4c7d6251fd2740f8d19a89a32f257dbffb", |
| "tensors": [ |
| "latent_deliberation.blocks.0.attn.k_norm.weight", |
| "latent_deliberation.blocks.0.attn.k_proj.weight", |
| "latent_deliberation.blocks.0.attn.o_proj.weight", |
| "latent_deliberation.blocks.0.attn.q_norm.weight", |
| "latent_deliberation.blocks.0.attn.q_proj.weight", |
| "latent_deliberation.blocks.0.attn.v_proj.weight", |
| "latent_deliberation.blocks.0.ff.down.weight", |
| "latent_deliberation.blocks.0.ff.gate.weight", |
| "latent_deliberation.blocks.0.ff.up.weight", |
| "latent_deliberation.blocks.0.ff_norm.weight", |
| "latent_deliberation.blocks.0.norm.weight", |
| "latent_deliberation.blocks.1.attn.k_norm.weight", |
| "latent_deliberation.blocks.1.attn.k_proj.weight", |
| "latent_deliberation.blocks.1.attn.o_proj.weight", |
| "latent_deliberation.blocks.1.attn.q_norm.weight", |
| "latent_deliberation.blocks.1.attn.q_proj.weight", |
| "latent_deliberation.blocks.1.attn.v_proj.weight", |
| "latent_deliberation.blocks.1.ff.down.weight", |
| "latent_deliberation.blocks.1.ff.gate.weight", |
| "latent_deliberation.blocks.1.ff.up.weight", |
| "latent_deliberation.blocks.1.ff_norm.weight", |
| "latent_deliberation.blocks.1.norm.weight", |
| "latent_deliberation.blocks.2.attn.k_norm.weight", |
| "latent_deliberation.blocks.2.attn.k_proj.weight", |
| "latent_deliberation.blocks.2.attn.o_proj.weight", |
| "latent_deliberation.blocks.2.attn.q_norm.weight", |
| "latent_deliberation.blocks.2.attn.q_proj.weight", |
| "latent_deliberation.blocks.2.attn.v_proj.weight", |
| "latent_deliberation.blocks.2.ff.down.weight", |
| "latent_deliberation.blocks.2.ff.gate.weight", |
| "latent_deliberation.blocks.2.ff.up.weight", |
| "latent_deliberation.blocks.2.ff_norm.weight", |
| "latent_deliberation.blocks.2.norm.weight", |
| "latent_deliberation.blocks.3.attn.k_norm.weight", |
| "latent_deliberation.blocks.3.attn.k_proj.weight", |
| "latent_deliberation.blocks.3.attn.o_proj.weight", |
| "latent_deliberation.blocks.3.attn.q_norm.weight", |
| "latent_deliberation.blocks.3.attn.q_proj.weight", |
| "latent_deliberation.blocks.3.attn.v_proj.weight", |
| "latent_deliberation.blocks.3.ff.down.weight", |
| "latent_deliberation.blocks.3.ff.gate.weight", |
| "latent_deliberation.blocks.3.ff.up.weight", |
| "latent_deliberation.blocks.3.ff_norm.weight", |
| "latent_deliberation.blocks.3.norm.weight", |
| "latent_deliberation.experience_in.weight", |
| "latent_deliberation.output_norm.weight", |
| "latent_deliberation.persistent_memory_bus.alpha", |
| "latent_deliberation.reason_embed.weight", |
| "latent_deliberation.trajectory.cell.a_log", |
| "latent_deliberation.trajectory.cell.b_proj.weight", |
| "latent_deliberation.trajectory.cell.dt_bias", |
| "latent_deliberation.trajectory.cell.f_proj.down.weight", |
| "latent_deliberation.trajectory.cell.f_proj.up.weight", |
| "latent_deliberation.trajectory.cell.g_proj.down.weight", |
| "latent_deliberation.trajectory.cell.g_proj.up.weight", |
| "latent_deliberation.trajectory.cell.k_proj.weight", |
| "latent_deliberation.trajectory.cell.o_proj.weight", |
| "latent_deliberation.trajectory.cell.q_proj.weight", |
| "latent_deliberation.trajectory.cell.v_proj.weight", |
| "latent_deliberation.trajectory.cell.w_proj.weight", |
| "latent_deliberation.trajectory.persistent.a_log", |
| "latent_deliberation.trajectory.persistent.b_proj.weight", |
| "latent_deliberation.trajectory.persistent.dt_bias", |
| "latent_deliberation.trajectory.persistent.f_proj.down.weight", |
| "latent_deliberation.trajectory.persistent.f_proj.up.weight", |
| "latent_deliberation.trajectory.persistent.g_proj.down.weight", |
| "latent_deliberation.trajectory.persistent.g_proj.up.weight", |
| "latent_deliberation.trajectory.persistent.k_proj.weight", |
| "latent_deliberation.trajectory.persistent.o_proj.weight", |
| "latent_deliberation.trajectory.persistent.q_proj.weight", |
| "latent_deliberation.trajectory.persistent.v_proj.weight", |
| "latent_deliberation.trajectory.persistent.w_proj.weight", |
| "latent_deliberation.trajectory.probe_embed.weight", |
| "latent_deliberation.trajectory.row.a_log", |
| "latent_deliberation.trajectory.row.b_proj.weight", |
| "latent_deliberation.trajectory.row.dt_bias", |
| "latent_deliberation.trajectory.row.f_proj.down.weight", |
| "latent_deliberation.trajectory.row.f_proj.up.weight", |
| "latent_deliberation.trajectory.row.g_proj.down.weight", |
| "latent_deliberation.trajectory.row.g_proj.up.weight", |
| "latent_deliberation.trajectory.row.k_proj.weight", |
| "latent_deliberation.trajectory.row.o_proj.weight", |
| "latent_deliberation.trajectory.row.q_proj.weight", |
| "latent_deliberation.trajectory.row.v_proj.weight", |
| "latent_deliberation.trajectory.row.w_proj.weight", |
| "latent_deliberation.working_memory_bus.address_norm.weight", |
| "latent_deliberation.working_memory_bus.alpha", |
| "latent_deliberation.working_memory_bus.k_proj.weight", |
| "latent_deliberation.working_memory_bus.memory_norm.weight", |
| "latent_deliberation.working_memory_bus.o_proj.0.weight", |
| "latent_deliberation.working_memory_bus.o_proj.1.weight", |
| "latent_deliberation.working_memory_bus.o_proj.2.weight", |
| "latent_deliberation.working_memory_bus.o_proj.3.weight", |
| "latent_deliberation.working_memory_bus.o_proj.4.weight", |
| "latent_deliberation.working_memory_bus.q_norm.weight", |
| "latent_deliberation.working_memory_bus.q_proj.0.weight", |
| "latent_deliberation.working_memory_bus.q_proj.1.weight", |
| "latent_deliberation.working_memory_bus.q_proj.2.weight", |
| "latent_deliberation.working_memory_bus.q_proj.3.weight", |
| "latent_deliberation.working_memory_bus.q_proj.4.weight", |
| "latent_deliberation.working_memory_bus.rel_bias", |
| "latent_deliberation.working_memory_bus.v_proj.weight" |
| ] |
| }, |
| { |
| "file": "model-00002.safetensors", |
| "bytes": 1991115265, |
| "sha256": "3b11844cc3f237a2b8008c1a7df7ffe2d6e78b4e4789af9e8dc756d619c72cf6", |
| "tensors": [ |
| "model.decoder.embed_tokens.weight", |
| "model.decoder.layers.0.experts.down_proj.linear.weight", |
| "model.decoder.layers.0.experts.down_proj.lora_a", |
| "model.decoder.layers.0.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00003.safetensors", |
| "bytes": 1645281286, |
| "sha256": "e4ca40905b0e8da647a20a735659268fce793827a64ccdf8de747b06fb604313", |
| "tensors": [ |
| "model.decoder.layers.0.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.0.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.0.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.0.input_layernorm.weight", |
| "model.decoder.layers.0.layer_scalar", |
| "model.decoder.layers.0.mlp.down_proj.linear.weight", |
| "model.decoder.layers.0.mlp.down_proj.lora_a", |
| "model.decoder.layers.0.mlp.down_proj.lora_b", |
| "model.decoder.layers.0.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.0.mlp.gate_proj.lora_a", |
| "model.decoder.layers.0.mlp.gate_proj.lora_b", |
| "model.decoder.layers.0.mlp.up_proj.linear.weight", |
| "model.decoder.layers.0.mlp.up_proj.lora_a", |
| "model.decoder.layers.0.mlp.up_proj.lora_b", |
| "model.decoder.layers.0.post_attention_layernorm.weight", |
| "model.decoder.layers.0.post_feedforward_layernorm.weight", |
| "model.decoder.layers.0.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.0.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.0.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.0.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.0.router.per_expert_scale", |
| "model.decoder.layers.0.router.proj.weight", |
| "model.decoder.layers.0.router.scale", |
| "model.decoder.layers.0.self_attn.k_norm.weight", |
| "model.decoder.layers.0.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.0.self_attn.k_proj.lora_a", |
| "model.decoder.layers.0.self_attn.k_proj.lora_b", |
| "model.decoder.layers.0.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.0.self_attn.o_proj.lora_a", |
| "model.decoder.layers.0.self_attn.o_proj.lora_b", |
| "model.decoder.layers.0.self_attn.q_norm.weight", |
| "model.decoder.layers.0.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.0.self_attn.q_proj.lora_a", |
| "model.decoder.layers.0.self_attn.q_proj.lora_b", |
| "model.decoder.layers.0.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.0.self_attn.v_proj.lora_a", |
| "model.decoder.layers.0.self_attn.v_proj.lora_b", |
| "model.decoder.layers.1.experts.down_proj.linear.weight", |
| "model.decoder.layers.1.experts.down_proj.lora_a", |
| "model.decoder.layers.1.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00004.safetensors", |
| "bytes": 1645281307, |
| "sha256": "8909471307dfe90a1b5d908f89d62ee9456754aa3fc1f4c5a1d36f2fb7b0ca6f", |
| "tensors": [ |
| "model.decoder.layers.1.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.1.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.1.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.1.input_layernorm.weight", |
| "model.decoder.layers.1.layer_scalar", |
| "model.decoder.layers.1.mlp.down_proj.linear.weight", |
| "model.decoder.layers.1.mlp.down_proj.lora_a", |
| "model.decoder.layers.1.mlp.down_proj.lora_b", |
| "model.decoder.layers.1.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.1.mlp.gate_proj.lora_a", |
| "model.decoder.layers.1.mlp.gate_proj.lora_b", |
| "model.decoder.layers.1.mlp.up_proj.linear.weight", |
| "model.decoder.layers.1.mlp.up_proj.lora_a", |
| "model.decoder.layers.1.mlp.up_proj.lora_b", |
| "model.decoder.layers.1.post_attention_layernorm.weight", |
| "model.decoder.layers.1.post_feedforward_layernorm.weight", |
| "model.decoder.layers.1.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.1.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.1.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.1.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.1.router.per_expert_scale", |
| "model.decoder.layers.1.router.proj.weight", |
| "model.decoder.layers.1.router.scale", |
| "model.decoder.layers.1.self_attn.k_norm.weight", |
| "model.decoder.layers.1.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.1.self_attn.k_proj.lora_a", |
| "model.decoder.layers.1.self_attn.k_proj.lora_b", |
| "model.decoder.layers.1.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.1.self_attn.o_proj.lora_a", |
| "model.decoder.layers.1.self_attn.o_proj.lora_b", |
| "model.decoder.layers.1.self_attn.q_norm.weight", |
| "model.decoder.layers.1.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.1.self_attn.q_proj.lora_a", |
| "model.decoder.layers.1.self_attn.q_proj.lora_b", |
| "model.decoder.layers.1.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.1.self_attn.v_proj.lora_a", |
| "model.decoder.layers.1.self_attn.v_proj.lora_b", |
| "model.decoder.layers.10.experts.down_proj.linear.weight", |
| "model.decoder.layers.10.experts.down_proj.lora_a", |
| "model.decoder.layers.10.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00005.safetensors", |
| "bytes": 1645281376, |
| "sha256": "f213f94cb031abe72a258c325145aeb6c0a6fecf9c6d17e46298207141d27cc7", |
| "tensors": [ |
| "model.decoder.layers.10.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.10.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.10.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.10.input_layernorm.weight", |
| "model.decoder.layers.10.layer_scalar", |
| "model.decoder.layers.10.mlp.down_proj.linear.weight", |
| "model.decoder.layers.10.mlp.down_proj.lora_a", |
| "model.decoder.layers.10.mlp.down_proj.lora_b", |
| "model.decoder.layers.10.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.10.mlp.gate_proj.lora_a", |
| "model.decoder.layers.10.mlp.gate_proj.lora_b", |
| "model.decoder.layers.10.mlp.up_proj.linear.weight", |
| "model.decoder.layers.10.mlp.up_proj.lora_a", |
| "model.decoder.layers.10.mlp.up_proj.lora_b", |
| "model.decoder.layers.10.post_attention_layernorm.weight", |
| "model.decoder.layers.10.post_feedforward_layernorm.weight", |
| "model.decoder.layers.10.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.10.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.10.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.10.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.10.router.per_expert_scale", |
| "model.decoder.layers.10.router.proj.weight", |
| "model.decoder.layers.10.router.scale", |
| "model.decoder.layers.10.self_attn.k_norm.weight", |
| "model.decoder.layers.10.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.10.self_attn.k_proj.lora_a", |
| "model.decoder.layers.10.self_attn.k_proj.lora_b", |
| "model.decoder.layers.10.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.10.self_attn.o_proj.lora_a", |
| "model.decoder.layers.10.self_attn.o_proj.lora_b", |
| "model.decoder.layers.10.self_attn.q_norm.weight", |
| "model.decoder.layers.10.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.10.self_attn.q_proj.lora_a", |
| "model.decoder.layers.10.self_attn.q_proj.lora_b", |
| "model.decoder.layers.10.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.10.self_attn.v_proj.lora_a", |
| "model.decoder.layers.10.self_attn.v_proj.lora_b", |
| "model.decoder.layers.11.experts.down_proj.linear.weight", |
| "model.decoder.layers.11.experts.down_proj.lora_a", |
| "model.decoder.layers.11.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00006.safetensors", |
| "bytes": 1674191579, |
| "sha256": "7d4997e1fdc157712d55572f1f427400bdb176888e6ad420bbb8567d6e50fbde", |
| "tensors": [ |
| "model.decoder.layers.11.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.11.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.11.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.11.input_layernorm.weight", |
| "model.decoder.layers.11.layer_scalar", |
| "model.decoder.layers.11.mlp.down_proj.linear.weight", |
| "model.decoder.layers.11.mlp.down_proj.lora_a", |
| "model.decoder.layers.11.mlp.down_proj.lora_b", |
| "model.decoder.layers.11.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.11.mlp.gate_proj.lora_a", |
| "model.decoder.layers.11.mlp.gate_proj.lora_b", |
| "model.decoder.layers.11.mlp.up_proj.linear.weight", |
| "model.decoder.layers.11.mlp.up_proj.lora_a", |
| "model.decoder.layers.11.mlp.up_proj.lora_b", |
| "model.decoder.layers.11.post_attention_layernorm.weight", |
| "model.decoder.layers.11.post_feedforward_layernorm.weight", |
| "model.decoder.layers.11.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.11.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.11.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.11.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.11.router.per_expert_scale", |
| "model.decoder.layers.11.router.proj.weight", |
| "model.decoder.layers.11.router.scale", |
| "model.decoder.layers.11.self_attn.k_norm.weight", |
| "model.decoder.layers.11.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.11.self_attn.k_proj.lora_a", |
| "model.decoder.layers.11.self_attn.k_proj.lora_b", |
| "model.decoder.layers.11.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.11.self_attn.o_proj.lora_a", |
| "model.decoder.layers.11.self_attn.o_proj.lora_b", |
| "model.decoder.layers.11.self_attn.q_norm.weight", |
| "model.decoder.layers.11.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.11.self_attn.q_proj.lora_a", |
| "model.decoder.layers.11.self_attn.q_proj.lora_b", |
| "model.decoder.layers.12.experts.down_proj.linear.weight", |
| "model.decoder.layers.12.experts.down_proj.lora_a", |
| "model.decoder.layers.12.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00007.safetensors", |
| "bytes": 1645281438, |
| "sha256": "ec97a3868a1d77d35ff51e38e718db99d353ef44e0d320136966ec437f3376da", |
| "tensors": [ |
| "model.decoder.layers.12.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.12.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.12.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.12.input_layernorm.weight", |
| "model.decoder.layers.12.layer_scalar", |
| "model.decoder.layers.12.mlp.down_proj.linear.weight", |
| "model.decoder.layers.12.mlp.down_proj.lora_a", |
| "model.decoder.layers.12.mlp.down_proj.lora_b", |
| "model.decoder.layers.12.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.12.mlp.gate_proj.lora_a", |
| "model.decoder.layers.12.mlp.gate_proj.lora_b", |
| "model.decoder.layers.12.mlp.up_proj.linear.weight", |
| "model.decoder.layers.12.mlp.up_proj.lora_a", |
| "model.decoder.layers.12.mlp.up_proj.lora_b", |
| "model.decoder.layers.12.post_attention_layernorm.weight", |
| "model.decoder.layers.12.post_feedforward_layernorm.weight", |
| "model.decoder.layers.12.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.12.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.12.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.12.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.12.router.per_expert_scale", |
| "model.decoder.layers.12.router.proj.weight", |
| "model.decoder.layers.12.router.scale", |
| "model.decoder.layers.12.self_attn.k_norm.weight", |
| "model.decoder.layers.12.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.12.self_attn.k_proj.lora_a", |
| "model.decoder.layers.12.self_attn.k_proj.lora_b", |
| "model.decoder.layers.12.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.12.self_attn.o_proj.lora_a", |
| "model.decoder.layers.12.self_attn.o_proj.lora_b", |
| "model.decoder.layers.12.self_attn.q_norm.weight", |
| "model.decoder.layers.12.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.12.self_attn.q_proj.lora_a", |
| "model.decoder.layers.12.self_attn.q_proj.lora_b", |
| "model.decoder.layers.12.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.12.self_attn.v_proj.lora_a", |
| "model.decoder.layers.12.self_attn.v_proj.lora_b", |
| "model.decoder.layers.13.experts.down_proj.linear.weight", |
| "model.decoder.layers.13.experts.down_proj.lora_a", |
| "model.decoder.layers.13.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00008.safetensors", |
| "bytes": 1645281390, |
| "sha256": "c8aeeb93591a35251e6669fcb56d10101db3714468b5f4d1eecb27ae69a7cb98", |
| "tensors": [ |
| "model.decoder.layers.13.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.13.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.13.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.13.input_layernorm.weight", |
| "model.decoder.layers.13.layer_scalar", |
| "model.decoder.layers.13.mlp.down_proj.linear.weight", |
| "model.decoder.layers.13.mlp.down_proj.lora_a", |
| "model.decoder.layers.13.mlp.down_proj.lora_b", |
| "model.decoder.layers.13.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.13.mlp.gate_proj.lora_a", |
| "model.decoder.layers.13.mlp.gate_proj.lora_b", |
| "model.decoder.layers.13.mlp.up_proj.linear.weight", |
| "model.decoder.layers.13.mlp.up_proj.lora_a", |
| "model.decoder.layers.13.mlp.up_proj.lora_b", |
| "model.decoder.layers.13.post_attention_layernorm.weight", |
| "model.decoder.layers.13.post_feedforward_layernorm.weight", |
| "model.decoder.layers.13.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.13.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.13.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.13.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.13.router.per_expert_scale", |
| "model.decoder.layers.13.router.proj.weight", |
| "model.decoder.layers.13.router.scale", |
| "model.decoder.layers.13.self_attn.k_norm.weight", |
| "model.decoder.layers.13.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.13.self_attn.k_proj.lora_a", |
| "model.decoder.layers.13.self_attn.k_proj.lora_b", |
| "model.decoder.layers.13.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.13.self_attn.o_proj.lora_a", |
| "model.decoder.layers.13.self_attn.o_proj.lora_b", |
| "model.decoder.layers.13.self_attn.q_norm.weight", |
| "model.decoder.layers.13.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.13.self_attn.q_proj.lora_a", |
| "model.decoder.layers.13.self_attn.q_proj.lora_b", |
| "model.decoder.layers.13.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.13.self_attn.v_proj.lora_a", |
| "model.decoder.layers.13.self_attn.v_proj.lora_b", |
| "model.decoder.layers.14.experts.down_proj.linear.weight", |
| "model.decoder.layers.14.experts.down_proj.lora_a", |
| "model.decoder.layers.14.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00009.safetensors", |
| "bytes": 1645281404, |
| "sha256": "43b686872a4680f4d021f2ddaf71a592601972645cc6f97136708a2b529dc4c2", |
| "tensors": [ |
| "model.decoder.layers.14.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.14.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.14.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.14.input_layernorm.weight", |
| "model.decoder.layers.14.layer_scalar", |
| "model.decoder.layers.14.mlp.down_proj.linear.weight", |
| "model.decoder.layers.14.mlp.down_proj.lora_a", |
| "model.decoder.layers.14.mlp.down_proj.lora_b", |
| "model.decoder.layers.14.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.14.mlp.gate_proj.lora_a", |
| "model.decoder.layers.14.mlp.gate_proj.lora_b", |
| "model.decoder.layers.14.mlp.up_proj.linear.weight", |
| "model.decoder.layers.14.mlp.up_proj.lora_a", |
| "model.decoder.layers.14.mlp.up_proj.lora_b", |
| "model.decoder.layers.14.post_attention_layernorm.weight", |
| "model.decoder.layers.14.post_feedforward_layernorm.weight", |
| "model.decoder.layers.14.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.14.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.14.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.14.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.14.router.per_expert_scale", |
| "model.decoder.layers.14.router.proj.weight", |
| "model.decoder.layers.14.router.scale", |
| "model.decoder.layers.14.self_attn.k_norm.weight", |
| "model.decoder.layers.14.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.14.self_attn.k_proj.lora_a", |
| "model.decoder.layers.14.self_attn.k_proj.lora_b", |
| "model.decoder.layers.14.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.14.self_attn.o_proj.lora_a", |
| "model.decoder.layers.14.self_attn.o_proj.lora_b", |
| "model.decoder.layers.14.self_attn.q_norm.weight", |
| "model.decoder.layers.14.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.14.self_attn.q_proj.lora_a", |
| "model.decoder.layers.14.self_attn.q_proj.lora_b", |
| "model.decoder.layers.14.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.14.self_attn.v_proj.lora_a", |
| "model.decoder.layers.14.self_attn.v_proj.lora_b", |
| "model.decoder.layers.15.experts.down_proj.linear.weight", |
| "model.decoder.layers.15.experts.down_proj.lora_a", |
| "model.decoder.layers.15.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00010.safetensors", |
| "bytes": 1645281370, |
| "sha256": "946aa5129578785476e9e241757b64a0f4ec5777c873b576b25bbdc67d2f5f08", |
| "tensors": [ |
| "model.decoder.layers.15.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.15.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.15.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.15.input_layernorm.weight", |
| "model.decoder.layers.15.layer_scalar", |
| "model.decoder.layers.15.mlp.down_proj.linear.weight", |
| "model.decoder.layers.15.mlp.down_proj.lora_a", |
| "model.decoder.layers.15.mlp.down_proj.lora_b", |
| "model.decoder.layers.15.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.15.mlp.gate_proj.lora_a", |
| "model.decoder.layers.15.mlp.gate_proj.lora_b", |
| "model.decoder.layers.15.mlp.up_proj.linear.weight", |
| "model.decoder.layers.15.mlp.up_proj.lora_a", |
| "model.decoder.layers.15.mlp.up_proj.lora_b", |
| "model.decoder.layers.15.post_attention_layernorm.weight", |
| "model.decoder.layers.15.post_feedforward_layernorm.weight", |
| "model.decoder.layers.15.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.15.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.15.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.15.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.15.router.per_expert_scale", |
| "model.decoder.layers.15.router.proj.weight", |
| "model.decoder.layers.15.router.scale", |
| "model.decoder.layers.15.self_attn.k_norm.weight", |
| "model.decoder.layers.15.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.15.self_attn.k_proj.lora_a", |
| "model.decoder.layers.15.self_attn.k_proj.lora_b", |
| "model.decoder.layers.15.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.15.self_attn.o_proj.lora_a", |
| "model.decoder.layers.15.self_attn.o_proj.lora_b", |
| "model.decoder.layers.15.self_attn.q_norm.weight", |
| "model.decoder.layers.15.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.15.self_attn.q_proj.lora_a", |
| "model.decoder.layers.15.self_attn.q_proj.lora_b", |
| "model.decoder.layers.15.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.15.self_attn.v_proj.lora_a", |
| "model.decoder.layers.15.self_attn.v_proj.lora_b", |
| "model.decoder.layers.16.experts.down_proj.linear.weight", |
| "model.decoder.layers.16.experts.down_proj.lora_a", |
| "model.decoder.layers.16.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00011.safetensors", |
| "bytes": 1645281424, |
| "sha256": "adb4df6d408098ec73a1ed3a6f4fbad0202fd93d4af150828996720341c141ec", |
| "tensors": [ |
| "model.decoder.layers.16.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.16.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.16.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.16.input_layernorm.weight", |
| "model.decoder.layers.16.layer_scalar", |
| "model.decoder.layers.16.mlp.down_proj.linear.weight", |
| "model.decoder.layers.16.mlp.down_proj.lora_a", |
| "model.decoder.layers.16.mlp.down_proj.lora_b", |
| "model.decoder.layers.16.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.16.mlp.gate_proj.lora_a", |
| "model.decoder.layers.16.mlp.gate_proj.lora_b", |
| "model.decoder.layers.16.mlp.up_proj.linear.weight", |
| "model.decoder.layers.16.mlp.up_proj.lora_a", |
| "model.decoder.layers.16.mlp.up_proj.lora_b", |
| "model.decoder.layers.16.post_attention_layernorm.weight", |
| "model.decoder.layers.16.post_feedforward_layernorm.weight", |
| "model.decoder.layers.16.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.16.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.16.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.16.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.16.router.per_expert_scale", |
| "model.decoder.layers.16.router.proj.weight", |
| "model.decoder.layers.16.router.scale", |
| "model.decoder.layers.16.self_attn.k_norm.weight", |
| "model.decoder.layers.16.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.16.self_attn.k_proj.lora_a", |
| "model.decoder.layers.16.self_attn.k_proj.lora_b", |
| "model.decoder.layers.16.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.16.self_attn.o_proj.lora_a", |
| "model.decoder.layers.16.self_attn.o_proj.lora_b", |
| "model.decoder.layers.16.self_attn.q_norm.weight", |
| "model.decoder.layers.16.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.16.self_attn.q_proj.lora_a", |
| "model.decoder.layers.16.self_attn.q_proj.lora_b", |
| "model.decoder.layers.16.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.16.self_attn.v_proj.lora_a", |
| "model.decoder.layers.16.self_attn.v_proj.lora_b", |
| "model.decoder.layers.17.experts.down_proj.linear.weight", |
| "model.decoder.layers.17.experts.down_proj.lora_a", |
| "model.decoder.layers.17.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00012.safetensors", |
| "bytes": 1674191611, |
| "sha256": "86c9845b281f09b894ae62b3dde3574cd779417a8906b52a471b601c6ae19890", |
| "tensors": [ |
| "model.decoder.layers.17.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.17.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.17.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.17.input_layernorm.weight", |
| "model.decoder.layers.17.layer_scalar", |
| "model.decoder.layers.17.mlp.down_proj.linear.weight", |
| "model.decoder.layers.17.mlp.down_proj.lora_a", |
| "model.decoder.layers.17.mlp.down_proj.lora_b", |
| "model.decoder.layers.17.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.17.mlp.gate_proj.lora_a", |
| "model.decoder.layers.17.mlp.gate_proj.lora_b", |
| "model.decoder.layers.17.mlp.up_proj.linear.weight", |
| "model.decoder.layers.17.mlp.up_proj.lora_a", |
| "model.decoder.layers.17.mlp.up_proj.lora_b", |
| "model.decoder.layers.17.post_attention_layernorm.weight", |
| "model.decoder.layers.17.post_feedforward_layernorm.weight", |
| "model.decoder.layers.17.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.17.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.17.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.17.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.17.router.per_expert_scale", |
| "model.decoder.layers.17.router.proj.weight", |
| "model.decoder.layers.17.router.scale", |
| "model.decoder.layers.17.self_attn.k_norm.weight", |
| "model.decoder.layers.17.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.17.self_attn.k_proj.lora_a", |
| "model.decoder.layers.17.self_attn.k_proj.lora_b", |
| "model.decoder.layers.17.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.17.self_attn.o_proj.lora_a", |
| "model.decoder.layers.17.self_attn.o_proj.lora_b", |
| "model.decoder.layers.17.self_attn.q_norm.weight", |
| "model.decoder.layers.17.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.17.self_attn.q_proj.lora_a", |
| "model.decoder.layers.17.self_attn.q_proj.lora_b", |
| "model.decoder.layers.18.experts.down_proj.linear.weight", |
| "model.decoder.layers.18.experts.down_proj.lora_a", |
| "model.decoder.layers.18.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00013.safetensors", |
| "bytes": 1645281386, |
| "sha256": "b2042c1d4bdb49b8a4ed0423a86c4694f2c0cf345e6d247022bc919b818db9e4", |
| "tensors": [ |
| "model.decoder.layers.18.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.18.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.18.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.18.input_layernorm.weight", |
| "model.decoder.layers.18.layer_scalar", |
| "model.decoder.layers.18.mlp.down_proj.linear.weight", |
| "model.decoder.layers.18.mlp.down_proj.lora_a", |
| "model.decoder.layers.18.mlp.down_proj.lora_b", |
| "model.decoder.layers.18.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.18.mlp.gate_proj.lora_a", |
| "model.decoder.layers.18.mlp.gate_proj.lora_b", |
| "model.decoder.layers.18.mlp.up_proj.linear.weight", |
| "model.decoder.layers.18.mlp.up_proj.lora_a", |
| "model.decoder.layers.18.mlp.up_proj.lora_b", |
| "model.decoder.layers.18.post_attention_layernorm.weight", |
| "model.decoder.layers.18.post_feedforward_layernorm.weight", |
| "model.decoder.layers.18.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.18.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.18.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.18.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.18.router.per_expert_scale", |
| "model.decoder.layers.18.router.proj.weight", |
| "model.decoder.layers.18.router.scale", |
| "model.decoder.layers.18.self_attn.k_norm.weight", |
| "model.decoder.layers.18.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.18.self_attn.k_proj.lora_a", |
| "model.decoder.layers.18.self_attn.k_proj.lora_b", |
| "model.decoder.layers.18.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.18.self_attn.o_proj.lora_a", |
| "model.decoder.layers.18.self_attn.o_proj.lora_b", |
| "model.decoder.layers.18.self_attn.q_norm.weight", |
| "model.decoder.layers.18.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.18.self_attn.q_proj.lora_a", |
| "model.decoder.layers.18.self_attn.q_proj.lora_b", |
| "model.decoder.layers.18.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.18.self_attn.v_proj.lora_a", |
| "model.decoder.layers.18.self_attn.v_proj.lora_b", |
| "model.decoder.layers.19.experts.down_proj.linear.weight", |
| "model.decoder.layers.19.experts.down_proj.lora_a", |
| "model.decoder.layers.19.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00014.safetensors", |
| "bytes": 1645281393, |
| "sha256": "0233e8d5a91a79edd030479e287621af3bd962f080d7b1717dd03ab6fb417c97", |
| "tensors": [ |
| "model.decoder.layers.19.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.19.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.19.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.19.input_layernorm.weight", |
| "model.decoder.layers.19.layer_scalar", |
| "model.decoder.layers.19.mlp.down_proj.linear.weight", |
| "model.decoder.layers.19.mlp.down_proj.lora_a", |
| "model.decoder.layers.19.mlp.down_proj.lora_b", |
| "model.decoder.layers.19.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.19.mlp.gate_proj.lora_a", |
| "model.decoder.layers.19.mlp.gate_proj.lora_b", |
| "model.decoder.layers.19.mlp.up_proj.linear.weight", |
| "model.decoder.layers.19.mlp.up_proj.lora_a", |
| "model.decoder.layers.19.mlp.up_proj.lora_b", |
| "model.decoder.layers.19.post_attention_layernorm.weight", |
| "model.decoder.layers.19.post_feedforward_layernorm.weight", |
| "model.decoder.layers.19.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.19.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.19.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.19.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.19.router.per_expert_scale", |
| "model.decoder.layers.19.router.proj.weight", |
| "model.decoder.layers.19.router.scale", |
| "model.decoder.layers.19.self_attn.k_norm.weight", |
| "model.decoder.layers.19.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.19.self_attn.k_proj.lora_a", |
| "model.decoder.layers.19.self_attn.k_proj.lora_b", |
| "model.decoder.layers.19.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.19.self_attn.o_proj.lora_a", |
| "model.decoder.layers.19.self_attn.o_proj.lora_b", |
| "model.decoder.layers.19.self_attn.q_norm.weight", |
| "model.decoder.layers.19.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.19.self_attn.q_proj.lora_a", |
| "model.decoder.layers.19.self_attn.q_proj.lora_b", |
| "model.decoder.layers.19.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.19.self_attn.v_proj.lora_a", |
| "model.decoder.layers.19.self_attn.v_proj.lora_b", |
| "model.decoder.layers.2.experts.down_proj.linear.weight", |
| "model.decoder.layers.2.experts.down_proj.lora_a", |
| "model.decoder.layers.2.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00015.safetensors", |
| "bytes": 1645281363, |
| "sha256": "75bab4540f1aa39d31271ec3d36e414e24fa500d3bce0dc5a00da0c1a8a25582", |
| "tensors": [ |
| "model.decoder.layers.2.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.2.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.2.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.2.input_layernorm.weight", |
| "model.decoder.layers.2.layer_scalar", |
| "model.decoder.layers.2.mlp.down_proj.linear.weight", |
| "model.decoder.layers.2.mlp.down_proj.lora_a", |
| "model.decoder.layers.2.mlp.down_proj.lora_b", |
| "model.decoder.layers.2.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.2.mlp.gate_proj.lora_a", |
| "model.decoder.layers.2.mlp.gate_proj.lora_b", |
| "model.decoder.layers.2.mlp.up_proj.linear.weight", |
| "model.decoder.layers.2.mlp.up_proj.lora_a", |
| "model.decoder.layers.2.mlp.up_proj.lora_b", |
| "model.decoder.layers.2.post_attention_layernorm.weight", |
| "model.decoder.layers.2.post_feedforward_layernorm.weight", |
| "model.decoder.layers.2.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.2.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.2.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.2.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.2.router.per_expert_scale", |
| "model.decoder.layers.2.router.proj.weight", |
| "model.decoder.layers.2.router.scale", |
| "model.decoder.layers.2.self_attn.k_norm.weight", |
| "model.decoder.layers.2.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.2.self_attn.k_proj.lora_a", |
| "model.decoder.layers.2.self_attn.k_proj.lora_b", |
| "model.decoder.layers.2.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.2.self_attn.o_proj.lora_a", |
| "model.decoder.layers.2.self_attn.o_proj.lora_b", |
| "model.decoder.layers.2.self_attn.q_norm.weight", |
| "model.decoder.layers.2.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.2.self_attn.q_proj.lora_a", |
| "model.decoder.layers.2.self_attn.q_proj.lora_b", |
| "model.decoder.layers.2.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.2.self_attn.v_proj.lora_a", |
| "model.decoder.layers.2.self_attn.v_proj.lora_b", |
| "model.decoder.layers.20.experts.down_proj.linear.weight", |
| "model.decoder.layers.20.experts.down_proj.lora_a", |
| "model.decoder.layers.20.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00016.safetensors", |
| "bytes": 1645281392, |
| "sha256": "37e7c3bbf2fc81bd14f95dd5e14d49e7e6cda481b8478d64562a8be0d2c989fa", |
| "tensors": [ |
| "model.decoder.layers.20.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.20.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.20.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.20.input_layernorm.weight", |
| "model.decoder.layers.20.layer_scalar", |
| "model.decoder.layers.20.mlp.down_proj.linear.weight", |
| "model.decoder.layers.20.mlp.down_proj.lora_a", |
| "model.decoder.layers.20.mlp.down_proj.lora_b", |
| "model.decoder.layers.20.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.20.mlp.gate_proj.lora_a", |
| "model.decoder.layers.20.mlp.gate_proj.lora_b", |
| "model.decoder.layers.20.mlp.up_proj.linear.weight", |
| "model.decoder.layers.20.mlp.up_proj.lora_a", |
| "model.decoder.layers.20.mlp.up_proj.lora_b", |
| "model.decoder.layers.20.post_attention_layernorm.weight", |
| "model.decoder.layers.20.post_feedforward_layernorm.weight", |
| "model.decoder.layers.20.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.20.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.20.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.20.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.20.router.per_expert_scale", |
| "model.decoder.layers.20.router.proj.weight", |
| "model.decoder.layers.20.router.scale", |
| "model.decoder.layers.20.self_attn.k_norm.weight", |
| "model.decoder.layers.20.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.20.self_attn.k_proj.lora_a", |
| "model.decoder.layers.20.self_attn.k_proj.lora_b", |
| "model.decoder.layers.20.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.20.self_attn.o_proj.lora_a", |
| "model.decoder.layers.20.self_attn.o_proj.lora_b", |
| "model.decoder.layers.20.self_attn.q_norm.weight", |
| "model.decoder.layers.20.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.20.self_attn.q_proj.lora_a", |
| "model.decoder.layers.20.self_attn.q_proj.lora_b", |
| "model.decoder.layers.20.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.20.self_attn.v_proj.lora_a", |
| "model.decoder.layers.20.self_attn.v_proj.lora_b", |
| "model.decoder.layers.21.experts.down_proj.linear.weight", |
| "model.decoder.layers.21.experts.down_proj.lora_a", |
| "model.decoder.layers.21.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00017.safetensors", |
| "bytes": 1645281346, |
| "sha256": "82dc6c75a7ca3a36ddb16733beead912507fcb5fbc236d06b9c249b9eb22ac09", |
| "tensors": [ |
| "model.decoder.layers.21.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.21.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.21.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.21.input_layernorm.weight", |
| "model.decoder.layers.21.layer_scalar", |
| "model.decoder.layers.21.mlp.down_proj.linear.weight", |
| "model.decoder.layers.21.mlp.down_proj.lora_a", |
| "model.decoder.layers.21.mlp.down_proj.lora_b", |
| "model.decoder.layers.21.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.21.mlp.gate_proj.lora_a", |
| "model.decoder.layers.21.mlp.gate_proj.lora_b", |
| "model.decoder.layers.21.mlp.up_proj.linear.weight", |
| "model.decoder.layers.21.mlp.up_proj.lora_a", |
| "model.decoder.layers.21.mlp.up_proj.lora_b", |
| "model.decoder.layers.21.post_attention_layernorm.weight", |
| "model.decoder.layers.21.post_feedforward_layernorm.weight", |
| "model.decoder.layers.21.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.21.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.21.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.21.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.21.router.per_expert_scale", |
| "model.decoder.layers.21.router.proj.weight", |
| "model.decoder.layers.21.router.scale", |
| "model.decoder.layers.21.self_attn.k_norm.weight", |
| "model.decoder.layers.21.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.21.self_attn.k_proj.lora_a", |
| "model.decoder.layers.21.self_attn.k_proj.lora_b", |
| "model.decoder.layers.21.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.21.self_attn.o_proj.lora_a", |
| "model.decoder.layers.21.self_attn.o_proj.lora_b", |
| "model.decoder.layers.21.self_attn.q_norm.weight", |
| "model.decoder.layers.21.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.21.self_attn.q_proj.lora_a", |
| "model.decoder.layers.21.self_attn.q_proj.lora_b", |
| "model.decoder.layers.21.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.21.self_attn.v_proj.lora_a", |
| "model.decoder.layers.21.self_attn.v_proj.lora_b", |
| "model.decoder.layers.22.experts.down_proj.linear.weight", |
| "model.decoder.layers.22.experts.down_proj.lora_a", |
| "model.decoder.layers.22.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00018.safetensors", |
| "bytes": 1645281342, |
| "sha256": "b49d0e174f0d8422d737d7023c283a555f359c994a07e593aae5d07eb291c0ae", |
| "tensors": [ |
| "model.decoder.layers.22.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.22.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.22.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.22.input_layernorm.weight", |
| "model.decoder.layers.22.layer_scalar", |
| "model.decoder.layers.22.mlp.down_proj.linear.weight", |
| "model.decoder.layers.22.mlp.down_proj.lora_a", |
| "model.decoder.layers.22.mlp.down_proj.lora_b", |
| "model.decoder.layers.22.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.22.mlp.gate_proj.lora_a", |
| "model.decoder.layers.22.mlp.gate_proj.lora_b", |
| "model.decoder.layers.22.mlp.up_proj.linear.weight", |
| "model.decoder.layers.22.mlp.up_proj.lora_a", |
| "model.decoder.layers.22.mlp.up_proj.lora_b", |
| "model.decoder.layers.22.post_attention_layernorm.weight", |
| "model.decoder.layers.22.post_feedforward_layernorm.weight", |
| "model.decoder.layers.22.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.22.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.22.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.22.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.22.router.per_expert_scale", |
| "model.decoder.layers.22.router.proj.weight", |
| "model.decoder.layers.22.router.scale", |
| "model.decoder.layers.22.self_attn.k_norm.weight", |
| "model.decoder.layers.22.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.22.self_attn.k_proj.lora_a", |
| "model.decoder.layers.22.self_attn.k_proj.lora_b", |
| "model.decoder.layers.22.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.22.self_attn.o_proj.lora_a", |
| "model.decoder.layers.22.self_attn.o_proj.lora_b", |
| "model.decoder.layers.22.self_attn.q_norm.weight", |
| "model.decoder.layers.22.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.22.self_attn.q_proj.lora_a", |
| "model.decoder.layers.22.self_attn.q_proj.lora_b", |
| "model.decoder.layers.22.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.22.self_attn.v_proj.lora_a", |
| "model.decoder.layers.22.self_attn.v_proj.lora_b", |
| "model.decoder.layers.23.experts.down_proj.linear.weight", |
| "model.decoder.layers.23.experts.down_proj.lora_a", |
| "model.decoder.layers.23.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00019.safetensors", |
| "bytes": 1674191627, |
| "sha256": "e8e66177d4ad899a85ca712eb15fce721a7db756121ce3123d13b16eb3c0b0a8", |
| "tensors": [ |
| "model.decoder.layers.23.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.23.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.23.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.23.input_layernorm.weight", |
| "model.decoder.layers.23.layer_scalar", |
| "model.decoder.layers.23.mlp.down_proj.linear.weight", |
| "model.decoder.layers.23.mlp.down_proj.lora_a", |
| "model.decoder.layers.23.mlp.down_proj.lora_b", |
| "model.decoder.layers.23.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.23.mlp.gate_proj.lora_a", |
| "model.decoder.layers.23.mlp.gate_proj.lora_b", |
| "model.decoder.layers.23.mlp.up_proj.linear.weight", |
| "model.decoder.layers.23.mlp.up_proj.lora_a", |
| "model.decoder.layers.23.mlp.up_proj.lora_b", |
| "model.decoder.layers.23.post_attention_layernorm.weight", |
| "model.decoder.layers.23.post_feedforward_layernorm.weight", |
| "model.decoder.layers.23.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.23.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.23.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.23.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.23.router.per_expert_scale", |
| "model.decoder.layers.23.router.proj.weight", |
| "model.decoder.layers.23.router.scale", |
| "model.decoder.layers.23.self_attn.k_norm.weight", |
| "model.decoder.layers.23.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.23.self_attn.k_proj.lora_a", |
| "model.decoder.layers.23.self_attn.k_proj.lora_b", |
| "model.decoder.layers.23.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.23.self_attn.o_proj.lora_a", |
| "model.decoder.layers.23.self_attn.o_proj.lora_b", |
| "model.decoder.layers.23.self_attn.q_norm.weight", |
| "model.decoder.layers.23.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.23.self_attn.q_proj.lora_a", |
| "model.decoder.layers.23.self_attn.q_proj.lora_b", |
| "model.decoder.layers.24.experts.down_proj.linear.weight", |
| "model.decoder.layers.24.experts.down_proj.lora_a", |
| "model.decoder.layers.24.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00020.safetensors", |
| "bytes": 1645281442, |
| "sha256": "d87e3d52c22cbec2e5681d7c048ddbe5109df7f8c7af99c713c00dfe351b2ae6", |
| "tensors": [ |
| "model.decoder.layers.24.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.24.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.24.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.24.input_layernorm.weight", |
| "model.decoder.layers.24.layer_scalar", |
| "model.decoder.layers.24.mlp.down_proj.linear.weight", |
| "model.decoder.layers.24.mlp.down_proj.lora_a", |
| "model.decoder.layers.24.mlp.down_proj.lora_b", |
| "model.decoder.layers.24.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.24.mlp.gate_proj.lora_a", |
| "model.decoder.layers.24.mlp.gate_proj.lora_b", |
| "model.decoder.layers.24.mlp.up_proj.linear.weight", |
| "model.decoder.layers.24.mlp.up_proj.lora_a", |
| "model.decoder.layers.24.mlp.up_proj.lora_b", |
| "model.decoder.layers.24.post_attention_layernorm.weight", |
| "model.decoder.layers.24.post_feedforward_layernorm.weight", |
| "model.decoder.layers.24.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.24.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.24.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.24.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.24.router.per_expert_scale", |
| "model.decoder.layers.24.router.proj.weight", |
| "model.decoder.layers.24.router.scale", |
| "model.decoder.layers.24.self_attn.k_norm.weight", |
| "model.decoder.layers.24.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.24.self_attn.k_proj.lora_a", |
| "model.decoder.layers.24.self_attn.k_proj.lora_b", |
| "model.decoder.layers.24.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.24.self_attn.o_proj.lora_a", |
| "model.decoder.layers.24.self_attn.o_proj.lora_b", |
| "model.decoder.layers.24.self_attn.q_norm.weight", |
| "model.decoder.layers.24.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.24.self_attn.q_proj.lora_a", |
| "model.decoder.layers.24.self_attn.q_proj.lora_b", |
| "model.decoder.layers.24.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.24.self_attn.v_proj.lora_a", |
| "model.decoder.layers.24.self_attn.v_proj.lora_b", |
| "model.decoder.layers.25.experts.down_proj.linear.weight", |
| "model.decoder.layers.25.experts.down_proj.lora_a", |
| "model.decoder.layers.25.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00021.safetensors", |
| "bytes": 1645281412, |
| "sha256": "44aecb5538ee528dcc90f49733268c67795a44a71b0fece8eb1b51b84cca59ea", |
| "tensors": [ |
| "model.decoder.layers.25.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.25.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.25.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.25.input_layernorm.weight", |
| "model.decoder.layers.25.layer_scalar", |
| "model.decoder.layers.25.mlp.down_proj.linear.weight", |
| "model.decoder.layers.25.mlp.down_proj.lora_a", |
| "model.decoder.layers.25.mlp.down_proj.lora_b", |
| "model.decoder.layers.25.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.25.mlp.gate_proj.lora_a", |
| "model.decoder.layers.25.mlp.gate_proj.lora_b", |
| "model.decoder.layers.25.mlp.up_proj.linear.weight", |
| "model.decoder.layers.25.mlp.up_proj.lora_a", |
| "model.decoder.layers.25.mlp.up_proj.lora_b", |
| "model.decoder.layers.25.post_attention_layernorm.weight", |
| "model.decoder.layers.25.post_feedforward_layernorm.weight", |
| "model.decoder.layers.25.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.25.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.25.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.25.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.25.router.per_expert_scale", |
| "model.decoder.layers.25.router.proj.weight", |
| "model.decoder.layers.25.router.scale", |
| "model.decoder.layers.25.self_attn.k_norm.weight", |
| "model.decoder.layers.25.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.25.self_attn.k_proj.lora_a", |
| "model.decoder.layers.25.self_attn.k_proj.lora_b", |
| "model.decoder.layers.25.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.25.self_attn.o_proj.lora_a", |
| "model.decoder.layers.25.self_attn.o_proj.lora_b", |
| "model.decoder.layers.25.self_attn.q_norm.weight", |
| "model.decoder.layers.25.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.25.self_attn.q_proj.lora_a", |
| "model.decoder.layers.25.self_attn.q_proj.lora_b", |
| "model.decoder.layers.25.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.25.self_attn.v_proj.lora_a", |
| "model.decoder.layers.25.self_attn.v_proj.lora_b", |
| "model.decoder.layers.26.experts.down_proj.linear.weight", |
| "model.decoder.layers.26.experts.down_proj.lora_a", |
| "model.decoder.layers.26.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00022.safetensors", |
| "bytes": 1645281408, |
| "sha256": "70f3124fee705d9cb5221936c59f39820ab9c37a2c70bad89176e309e1c1ca07", |
| "tensors": [ |
| "model.decoder.layers.26.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.26.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.26.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.26.input_layernorm.weight", |
| "model.decoder.layers.26.layer_scalar", |
| "model.decoder.layers.26.mlp.down_proj.linear.weight", |
| "model.decoder.layers.26.mlp.down_proj.lora_a", |
| "model.decoder.layers.26.mlp.down_proj.lora_b", |
| "model.decoder.layers.26.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.26.mlp.gate_proj.lora_a", |
| "model.decoder.layers.26.mlp.gate_proj.lora_b", |
| "model.decoder.layers.26.mlp.up_proj.linear.weight", |
| "model.decoder.layers.26.mlp.up_proj.lora_a", |
| "model.decoder.layers.26.mlp.up_proj.lora_b", |
| "model.decoder.layers.26.post_attention_layernorm.weight", |
| "model.decoder.layers.26.post_feedforward_layernorm.weight", |
| "model.decoder.layers.26.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.26.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.26.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.26.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.26.router.per_expert_scale", |
| "model.decoder.layers.26.router.proj.weight", |
| "model.decoder.layers.26.router.scale", |
| "model.decoder.layers.26.self_attn.k_norm.weight", |
| "model.decoder.layers.26.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.26.self_attn.k_proj.lora_a", |
| "model.decoder.layers.26.self_attn.k_proj.lora_b", |
| "model.decoder.layers.26.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.26.self_attn.o_proj.lora_a", |
| "model.decoder.layers.26.self_attn.o_proj.lora_b", |
| "model.decoder.layers.26.self_attn.q_norm.weight", |
| "model.decoder.layers.26.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.26.self_attn.q_proj.lora_a", |
| "model.decoder.layers.26.self_attn.q_proj.lora_b", |
| "model.decoder.layers.26.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.26.self_attn.v_proj.lora_a", |
| "model.decoder.layers.26.self_attn.v_proj.lora_b", |
| "model.decoder.layers.27.experts.down_proj.linear.weight", |
| "model.decoder.layers.27.experts.down_proj.lora_a", |
| "model.decoder.layers.27.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00023.safetensors", |
| "bytes": 1645281368, |
| "sha256": "499f422a0073dce797799000ab7ea6df9e77f845aad9cff999ac9d404a47b8a0", |
| "tensors": [ |
| "model.decoder.layers.27.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.27.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.27.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.27.input_layernorm.weight", |
| "model.decoder.layers.27.layer_scalar", |
| "model.decoder.layers.27.mlp.down_proj.linear.weight", |
| "model.decoder.layers.27.mlp.down_proj.lora_a", |
| "model.decoder.layers.27.mlp.down_proj.lora_b", |
| "model.decoder.layers.27.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.27.mlp.gate_proj.lora_a", |
| "model.decoder.layers.27.mlp.gate_proj.lora_b", |
| "model.decoder.layers.27.mlp.up_proj.linear.weight", |
| "model.decoder.layers.27.mlp.up_proj.lora_a", |
| "model.decoder.layers.27.mlp.up_proj.lora_b", |
| "model.decoder.layers.27.post_attention_layernorm.weight", |
| "model.decoder.layers.27.post_feedforward_layernorm.weight", |
| "model.decoder.layers.27.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.27.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.27.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.27.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.27.router.per_expert_scale", |
| "model.decoder.layers.27.router.proj.weight", |
| "model.decoder.layers.27.router.scale", |
| "model.decoder.layers.27.self_attn.k_norm.weight", |
| "model.decoder.layers.27.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.27.self_attn.k_proj.lora_a", |
| "model.decoder.layers.27.self_attn.k_proj.lora_b", |
| "model.decoder.layers.27.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.27.self_attn.o_proj.lora_a", |
| "model.decoder.layers.27.self_attn.o_proj.lora_b", |
| "model.decoder.layers.27.self_attn.q_norm.weight", |
| "model.decoder.layers.27.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.27.self_attn.q_proj.lora_a", |
| "model.decoder.layers.27.self_attn.q_proj.lora_b", |
| "model.decoder.layers.27.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.27.self_attn.v_proj.lora_a", |
| "model.decoder.layers.27.self_attn.v_proj.lora_b", |
| "model.decoder.layers.28.experts.down_proj.linear.weight", |
| "model.decoder.layers.28.experts.down_proj.lora_a", |
| "model.decoder.layers.28.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00024.safetensors", |
| "bytes": 1645281350, |
| "sha256": "e7cd9ffad7914610f2cf54be55ac46a5d2082d5faee7f3d7d2854fa46315d46e", |
| "tensors": [ |
| "model.decoder.layers.28.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.28.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.28.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.28.input_layernorm.weight", |
| "model.decoder.layers.28.layer_scalar", |
| "model.decoder.layers.28.mlp.down_proj.linear.weight", |
| "model.decoder.layers.28.mlp.down_proj.lora_a", |
| "model.decoder.layers.28.mlp.down_proj.lora_b", |
| "model.decoder.layers.28.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.28.mlp.gate_proj.lora_a", |
| "model.decoder.layers.28.mlp.gate_proj.lora_b", |
| "model.decoder.layers.28.mlp.up_proj.linear.weight", |
| "model.decoder.layers.28.mlp.up_proj.lora_a", |
| "model.decoder.layers.28.mlp.up_proj.lora_b", |
| "model.decoder.layers.28.post_attention_layernorm.weight", |
| "model.decoder.layers.28.post_feedforward_layernorm.weight", |
| "model.decoder.layers.28.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.28.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.28.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.28.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.28.router.per_expert_scale", |
| "model.decoder.layers.28.router.proj.weight", |
| "model.decoder.layers.28.router.scale", |
| "model.decoder.layers.28.self_attn.k_norm.weight", |
| "model.decoder.layers.28.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.28.self_attn.k_proj.lora_a", |
| "model.decoder.layers.28.self_attn.k_proj.lora_b", |
| "model.decoder.layers.28.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.28.self_attn.o_proj.lora_a", |
| "model.decoder.layers.28.self_attn.o_proj.lora_b", |
| "model.decoder.layers.28.self_attn.q_norm.weight", |
| "model.decoder.layers.28.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.28.self_attn.q_proj.lora_a", |
| "model.decoder.layers.28.self_attn.q_proj.lora_b", |
| "model.decoder.layers.28.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.28.self_attn.v_proj.lora_a", |
| "model.decoder.layers.28.self_attn.v_proj.lora_b", |
| "model.decoder.layers.29.experts.down_proj.linear.weight", |
| "model.decoder.layers.29.experts.down_proj.lora_a", |
| "model.decoder.layers.29.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00025.safetensors", |
| "bytes": 1674191592, |
| "sha256": "234e835f2a3abdcd98cdbba245f057948573dd480301c94c00921f6d5c91bd96", |
| "tensors": [ |
| "model.decoder.layers.29.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.29.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.29.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.29.input_layernorm.weight", |
| "model.decoder.layers.29.layer_scalar", |
| "model.decoder.layers.29.mlp.down_proj.linear.weight", |
| "model.decoder.layers.29.mlp.down_proj.lora_a", |
| "model.decoder.layers.29.mlp.down_proj.lora_b", |
| "model.decoder.layers.29.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.29.mlp.gate_proj.lora_a", |
| "model.decoder.layers.29.mlp.gate_proj.lora_b", |
| "model.decoder.layers.29.mlp.up_proj.linear.weight", |
| "model.decoder.layers.29.mlp.up_proj.lora_a", |
| "model.decoder.layers.29.mlp.up_proj.lora_b", |
| "model.decoder.layers.29.post_attention_layernorm.weight", |
| "model.decoder.layers.29.post_feedforward_layernorm.weight", |
| "model.decoder.layers.29.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.29.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.29.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.29.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.29.router.per_expert_scale", |
| "model.decoder.layers.29.router.proj.weight", |
| "model.decoder.layers.29.router.scale", |
| "model.decoder.layers.29.self_attn.k_norm.weight", |
| "model.decoder.layers.29.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.29.self_attn.k_proj.lora_a", |
| "model.decoder.layers.29.self_attn.k_proj.lora_b", |
| "model.decoder.layers.29.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.29.self_attn.o_proj.lora_a", |
| "model.decoder.layers.29.self_attn.o_proj.lora_b", |
| "model.decoder.layers.29.self_attn.q_norm.weight", |
| "model.decoder.layers.29.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.29.self_attn.q_proj.lora_a", |
| "model.decoder.layers.29.self_attn.q_proj.lora_b", |
| "model.decoder.layers.3.experts.down_proj.linear.weight", |
| "model.decoder.layers.3.experts.down_proj.lora_a", |
| "model.decoder.layers.3.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00026.safetensors", |
| "bytes": 1645281350, |
| "sha256": "57581d8a8c3efc5f19ac487469272cbf8d9fa68cb978c5d8f605701f3dca7fa2", |
| "tensors": [ |
| "model.decoder.layers.3.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.3.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.3.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.3.input_layernorm.weight", |
| "model.decoder.layers.3.layer_scalar", |
| "model.decoder.layers.3.mlp.down_proj.linear.weight", |
| "model.decoder.layers.3.mlp.down_proj.lora_a", |
| "model.decoder.layers.3.mlp.down_proj.lora_b", |
| "model.decoder.layers.3.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.3.mlp.gate_proj.lora_a", |
| "model.decoder.layers.3.mlp.gate_proj.lora_b", |
| "model.decoder.layers.3.mlp.up_proj.linear.weight", |
| "model.decoder.layers.3.mlp.up_proj.lora_a", |
| "model.decoder.layers.3.mlp.up_proj.lora_b", |
| "model.decoder.layers.3.post_attention_layernorm.weight", |
| "model.decoder.layers.3.post_feedforward_layernorm.weight", |
| "model.decoder.layers.3.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.3.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.3.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.3.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.3.router.per_expert_scale", |
| "model.decoder.layers.3.router.proj.weight", |
| "model.decoder.layers.3.router.scale", |
| "model.decoder.layers.3.self_attn.k_norm.weight", |
| "model.decoder.layers.3.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.3.self_attn.k_proj.lora_a", |
| "model.decoder.layers.3.self_attn.k_proj.lora_b", |
| "model.decoder.layers.3.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.3.self_attn.o_proj.lora_a", |
| "model.decoder.layers.3.self_attn.o_proj.lora_b", |
| "model.decoder.layers.3.self_attn.q_norm.weight", |
| "model.decoder.layers.3.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.3.self_attn.q_proj.lora_a", |
| "model.decoder.layers.3.self_attn.q_proj.lora_b", |
| "model.decoder.layers.3.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.3.self_attn.v_proj.lora_a", |
| "model.decoder.layers.3.self_attn.v_proj.lora_b", |
| "model.decoder.layers.4.experts.down_proj.linear.weight", |
| "model.decoder.layers.4.experts.down_proj.lora_a", |
| "model.decoder.layers.4.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00027.safetensors", |
| "bytes": 1645281304, |
| "sha256": "a2d814a172600dd0169ceac390fde8d8683926d72be9bc025c368c9c9bf5e5e9", |
| "tensors": [ |
| "model.decoder.layers.4.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.4.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.4.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.4.input_layernorm.weight", |
| "model.decoder.layers.4.layer_scalar", |
| "model.decoder.layers.4.mlp.down_proj.linear.weight", |
| "model.decoder.layers.4.mlp.down_proj.lora_a", |
| "model.decoder.layers.4.mlp.down_proj.lora_b", |
| "model.decoder.layers.4.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.4.mlp.gate_proj.lora_a", |
| "model.decoder.layers.4.mlp.gate_proj.lora_b", |
| "model.decoder.layers.4.mlp.up_proj.linear.weight", |
| "model.decoder.layers.4.mlp.up_proj.lora_a", |
| "model.decoder.layers.4.mlp.up_proj.lora_b", |
| "model.decoder.layers.4.post_attention_layernorm.weight", |
| "model.decoder.layers.4.post_feedforward_layernorm.weight", |
| "model.decoder.layers.4.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.4.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.4.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.4.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.4.router.per_expert_scale", |
| "model.decoder.layers.4.router.proj.weight", |
| "model.decoder.layers.4.router.scale", |
| "model.decoder.layers.4.self_attn.k_norm.weight", |
| "model.decoder.layers.4.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.4.self_attn.k_proj.lora_a", |
| "model.decoder.layers.4.self_attn.k_proj.lora_b", |
| "model.decoder.layers.4.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.4.self_attn.o_proj.lora_a", |
| "model.decoder.layers.4.self_attn.o_proj.lora_b", |
| "model.decoder.layers.4.self_attn.q_norm.weight", |
| "model.decoder.layers.4.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.4.self_attn.q_proj.lora_a", |
| "model.decoder.layers.4.self_attn.q_proj.lora_b", |
| "model.decoder.layers.4.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.4.self_attn.v_proj.lora_a", |
| "model.decoder.layers.4.self_attn.v_proj.lora_b", |
| "model.decoder.layers.5.experts.down_proj.linear.weight", |
| "model.decoder.layers.5.experts.down_proj.lora_a", |
| "model.decoder.layers.5.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00028.safetensors", |
| "bytes": 1674191574, |
| "sha256": "a7ee3b8626e6b76a82e0fbd78508a66151504ad02e5983f2895c735f3c41e704", |
| "tensors": [ |
| "model.decoder.layers.5.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.5.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.5.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.5.input_layernorm.weight", |
| "model.decoder.layers.5.layer_scalar", |
| "model.decoder.layers.5.mlp.down_proj.linear.weight", |
| "model.decoder.layers.5.mlp.down_proj.lora_a", |
| "model.decoder.layers.5.mlp.down_proj.lora_b", |
| "model.decoder.layers.5.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.5.mlp.gate_proj.lora_a", |
| "model.decoder.layers.5.mlp.gate_proj.lora_b", |
| "model.decoder.layers.5.mlp.up_proj.linear.weight", |
| "model.decoder.layers.5.mlp.up_proj.lora_a", |
| "model.decoder.layers.5.mlp.up_proj.lora_b", |
| "model.decoder.layers.5.post_attention_layernorm.weight", |
| "model.decoder.layers.5.post_feedforward_layernorm.weight", |
| "model.decoder.layers.5.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.5.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.5.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.5.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.5.router.per_expert_scale", |
| "model.decoder.layers.5.router.proj.weight", |
| "model.decoder.layers.5.router.scale", |
| "model.decoder.layers.5.self_attn.k_norm.weight", |
| "model.decoder.layers.5.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.5.self_attn.k_proj.lora_a", |
| "model.decoder.layers.5.self_attn.k_proj.lora_b", |
| "model.decoder.layers.5.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.5.self_attn.o_proj.lora_a", |
| "model.decoder.layers.5.self_attn.o_proj.lora_b", |
| "model.decoder.layers.5.self_attn.q_norm.weight", |
| "model.decoder.layers.5.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.5.self_attn.q_proj.lora_a", |
| "model.decoder.layers.5.self_attn.q_proj.lora_b", |
| "model.decoder.layers.6.experts.down_proj.linear.weight", |
| "model.decoder.layers.6.experts.down_proj.lora_a", |
| "model.decoder.layers.6.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00029.safetensors", |
| "bytes": 1645281378, |
| "sha256": "51e757bfbf302e147a926faf4ad7ab2baa97355b5e8ffd688b06a62f74b95f6b", |
| "tensors": [ |
| "model.decoder.layers.6.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.6.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.6.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.6.input_layernorm.weight", |
| "model.decoder.layers.6.layer_scalar", |
| "model.decoder.layers.6.mlp.down_proj.linear.weight", |
| "model.decoder.layers.6.mlp.down_proj.lora_a", |
| "model.decoder.layers.6.mlp.down_proj.lora_b", |
| "model.decoder.layers.6.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.6.mlp.gate_proj.lora_a", |
| "model.decoder.layers.6.mlp.gate_proj.lora_b", |
| "model.decoder.layers.6.mlp.up_proj.linear.weight", |
| "model.decoder.layers.6.mlp.up_proj.lora_a", |
| "model.decoder.layers.6.mlp.up_proj.lora_b", |
| "model.decoder.layers.6.post_attention_layernorm.weight", |
| "model.decoder.layers.6.post_feedforward_layernorm.weight", |
| "model.decoder.layers.6.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.6.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.6.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.6.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.6.router.per_expert_scale", |
| "model.decoder.layers.6.router.proj.weight", |
| "model.decoder.layers.6.router.scale", |
| "model.decoder.layers.6.self_attn.k_norm.weight", |
| "model.decoder.layers.6.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.6.self_attn.k_proj.lora_a", |
| "model.decoder.layers.6.self_attn.k_proj.lora_b", |
| "model.decoder.layers.6.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.6.self_attn.o_proj.lora_a", |
| "model.decoder.layers.6.self_attn.o_proj.lora_b", |
| "model.decoder.layers.6.self_attn.q_norm.weight", |
| "model.decoder.layers.6.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.6.self_attn.q_proj.lora_a", |
| "model.decoder.layers.6.self_attn.q_proj.lora_b", |
| "model.decoder.layers.6.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.6.self_attn.v_proj.lora_a", |
| "model.decoder.layers.6.self_attn.v_proj.lora_b", |
| "model.decoder.layers.7.experts.down_proj.linear.weight", |
| "model.decoder.layers.7.experts.down_proj.lora_a", |
| "model.decoder.layers.7.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00030.safetensors", |
| "bytes": 1645281282, |
| "sha256": "2dcc2427d1c1bd6d143283f4d2cede73940db3c093b59b45c1c1b879b5406631", |
| "tensors": [ |
| "model.decoder.layers.7.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.7.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.7.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.7.input_layernorm.weight", |
| "model.decoder.layers.7.layer_scalar", |
| "model.decoder.layers.7.mlp.down_proj.linear.weight", |
| "model.decoder.layers.7.mlp.down_proj.lora_a", |
| "model.decoder.layers.7.mlp.down_proj.lora_b", |
| "model.decoder.layers.7.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.7.mlp.gate_proj.lora_a", |
| "model.decoder.layers.7.mlp.gate_proj.lora_b", |
| "model.decoder.layers.7.mlp.up_proj.linear.weight", |
| "model.decoder.layers.7.mlp.up_proj.lora_a", |
| "model.decoder.layers.7.mlp.up_proj.lora_b", |
| "model.decoder.layers.7.post_attention_layernorm.weight", |
| "model.decoder.layers.7.post_feedforward_layernorm.weight", |
| "model.decoder.layers.7.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.7.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.7.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.7.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.7.router.per_expert_scale", |
| "model.decoder.layers.7.router.proj.weight", |
| "model.decoder.layers.7.router.scale", |
| "model.decoder.layers.7.self_attn.k_norm.weight", |
| "model.decoder.layers.7.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.7.self_attn.k_proj.lora_a", |
| "model.decoder.layers.7.self_attn.k_proj.lora_b", |
| "model.decoder.layers.7.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.7.self_attn.o_proj.lora_a", |
| "model.decoder.layers.7.self_attn.o_proj.lora_b", |
| "model.decoder.layers.7.self_attn.q_norm.weight", |
| "model.decoder.layers.7.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.7.self_attn.q_proj.lora_a", |
| "model.decoder.layers.7.self_attn.q_proj.lora_b", |
| "model.decoder.layers.7.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.7.self_attn.v_proj.lora_a", |
| "model.decoder.layers.7.self_attn.v_proj.lora_b", |
| "model.decoder.layers.8.experts.down_proj.linear.weight", |
| "model.decoder.layers.8.experts.down_proj.lora_a", |
| "model.decoder.layers.8.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00031.safetensors", |
| "bytes": 1645281334, |
| "sha256": "df89b8419ce01b7d78f761aac8e324df22004cfeba4c9ae81c627dde94a73535", |
| "tensors": [ |
| "model.decoder.layers.8.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.8.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.8.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.8.input_layernorm.weight", |
| "model.decoder.layers.8.layer_scalar", |
| "model.decoder.layers.8.mlp.down_proj.linear.weight", |
| "model.decoder.layers.8.mlp.down_proj.lora_a", |
| "model.decoder.layers.8.mlp.down_proj.lora_b", |
| "model.decoder.layers.8.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.8.mlp.gate_proj.lora_a", |
| "model.decoder.layers.8.mlp.gate_proj.lora_b", |
| "model.decoder.layers.8.mlp.up_proj.linear.weight", |
| "model.decoder.layers.8.mlp.up_proj.lora_a", |
| "model.decoder.layers.8.mlp.up_proj.lora_b", |
| "model.decoder.layers.8.post_attention_layernorm.weight", |
| "model.decoder.layers.8.post_feedforward_layernorm.weight", |
| "model.decoder.layers.8.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.8.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.8.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.8.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.8.router.per_expert_scale", |
| "model.decoder.layers.8.router.proj.weight", |
| "model.decoder.layers.8.router.scale", |
| "model.decoder.layers.8.self_attn.k_norm.weight", |
| "model.decoder.layers.8.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.8.self_attn.k_proj.lora_a", |
| "model.decoder.layers.8.self_attn.k_proj.lora_b", |
| "model.decoder.layers.8.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.8.self_attn.o_proj.lora_a", |
| "model.decoder.layers.8.self_attn.o_proj.lora_b", |
| "model.decoder.layers.8.self_attn.q_norm.weight", |
| "model.decoder.layers.8.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.8.self_attn.q_proj.lora_a", |
| "model.decoder.layers.8.self_attn.q_proj.lora_b", |
| "model.decoder.layers.8.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.8.self_attn.v_proj.lora_a", |
| "model.decoder.layers.8.self_attn.v_proj.lora_b", |
| "model.decoder.layers.9.experts.down_proj.linear.weight", |
| "model.decoder.layers.9.experts.down_proj.lora_a", |
| "model.decoder.layers.9.experts.down_proj.lora_b" |
| ] |
| }, |
| { |
| "file": "model-00032.safetensors", |
| "bytes": 1166260864, |
| "sha256": "c62b07e428849f2db92bf286a58d2657c685deb2128692ef0de49d5df071f226", |
| "tensors": [ |
| "model.decoder.layers.9.experts.gate_up_proj.linear.weight", |
| "model.decoder.layers.9.experts.gate_up_proj.lora_a", |
| "model.decoder.layers.9.experts.gate_up_proj.lora_b", |
| "model.decoder.layers.9.input_layernorm.weight", |
| "model.decoder.layers.9.layer_scalar", |
| "model.decoder.layers.9.mlp.down_proj.linear.weight", |
| "model.decoder.layers.9.mlp.down_proj.lora_a", |
| "model.decoder.layers.9.mlp.down_proj.lora_b", |
| "model.decoder.layers.9.mlp.gate_proj.linear.weight", |
| "model.decoder.layers.9.mlp.gate_proj.lora_a", |
| "model.decoder.layers.9.mlp.gate_proj.lora_b", |
| "model.decoder.layers.9.mlp.up_proj.linear.weight", |
| "model.decoder.layers.9.mlp.up_proj.lora_a", |
| "model.decoder.layers.9.mlp.up_proj.lora_b", |
| "model.decoder.layers.9.post_attention_layernorm.weight", |
| "model.decoder.layers.9.post_feedforward_layernorm.weight", |
| "model.decoder.layers.9.post_feedforward_layernorm_1.weight", |
| "model.decoder.layers.9.post_feedforward_layernorm_2.weight", |
| "model.decoder.layers.9.pre_feedforward_layernorm.weight", |
| "model.decoder.layers.9.pre_feedforward_layernorm_2.weight", |
| "model.decoder.layers.9.router.per_expert_scale", |
| "model.decoder.layers.9.router.proj.weight", |
| "model.decoder.layers.9.router.scale", |
| "model.decoder.layers.9.self_attn.k_norm.weight", |
| "model.decoder.layers.9.self_attn.k_proj.linear.weight", |
| "model.decoder.layers.9.self_attn.k_proj.lora_a", |
| "model.decoder.layers.9.self_attn.k_proj.lora_b", |
| "model.decoder.layers.9.self_attn.o_proj.linear.weight", |
| "model.decoder.layers.9.self_attn.o_proj.lora_a", |
| "model.decoder.layers.9.self_attn.o_proj.lora_b", |
| "model.decoder.layers.9.self_attn.q_norm.weight", |
| "model.decoder.layers.9.self_attn.q_proj.linear.weight", |
| "model.decoder.layers.9.self_attn.q_proj.lora_a", |
| "model.decoder.layers.9.self_attn.q_proj.lora_b", |
| "model.decoder.layers.9.self_attn.v_proj.linear.weight", |
| "model.decoder.layers.9.self_attn.v_proj.lora_a", |
| "model.decoder.layers.9.self_attn.v_proj.lora_b", |
| "model.decoder.norm.weight", |
| "model.decoder.self_conditioning.down_proj.weight", |
| "model.decoder.self_conditioning.gate_proj.weight", |
| "model.decoder.self_conditioning.pre_norm.weight", |
| "model.decoder.self_conditioning.up_proj.weight", |
| "model.encoder.language_model.layers.0.layer_scalar", |
| "model.encoder.language_model.layers.1.layer_scalar", |
| "model.encoder.language_model.layers.10.layer_scalar", |
| "model.encoder.language_model.layers.11.layer_scalar", |
| "model.encoder.language_model.layers.12.layer_scalar", |
| "model.encoder.language_model.layers.13.layer_scalar", |
| "model.encoder.language_model.layers.14.layer_scalar", |
| "model.encoder.language_model.layers.15.layer_scalar", |
| "model.encoder.language_model.layers.16.layer_scalar", |
| "model.encoder.language_model.layers.17.layer_scalar", |
| "model.encoder.language_model.layers.18.layer_scalar", |
| "model.encoder.language_model.layers.19.layer_scalar", |
| "model.encoder.language_model.layers.2.layer_scalar", |
| "model.encoder.language_model.layers.20.layer_scalar", |
| "model.encoder.language_model.layers.21.layer_scalar", |
| "model.encoder.language_model.layers.22.layer_scalar", |
| "model.encoder.language_model.layers.23.layer_scalar", |
| "model.encoder.language_model.layers.24.layer_scalar", |
| "model.encoder.language_model.layers.25.layer_scalar", |
| "model.encoder.language_model.layers.26.layer_scalar", |
| "model.encoder.language_model.layers.27.layer_scalar", |
| "model.encoder.language_model.layers.28.layer_scalar", |
| "model.encoder.language_model.layers.29.layer_scalar", |
| "model.encoder.language_model.layers.3.layer_scalar", |
| "model.encoder.language_model.layers.4.layer_scalar", |
| "model.encoder.language_model.layers.5.layer_scalar", |
| "model.encoder.language_model.layers.6.layer_scalar", |
| "model.encoder.language_model.layers.7.layer_scalar", |
| "model.encoder.language_model.layers.8.layer_scalar", |
| "model.encoder.language_model.layers.9.layer_scalar" |
| ] |
| } |
| ], |
| "dependencies": { |
| "mlx": "0.32.2", |
| "mlx-lm": "0.31.3", |
| "mlx-vlm": "0.7.2", |
| "transformers": "5.14.1", |
| "huggingface-hub": "1.20.1", |
| "safetensors": "0.8.0", |
| "numpy": "2.5.0" |
| } |
| } |
|
|
