{
  "format": "modilify_mk2_native_mlx_inference_v1",
  "created_at": "2026-10-06T07:00:46.116333+00:00",
  "state_schema_version": 25,
  "training_scheme": "gold_prefix_shared_commit_0_256_committed_ce_calibration_causal_throughput_terminal_sft",
  "memory_scheme": "dual_timescale_gdn2_trajectory_memory",
  "memory_architecture": "compact_gdn2_v2",
  "global_step": 1250,
  "lora_config": {
    "r": 16,
    "alpha": 16,
    "dropout": 0.05,
    "target_modules": [
      "q_proj",
      "k_proj",
      "v_proj",
      "o_proj",
      "gate_proj",
      "up_proj",
      "down_proj",
      "proj"
    ],
    "expert_r": 8,
    "expert_alpha": 8
  },
  "precision_policy": "gdn2_small_fp32_v1",
  "release_version": "1.0.0",
  "model_id": "Modilify/Modilify-Mk2-preview-mlx",
  "base_model": "google/diffusiongemma-26B-A4B-it",
  "source_manifest_sha256": "450600bb96c5d3d2c85dff82e7de2bede33a20df142f57c1f765b71322bbb827",
  "trainable_names": [
    "latent_deliberation.blocks.0.attn.k_norm.weight",
    "latent_deliberation.blocks.0.attn.k_proj.weight",
    "latent_deliberation.blocks.0.attn.o_proj.weight",
    "latent_deliberation.blocks.0.attn.q_norm.weight",
    "latent_deliberation.blocks.0.attn.q_proj.weight",
    "latent_deliberation.blocks.0.attn.v_proj.weight",
    "latent_deliberation.blocks.0.ff.down.weight",
    "latent_deliberation.blocks.0.ff.gate.weight",
    "latent_deliberation.blocks.0.ff.up.weight",
    "latent_deliberation.blocks.0.ff_norm.weight",
    "latent_deliberation.blocks.0.norm.weight",
    "latent_deliberation.blocks.1.attn.k_norm.weight",
    "latent_deliberation.blocks.1.attn.k_proj.weight",
    "latent_deliberation.blocks.1.attn.o_proj.weight",
    "latent_deliberation.blocks.1.attn.q_norm.weight",
    "latent_deliberation.blocks.1.attn.q_proj.weight",
    "latent_deliberation.blocks.1.attn.v_proj.weight",
    "latent_deliberation.blocks.1.ff.down.weight",
    "latent_deliberation.blocks.1.ff.gate.weight",
    "latent_deliberation.blocks.1.ff.up.weight",
    "latent_deliberation.blocks.1.ff_norm.weight",
    "latent_deliberation.blocks.1.norm.weight",
    "latent_deliberation.blocks.2.attn.k_norm.weight",
    "latent_deliberation.blocks.2.attn.k_proj.weight",
    "latent_deliberation.blocks.2.attn.o_proj.weight",
    "latent_deliberation.blocks.2.attn.q_norm.weight",
    "latent_deliberation.blocks.2.attn.q_proj.weight",
    "latent_deliberation.blocks.2.attn.v_proj.weight",
    "latent_deliberation.blocks.2.ff.down.weight",
    "latent_deliberation.blocks.2.ff.gate.weight",
    "latent_deliberation.blocks.2.ff.up.weight",
    "latent_deliberation.blocks.2.ff_norm.weight",
    "latent_deliberation.blocks.2.norm.weight",
    "latent_deliberation.blocks.3.attn.k_norm.weight",
    "latent_deliberation.blocks.3.attn.k_proj.weight",
    "latent_deliberation.blocks.3.attn.o_proj.weight",
    "latent_deliberation.blocks.3.attn.q_norm.weight",
    "latent_deliberation.blocks.3.attn.q_proj.weight",
    "latent_deliberation.blocks.3.attn.v_proj.weight",
    "latent_deliberation.blocks.3.ff.down.weight",
    "latent_deliberation.blocks.3.ff.gate.weight",
    "latent_deliberation.blocks.3.ff.up.weight",
    "latent_deliberation.blocks.3.ff_norm.weight",
    "latent_deliberation.blocks.3.norm.weight",
    "latent_deliberation.experience_in.weight",
    "latent_deliberation.output_norm.weight",
    "latent_deliberation.persistent_memory_bus.alpha",
    "latent_deliberation.reason_embed.weight",
    "latent_deliberation.trajectory.cell.a_log",
    "latent_deliberation.trajectory.cell.b_proj.weight",
    "latent_deliberation.trajectory.cell.dt_bias",
    "latent_deliberation.trajectory.cell.f_proj.down.weight",
    "latent_deliberation.trajectory.cell.f_proj.up.weight",
    "latent_deliberation.trajectory.cell.g_proj.down.weight",
    "latent_deliberation.trajectory.cell.g_proj.up.weight",
    "latent_deliberation.trajectory.cell.k_proj.weight",
    "latent_deliberation.trajectory.cell.o_proj.weight",
    "latent_deliberation.trajectory.cell.q_proj.weight",
    "latent_deliberation.trajectory.cell.v_proj.weight",
    "latent_deliberation.trajectory.cell.w_proj.weight",
    "latent_deliberation.trajectory.persistent.a_log",
    "latent_deliberation.trajectory.persistent.b_proj.weight",
    "latent_deliberation.trajectory.persistent.dt_bias",
    "latent_deliberation.trajectory.persistent.f_proj.down.weight",
    "latent_deliberation.trajectory.persistent.f_proj.up.weight",
    "latent_deliberation.trajectory.persistent.g_proj.down.weight",
    "latent_deliberation.trajectory.persistent.g_proj.up.weight",
    "latent_deliberation.trajectory.persistent.k_proj.weight",
    "latent_deliberation.trajectory.persistent.o_proj.weight",
    "latent_deliberation.trajectory.persistent.q_proj.weight",
    "latent_deliberation.trajectory.persistent.v_proj.weight",
    "latent_deliberation.trajectory.persistent.w_proj.weight",
    "latent_deliberation.trajectory.probe_embed.weight",
    "latent_deliberation.trajectory.row.a_log",
    "latent_deliberation.trajectory.row.b_proj.weight",
    "latent_deliberation.trajectory.row.dt_bias",
    "latent_deliberation.trajectory.row.f_proj.down.weight",
    "latent_deliberation.trajectory.row.f_proj.up.weight",
    "latent_deliberation.trajectory.row.g_proj.down.weight",
    "latent_deliberation.trajectory.row.g_proj.up.weight",
    "latent_deliberation.trajectory.row.k_proj.weight",
    "latent_deliberation.trajectory.row.o_proj.weight",
    "latent_deliberation.trajectory.row.q_proj.weight",
    "latent_deliberation.trajectory.row.v_proj.weight",
    "latent_deliberation.trajectory.row.w_proj.weight",
    "latent_deliberation.working_memory_bus.address_norm.weight",
    "latent_deliberation.working_memory_bus.alpha",
    "latent_deliberation.working_memory_bus.k_proj.weight",
    "latent_deliberation.working_memory_bus.memory_norm.weight",
    "latent_deliberation.working_memory_bus.o_proj.0.weight",
    "latent_deliberation.working_memory_bus.o_proj.1.weight",
    "latent_deliberation.working_memory_bus.o_proj.2.weight",
    "latent_deliberation.working_memory_bus.o_proj.3.weight",
    "latent_deliberation.working_memory_bus.o_proj.4.weight",
    "latent_deliberation.working_memory_bus.q_norm.weight",
    "latent_deliberation.working_memory_bus.q_proj.0.weight",
    "latent_deliberation.working_memory_bus.q_proj.1.weight",
    "latent_deliberation.working_memory_bus.q_proj.2.weight",
    "latent_deliberation.working_memory_bus.q_proj.3.weight",
    "latent_deliberation.working_memory_bus.q_proj.4.weight",
    "latent_deliberation.working_memory_bus.rel_bias",
    "latent_deliberation.working_memory_bus.v_proj.weight",
    "model.decoder.layers.0.experts.down_proj.lora_a",
    "model.decoder.layers.0.experts.down_proj.lora_b",
    "model.decoder.layers.0.experts.gate_up_proj.lora_a",
    "model.decoder.layers.0.experts.gate_up_proj.lora_b",
    "model.decoder.layers.0.mlp.down_proj.lora_a",
    "model.decoder.layers.0.mlp.down_proj.lora_b",
    "model.decoder.layers.0.mlp.gate_proj.lora_a",
    "model.decoder.layers.0.mlp.gate_proj.lora_b",
    "model.decoder.layers.0.mlp.up_proj.lora_a",
    "model.decoder.layers.0.mlp.up_proj.lora_b",
    "model.decoder.layers.0.self_attn.k_proj.lora_a",
    "model.decoder.layers.0.self_attn.k_proj.lora_b",
    "model.decoder.layers.0.self_attn.o_proj.lora_a",
    "model.decoder.layers.0.self_attn.o_proj.lora_b",
    "model.decoder.layers.0.self_attn.q_proj.lora_a",
    "model.decoder.layers.0.self_attn.q_proj.lora_b",
    "model.decoder.layers.0.self_attn.v_proj.lora_a",
    "model.decoder.layers.0.self_attn.v_proj.lora_b",
    "model.decoder.layers.1.experts.down_proj.lora_a",
    "model.decoder.layers.1.experts.down_proj.lora_b",
    "model.decoder.layers.1.experts.gate_up_proj.lora_a",
    "model.decoder.layers.1.experts.gate_up_proj.lora_b",
    "model.decoder.layers.1.mlp.down_proj.lora_a",
    "model.decoder.layers.1.mlp.down_proj.lora_b",
    "model.decoder.layers.1.mlp.gate_proj.lora_a",
    "model.decoder.layers.1.mlp.gate_proj.lora_b",
    "model.decoder.layers.1.mlp.up_proj.lora_a",
    "model.decoder.layers.1.mlp.up_proj.lora_b",
    "model.decoder.layers.1.self_attn.k_proj.lora_a",
    "model.decoder.layers.1.self_attn.k_proj.lora_b",
    "model.decoder.layers.1.self_attn.o_proj.lora_a",
    "model.decoder.layers.1.self_attn.o_proj.lora_b",
    "model.decoder.layers.1.self_attn.q_proj.lora_a",
    "model.decoder.layers.1.self_attn.q_proj.lora_b",
    "model.decoder.layers.1.self_attn.v_proj.lora_a",
    "model.decoder.layers.1.self_attn.v_proj.lora_b",
    "model.decoder.layers.10.experts.down_proj.lora_a",
    "model.decoder.layers.10.experts.down_proj.lora_b",
    "model.decoder.layers.10.experts.gate_up_proj.lora_a",
    "model.decoder.layers.10.experts.gate_up_proj.lora_b",
    "model.decoder.layers.10.mlp.down_proj.lora_a",
    "model.decoder.layers.10.mlp.down_proj.lora_b",
    "model.decoder.layers.10.mlp.gate_proj.lora_a",
    "model.decoder.layers.10.mlp.gate_proj.lora_b",
    "model.decoder.layers.10.mlp.up_proj.lora_a",
    "model.decoder.layers.10.mlp.up_proj.lora_b",
    "model.decoder.layers.10.self_attn.k_proj.lora_a",
    "model.decoder.layers.10.self_attn.k_proj.lora_b",
    "model.decoder.layers.10.self_attn.o_proj.lora_a",
    "model.decoder.layers.10.self_attn.o_proj.lora_b",
    "model.decoder.layers.10.self_attn.q_proj.lora_a",
    "model.decoder.layers.10.self_attn.q_proj.lora_b",
    "model.decoder.layers.10.self_attn.v_proj.lora_a",
    "model.decoder.layers.10.self_attn.v_proj.lora_b",
    "model.decoder.layers.11.experts.down_proj.lora_a",
    "model.decoder.layers.11.experts.down_proj.lora_b",
    "model.decoder.layers.11.experts.gate_up_proj.lora_a",
    "model.decoder.layers.11.experts.gate_up_proj.lora_b",
    "model.decoder.layers.11.mlp.down_proj.lora_a",
    "model.decoder.layers.11.mlp.down_proj.lora_b",
    "model.decoder.layers.11.mlp.gate_proj.lora_a",
    "model.decoder.layers.11.mlp.gate_proj.lora_b",
    "model.decoder.layers.11.mlp.up_proj.lora_a",
    "model.decoder.layers.11.mlp.up_proj.lora_b",
    "model.decoder.layers.11.self_attn.k_proj.lora_a",
    "model.decoder.layers.11.self_attn.k_proj.lora_b",
    "model.decoder.layers.11.self_attn.o_proj.lora_a",
    "model.decoder.layers.11.self_attn.o_proj.lora_b",
    "model.decoder.layers.11.self_attn.q_proj.lora_a",
    "model.decoder.layers.11.self_attn.q_proj.lora_b",
    "model.decoder.layers.12.experts.down_proj.lora_a",
    "model.decoder.layers.12.experts.down_proj.lora_b",
    "model.decoder.layers.12.experts.gate_up_proj.lora_a",
    "model.decoder.layers.12.experts.gate_up_proj.lora_b",
    "model.decoder.layers.12.mlp.down_proj.lora_a",
    "model.decoder.layers.12.mlp.down_proj.lora_b",
    "model.decoder.layers.12.mlp.gate_proj.lora_a",
    "model.decoder.layers.12.mlp.gate_proj.lora_b",
    "model.decoder.layers.12.mlp.up_proj.lora_a",
    "model.decoder.layers.12.mlp.up_proj.lora_b",
    "model.decoder.layers.12.self_attn.k_proj.lora_a",
    "model.decoder.layers.12.self_attn.k_proj.lora_b",
    "model.decoder.layers.12.self_attn.o_proj.lora_a",
    "model.decoder.layers.12.self_attn.o_proj.lora_b",
    "model.decoder.layers.12.self_attn.q_proj.lora_a",
    "model.decoder.layers.12.self_attn.q_proj.lora_b",
    "model.decoder.layers.12.self_attn.v_proj.lora_a",
    "model.decoder.layers.12.self_attn.v_proj.lora_b",
    "model.decoder.layers.13.experts.down_proj.lora_a",
    "model.decoder.layers.13.experts.down_proj.lora_b",
    "model.decoder.layers.13.experts.gate_up_proj.lora_a",
    "model.decoder.layers.13.experts.gate_up_proj.lora_b",
    "model.decoder.layers.13.mlp.down_proj.lora_a",
    "model.decoder.layers.13.mlp.down_proj.lora_b",
    "model.decoder.layers.13.mlp.gate_proj.lora_a",
    "model.decoder.layers.13.mlp.gate_proj.lora_b",
    "model.decoder.layers.13.mlp.up_proj.lora_a",
    "model.decoder.layers.13.mlp.up_proj.lora_b",
    "model.decoder.layers.13.self_attn.k_proj.lora_a",
    "model.decoder.layers.13.self_attn.k_proj.lora_b",
    "model.decoder.layers.13.self_attn.o_proj.lora_a",
    "model.decoder.layers.13.self_attn.o_proj.lora_b",
    "model.decoder.layers.13.self_attn.q_proj.lora_a",
    "model.decoder.layers.13.self_attn.q_proj.lora_b",
    "model.decoder.layers.13.self_attn.v_proj.lora_a",
    "model.decoder.layers.13.self_attn.v_proj.lora_b",
    "model.decoder.layers.14.experts.down_proj.lora_a",
    "model.decoder.layers.14.experts.down_proj.lora_b",
    "model.decoder.layers.14.experts.gate_up_proj.lora_a",
    "model.decoder.layers.14.experts.gate_up_proj.lora_b",
    "model.decoder.layers.14.mlp.down_proj.lora_a",
    "model.decoder.layers.14.mlp.down_proj.lora_b",
    "model.decoder.layers.14.mlp.gate_proj.lora_a",
    "model.decoder.layers.14.mlp.gate_proj.lora_b",
    "model.decoder.layers.14.mlp.up_proj.lora_a",
    "model.decoder.layers.14.mlp.up_proj.lora_b",
    "model.decoder.layers.14.self_attn.k_proj.lora_a",
    "model.decoder.layers.14.self_attn.k_proj.lora_b",
    "model.decoder.layers.14.self_attn.o_proj.lora_a",
    "model.decoder.layers.14.self_attn.o_proj.lora_b",
    "model.decoder.layers.14.self_attn.q_proj.lora_a",
    "model.decoder.layers.14.self_attn.q_proj.lora_b",
    "model.decoder.layers.14.self_attn.v_proj.lora_a",
    "model.decoder.layers.14.self_attn.v_proj.lora_b",
    "model.decoder.layers.15.experts.down_proj.lora_a",
    "model.decoder.layers.15.experts.down_proj.lora_b",
    "model.decoder.layers.15.experts.gate_up_proj.lora_a",
    "model.decoder.layers.15.experts.gate_up_proj.lora_b",
    "model.decoder.layers.15.mlp.down_proj.lora_a",
    "model.decoder.layers.15.mlp.down_proj.lora_b",
    "model.decoder.layers.15.mlp.gate_proj.lora_a",
    "model.decoder.layers.15.mlp.gate_proj.lora_b",
    "model.decoder.layers.15.mlp.up_proj.lora_a",
    "model.decoder.layers.15.mlp.up_proj.lora_b",
    "model.decoder.layers.15.self_attn.k_proj.lora_a",
    "model.decoder.layers.15.self_attn.k_proj.lora_b",
    "model.decoder.layers.15.self_attn.o_proj.lora_a",
    "model.decoder.layers.15.self_attn.o_proj.lora_b",
    "model.decoder.layers.15.self_attn.q_proj.lora_a",
    "model.decoder.layers.15.self_attn.q_proj.lora_b",
    "model.decoder.layers.15.self_attn.v_proj.lora_a",
    "model.decoder.layers.15.self_attn.v_proj.lora_b",
    "model.decoder.layers.16.experts.down_proj.lora_a",
    "model.decoder.layers.16.experts.down_proj.lora_b",
    "model.decoder.layers.16.experts.gate_up_proj.lora_a",
    "model.decoder.layers.16.experts.gate_up_proj.lora_b",
    "model.decoder.layers.16.mlp.down_proj.lora_a",
    "model.decoder.layers.16.mlp.down_proj.lora_b",
    "model.decoder.layers.16.mlp.gate_proj.lora_a",
    "model.decoder.layers.16.mlp.gate_proj.lora_b",
    "model.decoder.layers.16.mlp.up_proj.lora_a",
    "model.decoder.layers.16.mlp.up_proj.lora_b",
    "model.decoder.layers.16.self_attn.k_proj.lora_a",
    "model.decoder.layers.16.self_attn.k_proj.lora_b",
    "model.decoder.layers.16.self_attn.o_proj.lora_a",
    "model.decoder.layers.16.self_attn.o_proj.lora_b",
    "model.decoder.layers.16.self_attn.q_proj.lora_a",
    "model.decoder.layers.16.self_attn.q_proj.lora_b",
    "model.decoder.layers.16.self_attn.v_proj.lora_a",
    "model.decoder.layers.16.self_attn.v_proj.lora_b",
    "model.decoder.layers.17.experts.down_proj.lora_a",
    "model.decoder.layers.17.experts.down_proj.lora_b",
    "model.decoder.layers.17.experts.gate_up_proj.lora_a",
    "model.decoder.layers.17.experts.gate_up_proj.lora_b",
    "model.decoder.layers.17.mlp.down_proj.lora_a",
    "model.decoder.layers.17.mlp.down_proj.lora_b",
    "model.decoder.layers.17.mlp.gate_proj.lora_a",
    "model.decoder.layers.17.mlp.gate_proj.lora_b",
    "model.decoder.layers.17.mlp.up_proj.lora_a",
    "model.decoder.layers.17.mlp.up_proj.lora_b",
    "model.decoder.layers.17.self_attn.k_proj.lora_a",
    "model.decoder.layers.17.self_attn.k_proj.lora_b",
    "model.decoder.layers.17.self_attn.o_proj.lora_a",
    "model.decoder.layers.17.self_attn.o_proj.lora_b",
    "model.decoder.layers.17.self_attn.q_proj.lora_a",
    "model.decoder.layers.17.self_attn.q_proj.lora_b",
    "model.decoder.layers.18.experts.down_proj.lora_a",
    "model.decoder.layers.18.experts.down_proj.lora_b",
    "model.decoder.layers.18.experts.gate_up_proj.lora_a",
    "model.decoder.layers.18.experts.gate_up_proj.lora_b",
    "model.decoder.layers.18.mlp.down_proj.lora_a",
    "model.decoder.layers.18.mlp.down_proj.lora_b",
    "model.decoder.layers.18.mlp.gate_proj.lora_a",
    "model.decoder.layers.18.mlp.gate_proj.lora_b",
    "model.decoder.layers.18.mlp.up_proj.lora_a",
    "model.decoder.layers.18.mlp.up_proj.lora_b",
    "model.decoder.layers.18.self_attn.k_proj.lora_a",
    "model.decoder.layers.18.self_attn.k_proj.lora_b",
    "model.decoder.layers.18.self_attn.o_proj.lora_a",
    "model.decoder.layers.18.self_attn.o_proj.lora_b",
    "model.decoder.layers.18.self_attn.q_proj.lora_a",
    "model.decoder.layers.18.self_attn.q_proj.lora_b",
    "model.decoder.layers.18.self_attn.v_proj.lora_a",
    "model.decoder.layers.18.self_attn.v_proj.lora_b",
    "model.decoder.layers.19.experts.down_proj.lora_a",
    "model.decoder.layers.19.experts.down_proj.lora_b",
    "model.decoder.layers.19.experts.gate_up_proj.lora_a",
    "model.decoder.layers.19.experts.gate_up_proj.lora_b",
    "model.decoder.layers.19.mlp.down_proj.lora_a",
    "model.decoder.layers.19.mlp.down_proj.lora_b",
    "model.decoder.layers.19.mlp.gate_proj.lora_a",
    "model.decoder.layers.19.mlp.gate_proj.lora_b",
    "model.decoder.layers.19.mlp.up_proj.lora_a",
    "model.decoder.layers.19.mlp.up_proj.lora_b",
    "model.decoder.layers.19.self_attn.k_proj.lora_a",
    "model.decoder.layers.19.self_attn.k_proj.lora_b",
    "model.decoder.layers.19.self_attn.o_proj.lora_a",
    "model.decoder.layers.19.self_attn.o_proj.lora_b",
    "model.decoder.layers.19.self_attn.q_proj.lora_a",
    "model.decoder.layers.19.self_attn.q_proj.lora_b",
    "model.decoder.layers.19.self_attn.v_proj.lora_a",
    "model.decoder.layers.19.self_attn.v_proj.lora_b",
    "model.decoder.layers.2.experts.down_proj.lora_a",
    "model.decoder.layers.2.experts.down_proj.lora_b",
    "model.decoder.layers.2.experts.gate_up_proj.lora_a",
    "model.decoder.layers.2.experts.gate_up_proj.lora_b",
    "model.decoder.layers.2.mlp.down_proj.lora_a",
    "model.decoder.layers.2.mlp.down_proj.lora_b",
    "model.decoder.layers.2.mlp.gate_proj.lora_a",
    "model.decoder.layers.2.mlp.gate_proj.lora_b",
    "model.decoder.layers.2.mlp.up_proj.lora_a",
    "model.decoder.layers.2.mlp.up_proj.lora_b",
    "model.decoder.layers.2.self_attn.k_proj.lora_a",
    "model.decoder.layers.2.self_attn.k_proj.lora_b",
    "model.decoder.layers.2.self_attn.o_proj.lora_a",
    "model.decoder.layers.2.self_attn.o_proj.lora_b",
    "model.decoder.layers.2.self_attn.q_proj.lora_a",
    "model.decoder.layers.2.self_attn.q_proj.lora_b",
    "model.decoder.layers.2.self_attn.v_proj.lora_a",
    "model.decoder.layers.2.self_attn.v_proj.lora_b",
    "model.decoder.layers.20.experts.down_proj.lora_a",
    "model.decoder.layers.20.experts.down_proj.lora_b",
    "model.decoder.layers.20.experts.gate_up_proj.lora_a",
    "model.decoder.layers.20.experts.gate_up_proj.lora_b",
    "model.decoder.layers.20.mlp.down_proj.lora_a",
    "model.decoder.layers.20.mlp.down_proj.lora_b",
    "model.decoder.layers.20.mlp.gate_proj.lora_a",
    "model.decoder.layers.20.mlp.gate_proj.lora_b",
    "model.decoder.layers.20.mlp.up_proj.lora_a",
    "model.decoder.layers.20.mlp.up_proj.lora_b",
    "model.decoder.layers.20.self_attn.k_proj.lora_a",
    "model.decoder.layers.20.self_attn.k_proj.lora_b",
    "model.decoder.layers.20.self_attn.o_proj.lora_a",
    "model.decoder.layers.20.self_attn.o_proj.lora_b",
    "model.decoder.layers.20.self_attn.q_proj.lora_a",
    "model.decoder.layers.20.self_attn.q_proj.lora_b",
    "model.decoder.layers.20.self_attn.v_proj.lora_a",
    "model.decoder.layers.20.self_attn.v_proj.lora_b",
    "model.decoder.layers.21.experts.down_proj.lora_a",
    "model.decoder.layers.21.experts.down_proj.lora_b",
    "model.decoder.layers.21.experts.gate_up_proj.lora_a",
    "model.decoder.layers.21.experts.gate_up_proj.lora_b",
    "model.decoder.layers.21.mlp.down_proj.lora_a",
    "model.decoder.layers.21.mlp.down_proj.lora_b",
    "model.decoder.layers.21.mlp.gate_proj.lora_a",
    "model.decoder.layers.21.mlp.gate_proj.lora_b",
    "model.decoder.layers.21.mlp.up_proj.lora_a",
    "model.decoder.layers.21.mlp.up_proj.lora_b",
    "model.decoder.layers.21.self_attn.k_proj.lora_a",
    "model.decoder.layers.21.self_attn.k_proj.lora_b",
    "model.decoder.layers.21.self_attn.o_proj.lora_a",
    "model.decoder.layers.21.self_attn.o_proj.lora_b",
    "model.decoder.layers.21.self_attn.q_proj.lora_a",
    "model.decoder.layers.21.self_attn.q_proj.lora_b",
    "model.decoder.layers.21.self_attn.v_proj.lora_a",
    "model.decoder.layers.21.self_attn.v_proj.lora_b",
    "model.decoder.layers.22.experts.down_proj.lora_a",
    "model.decoder.layers.22.experts.down_proj.lora_b",
    "model.decoder.layers.22.experts.gate_up_proj.lora_a",
    "model.decoder.layers.22.experts.gate_up_proj.lora_b",
    "model.decoder.layers.22.mlp.down_proj.lora_a",
    "model.decoder.layers.22.mlp.down_proj.lora_b",
    "model.decoder.layers.22.mlp.gate_proj.lora_a",
    "model.decoder.layers.22.mlp.gate_proj.lora_b",
    "model.decoder.layers.22.mlp.up_proj.lora_a",
    "model.decoder.layers.22.mlp.up_proj.lora_b",
    "model.decoder.layers.22.self_attn.k_proj.lora_a",
    "model.decoder.layers.22.self_attn.k_proj.lora_b",
    "model.decoder.layers.22.self_attn.o_proj.lora_a",
    "model.decoder.layers.22.self_attn.o_proj.lora_b",
    "model.decoder.layers.22.self_attn.q_proj.lora_a",
    "model.decoder.layers.22.self_attn.q_proj.lora_b",
    "model.decoder.layers.22.self_attn.v_proj.lora_a",
    "model.decoder.layers.22.self_attn.v_proj.lora_b",
    "model.decoder.layers.23.experts.down_proj.lora_a",
    "model.decoder.layers.23.experts.down_proj.lora_b",
    "model.decoder.layers.23.experts.gate_up_proj.lora_a",
    "model.decoder.layers.23.experts.gate_up_proj.lora_b",
    "model.decoder.layers.23.mlp.down_proj.lora_a",
    "model.decoder.layers.23.mlp.down_proj.lora_b",
    "model.decoder.layers.23.mlp.gate_proj.lora_a",
    "model.decoder.layers.23.mlp.gate_proj.lora_b",
    "model.decoder.layers.23.mlp.up_proj.lora_a",
    "model.decoder.layers.23.mlp.up_proj.lora_b",
    "model.decoder.layers.23.self_attn.k_proj.lora_a",
    "model.decoder.layers.23.self_attn.k_proj.lora_b",
    "model.decoder.layers.23.self_attn.o_proj.lora_a",
    "model.decoder.layers.23.self_attn.o_proj.lora_b",
    "model.decoder.layers.23.self_attn.q_proj.lora_a",
    "model.decoder.layers.23.self_attn.q_proj.lora_b",
    "model.decoder.layers.24.experts.down_proj.lora_a",
    "model.decoder.layers.24.experts.down_proj.lora_b",
    "model.decoder.layers.24.experts.gate_up_proj.lora_a",
    "model.decoder.layers.24.experts.gate_up_proj.lora_b",
    "model.decoder.layers.24.mlp.down_proj.lora_a",
    "model.decoder.layers.24.mlp.down_proj.lora_b",
    "model.decoder.layers.24.mlp.gate_proj.lora_a",
    "model.decoder.layers.24.mlp.gate_proj.lora_b",
    "model.decoder.layers.24.mlp.up_proj.lora_a",
    "model.decoder.layers.24.mlp.up_proj.lora_b",
    "model.decoder.layers.24.self_attn.k_proj.lora_a",
    "model.decoder.layers.24.self_attn.k_proj.lora_b",
    "model.decoder.layers.24.self_attn.o_proj.lora_a",
    "model.decoder.layers.24.self_attn.o_proj.lora_b",
    "model.decoder.layers.24.self_attn.q_proj.lora_a",
    "model.decoder.layers.24.self_attn.q_proj.lora_b",
    "model.decoder.layers.24.self_attn.v_proj.lora_a",
    "model.decoder.layers.24.self_attn.v_proj.lora_b",
    "model.decoder.layers.25.experts.down_proj.lora_a",
    "model.decoder.layers.25.experts.down_proj.lora_b",
    "model.decoder.layers.25.experts.gate_up_proj.lora_a",
    "model.decoder.layers.25.experts.gate_up_proj.lora_b",
    "model.decoder.layers.25.mlp.down_proj.lora_a",
    "model.decoder.layers.25.mlp.down_proj.lora_b",
    "model.decoder.layers.25.mlp.gate_proj.lora_a",
    "model.decoder.layers.25.mlp.gate_proj.lora_b",
    "model.decoder.layers.25.mlp.up_proj.lora_a",
    "model.decoder.layers.25.mlp.up_proj.lora_b",
    "model.decoder.layers.25.self_attn.k_proj.lora_a",
    "model.decoder.layers.25.self_attn.k_proj.lora_b",
    "model.decoder.layers.25.self_attn.o_proj.lora_a",
    "model.decoder.layers.25.self_attn.o_proj.lora_b",
    "model.decoder.layers.25.self_attn.q_proj.lora_a",
    "model.decoder.layers.25.self_attn.q_proj.lora_b",
    "model.decoder.layers.25.self_attn.v_proj.lora_a",
    "model.decoder.layers.25.self_attn.v_proj.lora_b",
    "model.decoder.layers.26.experts.down_proj.lora_a",
    "model.decoder.layers.26.experts.down_proj.lora_b",
    "model.decoder.layers.26.experts.gate_up_proj.lora_a",
    "model.decoder.layers.26.experts.gate_up_proj.lora_b",
    "model.decoder.layers.26.mlp.down_proj.lora_a",
    "model.decoder.layers.26.mlp.down_proj.lora_b",
    "model.decoder.layers.26.mlp.gate_proj.lora_a",
    "model.decoder.layers.26.mlp.gate_proj.lora_b",
    "model.decoder.layers.26.mlp.up_proj.lora_a",
    "model.decoder.layers.26.mlp.up_proj.lora_b",
    "model.decoder.layers.26.self_attn.k_proj.lora_a",
    "model.decoder.layers.26.self_attn.k_proj.lora_b",
    "model.decoder.layers.26.self_attn.o_proj.lora_a",
    "model.decoder.layers.26.self_attn.o_proj.lora_b",
    "model.decoder.layers.26.self_attn.q_proj.lora_a",
    "model.decoder.layers.26.self_attn.q_proj.lora_b",
    "model.decoder.layers.26.self_attn.v_proj.lora_a",
    "model.decoder.layers.26.self_attn.v_proj.lora_b",
    "model.decoder.layers.27.experts.down_proj.lora_a",
    "model.decoder.layers.27.experts.down_proj.lora_b",
    "model.decoder.layers.27.experts.gate_up_proj.lora_a",
    "model.decoder.layers.27.experts.gate_up_proj.lora_b",
    "model.decoder.layers.27.mlp.down_proj.lora_a",
    "model.decoder.layers.27.mlp.down_proj.lora_b",
    "model.decoder.layers.27.mlp.gate_proj.lora_a",
    "model.decoder.layers.27.mlp.gate_proj.lora_b",
    "model.decoder.layers.27.mlp.up_proj.lora_a",
    "model.decoder.layers.27.mlp.up_proj.lora_b",
    "model.decoder.layers.27.self_attn.k_proj.lora_a",
    "model.decoder.layers.27.self_attn.k_proj.lora_b",
    "model.decoder.layers.27.self_attn.o_proj.lora_a",
    "model.decoder.layers.27.self_attn.o_proj.lora_b",
    "model.decoder.layers.27.self_attn.q_proj.lora_a",
    "model.decoder.layers.27.self_attn.q_proj.lora_b",
    "model.decoder.layers.27.self_attn.v_proj.lora_a",
    "model.decoder.layers.27.self_attn.v_proj.lora_b",
    "model.decoder.layers.28.experts.down_proj.lora_a",
    "model.decoder.layers.28.experts.down_proj.lora_b",
    "model.decoder.layers.28.experts.gate_up_proj.lora_a",
    "model.decoder.layers.28.experts.gate_up_proj.lora_b",
    "model.decoder.layers.28.mlp.down_proj.lora_a",
    "model.decoder.layers.28.mlp.down_proj.lora_b",
    "model.decoder.layers.28.mlp.gate_proj.lora_a",
    "model.decoder.layers.28.mlp.gate_proj.lora_b",
    "model.decoder.layers.28.mlp.up_proj.lora_a",
    "model.decoder.layers.28.mlp.up_proj.lora_b",
    "model.decoder.layers.28.self_attn.k_proj.lora_a",
    "model.decoder.layers.28.self_attn.k_proj.lora_b",
    "model.decoder.layers.28.self_attn.o_proj.lora_a",
    "model.decoder.layers.28.self_attn.o_proj.lora_b",
    "model.decoder.layers.28.self_attn.q_proj.lora_a",
    "model.decoder.layers.28.self_attn.q_proj.lora_b",
    "model.decoder.layers.28.self_attn.v_proj.lora_a",
    "model.decoder.layers.28.self_attn.v_proj.lora_b",
    "model.decoder.layers.29.experts.down_proj.lora_a",
    "model.decoder.layers.29.experts.down_proj.lora_b",
    "model.decoder.layers.29.experts.gate_up_proj.lora_a",
    "model.decoder.layers.29.experts.gate_up_proj.lora_b",
    "model.decoder.layers.29.mlp.down_proj.lora_a",
    "model.decoder.layers.29.mlp.down_proj.lora_b",
    "model.decoder.layers.29.mlp.gate_proj.lora_a",
    "model.decoder.layers.29.mlp.gate_proj.lora_b",
    "model.decoder.layers.29.mlp.up_proj.lora_a",
    "model.decoder.layers.29.mlp.up_proj.lora_b",
    "model.decoder.layers.29.self_attn.k_proj.lora_a",
    "model.decoder.layers.29.self_attn.k_proj.lora_b",
    "model.decoder.layers.29.self_attn.o_proj.lora_a",
    "model.decoder.layers.29.self_attn.o_proj.lora_b",
    "model.decoder.layers.29.self_attn.q_proj.lora_a",
    "model.decoder.layers.29.self_attn.q_proj.lora_b",
    "model.decoder.layers.3.experts.down_proj.lora_a",
    "model.decoder.layers.3.experts.down_proj.lora_b",
    "model.decoder.layers.3.experts.gate_up_proj.lora_a",
    "model.decoder.layers.3.experts.gate_up_proj.lora_b",
    "model.decoder.layers.3.mlp.down_proj.lora_a",
    "model.decoder.layers.3.mlp.down_proj.lora_b",
    "model.decoder.layers.3.mlp.gate_proj.lora_a",
    "model.decoder.layers.3.mlp.gate_proj.lora_b",
    "model.decoder.layers.3.mlp.up_proj.lora_a",
    "model.decoder.layers.3.mlp.up_proj.lora_b",
    "model.decoder.layers.3.self_attn.k_proj.lora_a",
    "model.decoder.layers.3.self_attn.k_proj.lora_b",
    "model.decoder.layers.3.self_attn.o_proj.lora_a",
    "model.decoder.layers.3.self_attn.o_proj.lora_b",
    "model.decoder.layers.3.self_attn.q_proj.lora_a",
    "model.decoder.layers.3.self_attn.q_proj.lora_b",
    "model.decoder.layers.3.self_attn.v_proj.lora_a",
    "model.decoder.layers.3.self_attn.v_proj.lora_b",
    "model.decoder.layers.4.experts.down_proj.lora_a",
    "model.decoder.layers.4.experts.down_proj.lora_b",
    "model.decoder.layers.4.experts.gate_up_proj.lora_a",
    "model.decoder.layers.4.experts.gate_up_proj.lora_b",
    "model.decoder.layers.4.mlp.down_proj.lora_a",
    "model.decoder.layers.4.mlp.down_proj.lora_b",
    "model.decoder.layers.4.mlp.gate_proj.lora_a",
    "model.decoder.layers.4.mlp.gate_proj.lora_b",
    "model.decoder.layers.4.mlp.up_proj.lora_a",
    "model.decoder.layers.4.mlp.up_proj.lora_b",
    "model.decoder.layers.4.self_attn.k_proj.lora_a",
    "model.decoder.layers.4.self_attn.k_proj.lora_b",
    "model.decoder.layers.4.self_attn.o_proj.lora_a",
    "model.decoder.layers.4.self_attn.o_proj.lora_b",
    "model.decoder.layers.4.self_attn.q_proj.lora_a",
    "model.decoder.layers.4.self_attn.q_proj.lora_b",
    "model.decoder.layers.4.self_attn.v_proj.lora_a",
    "model.decoder.layers.4.self_attn.v_proj.lora_b",
    "model.decoder.layers.5.experts.down_proj.lora_a",
    "model.decoder.layers.5.experts.down_proj.lora_b",
    "model.decoder.layers.5.experts.gate_up_proj.lora_a",
    "model.decoder.layers.5.experts.gate_up_proj.lora_b",
    "model.decoder.layers.5.mlp.down_proj.lora_a",
    "model.decoder.layers.5.mlp.down_proj.lora_b",
    "model.decoder.layers.5.mlp.gate_proj.lora_a",
    "model.decoder.layers.5.mlp.gate_proj.lora_b",
    "model.decoder.layers.5.mlp.up_proj.lora_a",
    "model.decoder.layers.5.mlp.up_proj.lora_b",
    "model.decoder.layers.5.self_attn.k_proj.lora_a",
    "model.decoder.layers.5.self_attn.k_proj.lora_b",
    "model.decoder.layers.5.self_attn.o_proj.lora_a",
    "model.decoder.layers.5.self_attn.o_proj.lora_b",
    "model.decoder.layers.5.self_attn.q_proj.lora_a",
    "model.decoder.layers.5.self_attn.q_proj.lora_b",
    "model.decoder.layers.6.experts.down_proj.lora_a",
    "model.decoder.layers.6.experts.down_proj.lora_b",
    "model.decoder.layers.6.experts.gate_up_proj.lora_a",
    "model.decoder.layers.6.experts.gate_up_proj.lora_b",
    "model.decoder.layers.6.mlp.down_proj.lora_a",
    "model.decoder.layers.6.mlp.down_proj.lora_b",
    "model.decoder.layers.6.mlp.gate_proj.lora_a",
    "model.decoder.layers.6.mlp.gate_proj.lora_b",
    "model.decoder.layers.6.mlp.up_proj.lora_a",
    "model.decoder.layers.6.mlp.up_proj.lora_b",
    "model.decoder.layers.6.self_attn.k_proj.lora_a",
    "model.decoder.layers.6.self_attn.k_proj.lora_b",
    "model.decoder.layers.6.self_attn.o_proj.lora_a",
    "model.decoder.layers.6.self_attn.o_proj.lora_b",
    "model.decoder.layers.6.self_attn.q_proj.lora_a",
    "model.decoder.layers.6.self_attn.q_proj.lora_b",
    "model.decoder.layers.6.self_attn.v_proj.lora_a",
    "model.decoder.layers.6.self_attn.v_proj.lora_b",
    "model.decoder.layers.7.experts.down_proj.lora_a",
    "model.decoder.layers.7.experts.down_proj.lora_b",
    "model.decoder.layers.7.experts.gate_up_proj.lora_a",
    "model.decoder.layers.7.experts.gate_up_proj.lora_b",
    "model.decoder.layers.7.mlp.down_proj.lora_a",
    "model.decoder.layers.7.mlp.down_proj.lora_b",
    "model.decoder.layers.7.mlp.gate_proj.lora_a",
    "model.decoder.layers.7.mlp.gate_proj.lora_b",
    "model.decoder.layers.7.mlp.up_proj.lora_a",
    "model.decoder.layers.7.mlp.up_proj.lora_b",
    "model.decoder.layers.7.self_attn.k_proj.lora_a",
    "model.decoder.layers.7.self_attn.k_proj.lora_b",
    "model.decoder.layers.7.self_attn.o_proj.lora_a",
    "model.decoder.layers.7.self_attn.o_proj.lora_b",
    "model.decoder.layers.7.self_attn.q_proj.lora_a",
    "model.decoder.layers.7.self_attn.q_proj.lora_b",
    "model.decoder.layers.7.self_attn.v_proj.lora_a",
    "model.decoder.layers.7.self_attn.v_proj.lora_b",
    "model.decoder.layers.8.experts.down_proj.lora_a",
    "model.decoder.layers.8.experts.down_proj.lora_b",
    "model.decoder.layers.8.experts.gate_up_proj.lora_a",
    "model.decoder.layers.8.experts.gate_up_proj.lora_b",
    "model.decoder.layers.8.mlp.down_proj.lora_a",
    "model.decoder.layers.8.mlp.down_proj.lora_b",
    "model.decoder.layers.8.mlp.gate_proj.lora_a",
    "model.decoder.layers.8.mlp.gate_proj.lora_b",
    "model.decoder.layers.8.mlp.up_proj.lora_a",
    "model.decoder.layers.8.mlp.up_proj.lora_b",
    "model.decoder.layers.8.self_attn.k_proj.lora_a",
    "model.decoder.layers.8.self_attn.k_proj.lora_b",
    "model.decoder.layers.8.self_attn.o_proj.lora_a",
    "model.decoder.layers.8.self_attn.o_proj.lora_b",
    "model.decoder.layers.8.self_attn.q_proj.lora_a",
    "model.decoder.layers.8.self_attn.q_proj.lora_b",
    "model.decoder.layers.8.self_attn.v_proj.lora_a",
    "model.decoder.layers.8.self_attn.v_proj.lora_b",
    "model.decoder.layers.9.experts.down_proj.lora_a",
    "model.decoder.layers.9.experts.down_proj.lora_b",
    "model.decoder.layers.9.experts.gate_up_proj.lora_a",
    "model.decoder.layers.9.experts.gate_up_proj.lora_b",
    "model.decoder.layers.9.mlp.down_proj.lora_a",
    "model.decoder.layers.9.mlp.down_proj.lora_b",
    "model.decoder.layers.9.mlp.gate_proj.lora_a",
    "model.decoder.layers.9.mlp.gate_proj.lora_b",
    "model.decoder.layers.9.mlp.up_proj.lora_a",
    "model.decoder.layers.9.mlp.up_proj.lora_b",
    "model.decoder.layers.9.self_attn.k_proj.lora_a",
    "model.decoder.layers.9.self_attn.k_proj.lora_b",
    "model.decoder.layers.9.self_attn.o_proj.lora_a",
    "model.decoder.layers.9.self_attn.o_proj.lora_b",
    "model.decoder.layers.9.self_attn.q_proj.lora_a",
    "model.decoder.layers.9.self_attn.q_proj.lora_b",
    "model.decoder.layers.9.self_attn.v_proj.lora_a",
    "model.decoder.layers.9.self_attn.v_proj.lora_b"
  ],
  "trainable_tensor_count": 632,
  "total_tensor_count": 1323,
  "total_size": 51791503298,
  "tensors": {
    "model.decoder.embed_tokens.weight": {
      "shape": [
        262144,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.0.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.1.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.2.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.3.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.4.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.self_attn.q_proj.linear.weight": {
      "shape": [
        8192,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        8192
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.self_attn.k_proj.linear.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        1024
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        8192
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.self_attn.o_proj.lora_a": {
      "shape": [
        8192,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.self_attn.q_norm.weight": {
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.self_attn.k_norm.weight": {
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.5.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.6.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.7.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.8.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.9.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.10.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.self_attn.q_proj.linear.weight": {
      "shape": [
        8192,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        8192
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.self_attn.k_proj.linear.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        1024
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        8192
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.self_attn.o_proj.lora_a": {
      "shape": [
        8192,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.self_attn.q_norm.weight": {
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.self_attn.k_norm.weight": {
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.11.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.12.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.13.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.14.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.15.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.16.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.self_attn.q_proj.linear.weight": {
      "shape": [
        8192,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        8192
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.self_attn.k_proj.linear.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        1024
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        8192
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.self_attn.o_proj.lora_a": {
      "shape": [
        8192,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.self_attn.q_norm.weight": {
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.self_attn.k_norm.weight": {
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.17.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.18.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.19.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.20.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.21.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.22.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.self_attn.q_proj.linear.weight": {
      "shape": [
        8192,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        8192
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.self_attn.k_proj.linear.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        1024
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        8192
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.self_attn.o_proj.lora_a": {
      "shape": [
        8192,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.self_attn.q_norm.weight": {
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.self_attn.k_norm.weight": {
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.23.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.24.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.25.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.26.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.27.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.self_attn.q_proj.linear.weight": {
      "shape": [
        4096,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.self_attn.k_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.self_attn.v_proj.linear.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.self_attn.v_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.self_attn.v_proj.lora_b": {
      "shape": [
        16,
        2048
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        4096
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.self_attn.o_proj.lora_a": {
      "shape": [
        4096,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.self_attn.q_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.self_attn.k_norm.weight": {
      "shape": [
        256
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.28.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.self_attn.q_proj.linear.weight": {
      "shape": [
        8192,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.self_attn.q_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.self_attn.q_proj.lora_b": {
      "shape": [
        16,
        8192
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.self_attn.k_proj.linear.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.self_attn.k_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.self_attn.k_proj.lora_b": {
      "shape": [
        16,
        1024
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.self_attn.o_proj.linear.weight": {
      "shape": [
        2816,
        8192
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.self_attn.o_proj.lora_a": {
      "shape": [
        8192,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.self_attn.o_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.self_attn.q_norm.weight": {
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.self_attn.k_norm.weight": {
      "shape": [
        512
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.mlp.gate_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.mlp.gate_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.mlp.gate_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.mlp.up_proj.linear.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.mlp.up_proj.lora_a": {
      "shape": [
        2816,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.mlp.up_proj.lora_b": {
      "shape": [
        16,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.mlp.down_proj.linear.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.mlp.down_proj.lora_a": {
      "shape": [
        2112,
        16
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.mlp.down_proj.lora_b": {
      "shape": [
        16,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.input_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.post_attention_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.pre_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.post_feedforward_layernorm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.router.proj.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.router.scale": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.router.per_expert_scale": {
      "shape": [
        128
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.experts.gate_up_proj.linear.weight": {
      "shape": [
        128,
        1408,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.experts.gate_up_proj.lora_a": {
      "shape": [
        128,
        8,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.experts.gate_up_proj.lora_b": {
      "shape": [
        128,
        1408,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.experts.down_proj.linear.weight": {
      "shape": [
        128,
        2816,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.experts.down_proj.lora_a": {
      "shape": [
        128,
        8,
        704
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.experts.down_proj.lora_b": {
      "shape": [
        128,
        2816,
        8
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.post_feedforward_layernorm_1.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.post_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.pre_feedforward_layernorm_2.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.layers.29.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.self_conditioning.pre_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.self_conditioning.gate_proj.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.self_conditioning.up_proj.weight": {
      "shape": [
        2112,
        2816
      ],
      "dtype": "bfloat16"
    },
    "model.decoder.self_conditioning.down_proj.weight": {
      "shape": [
        2816,
        2112
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.0.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.1.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.2.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.3.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.4.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.5.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.6.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.7.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.8.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.9.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.10.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.11.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.12.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.13.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.14.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.15.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.16.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.17.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.18.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.19.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.20.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.21.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.22.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.23.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.24.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.25.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.26.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.27.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.28.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "model.encoder.language_model.layers.29.layer_scalar": {
      "shape": [
        1
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.cell.q_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.cell.k_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.cell.v_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.cell.f_proj.down.weight": {
      "shape": [
        64,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.cell.f_proj.up.weight": {
      "shape": [
        1024,
        64
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.cell.b_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.cell.w_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.cell.g_proj.down.weight": {
      "shape": [
        64,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.cell.g_proj.up.weight": {
      "shape": [
        1024,
        64
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.cell.o_proj.weight": {
      "shape": [
        2816,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.cell.a_log": {
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "latent_deliberation.trajectory.cell.dt_bias": {
      "shape": [
        16,
        64
      ],
      "dtype": "float32"
    },
    "latent_deliberation.trajectory.row.q_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.row.k_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.row.v_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.row.f_proj.down.weight": {
      "shape": [
        64,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.row.f_proj.up.weight": {
      "shape": [
        1024,
        64
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.row.b_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.row.w_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.row.g_proj.down.weight": {
      "shape": [
        64,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.row.g_proj.up.weight": {
      "shape": [
        1024,
        64
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.row.o_proj.weight": {
      "shape": [
        2816,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.row.a_log": {
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "latent_deliberation.trajectory.row.dt_bias": {
      "shape": [
        16,
        64
      ],
      "dtype": "float32"
    },
    "latent_deliberation.trajectory.persistent.q_proj.weight": {
      "shape": [
        2048,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.persistent.k_proj.weight": {
      "shape": [
        2048,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.persistent.v_proj.weight": {
      "shape": [
        2048,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.persistent.f_proj.down.weight": {
      "shape": [
        128,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.persistent.f_proj.up.weight": {
      "shape": [
        2048,
        128
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.persistent.b_proj.weight": {
      "shape": [
        2048,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.persistent.w_proj.weight": {
      "shape": [
        2048,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.persistent.g_proj.down.weight": {
      "shape": [
        128,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.persistent.g_proj.up.weight": {
      "shape": [
        2048,
        128
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.persistent.o_proj.weight": {
      "shape": [
        2816,
        2048
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.trajectory.persistent.a_log": {
      "shape": [
        16
      ],
      "dtype": "float32"
    },
    "latent_deliberation.trajectory.persistent.dt_bias": {
      "shape": [
        16,
        128
      ],
      "dtype": "float32"
    },
    "latent_deliberation.trajectory.probe_embed.weight": {
      "shape": [
        4,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.0.norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.0.attn.q_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.0.attn.k_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.0.attn.v_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.0.attn.o_proj.weight": {
      "shape": [
        2816,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.0.attn.q_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.0.attn.k_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.0.ff_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.0.ff.gate.weight": {
      "shape": [
        7168,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.0.ff.up.weight": {
      "shape": [
        7168,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.0.ff.down.weight": {
      "shape": [
        2816,
        7168
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.1.norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.1.attn.q_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.1.attn.k_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.1.attn.v_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.1.attn.o_proj.weight": {
      "shape": [
        2816,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.1.attn.q_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.1.attn.k_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.1.ff_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.1.ff.gate.weight": {
      "shape": [
        7168,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.1.ff.up.weight": {
      "shape": [
        7168,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.1.ff.down.weight": {
      "shape": [
        2816,
        7168
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.2.norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.2.attn.q_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.2.attn.k_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.2.attn.v_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.2.attn.o_proj.weight": {
      "shape": [
        2816,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.2.attn.q_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.2.attn.k_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.2.ff_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.2.ff.gate.weight": {
      "shape": [
        7168,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.2.ff.up.weight": {
      "shape": [
        7168,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.2.ff.down.weight": {
      "shape": [
        2816,
        7168
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.3.norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.3.attn.q_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.3.attn.k_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.3.attn.v_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.3.attn.o_proj.weight": {
      "shape": [
        2816,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.3.attn.q_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.3.attn.k_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.3.ff_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.blocks.3.ff.gate.weight": {
      "shape": [
        7168,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.3.ff.up.weight": {
      "shape": [
        7168,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.blocks.3.ff.down.weight": {
      "shape": [
        2816,
        7168
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.output_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.working_memory_bus.memory_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.working_memory_bus.address_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.working_memory_bus.k_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.working_memory_bus.v_proj.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.working_memory_bus.q_norm.weight": {
      "shape": [
        2816
      ],
      "dtype": "float32"
    },
    "latent_deliberation.working_memory_bus.q_proj.0.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.working_memory_bus.q_proj.1.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.working_memory_bus.q_proj.2.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.working_memory_bus.q_proj.3.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.working_memory_bus.q_proj.4.weight": {
      "shape": [
        1024,
        2816
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.working_memory_bus.o_proj.0.weight": {
      "shape": [
        2816,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.working_memory_bus.o_proj.1.weight": {
      "shape": [
        2816,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.working_memory_bus.o_proj.2.weight": {
      "shape": [
        2816,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.working_memory_bus.o_proj.3.weight": {
      "shape": [
        2816,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.working_memory_bus.o_proj.4.weight": {
      "shape": [
        2816,
        1024
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.working_memory_bus.alpha": {
      "shape": [
        5,
        16
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.working_memory_bus.rel_bias": {
      "shape": [
        16,
        511
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.persistent_memory_bus.alpha": {
      "shape": [
        5,
        1
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.experience_in.weight": {
      "shape": [
        1024,
        8448
      ],
      "dtype": "bfloat16"
    },
    "latent_deliberation.reason_embed.weight": {
      "shape": [
        6,
        1024
      ],
      "dtype": "bfloat16"
    }
  },
  "shards": [
    {
      "file": "model-00001.safetensors",
      "bytes": 776579654,
      "sha256": "374380ce2183b488367e840c546c1f4c7d6251fd2740f8d19a89a32f257dbffb",
      "tensors": [
        "latent_deliberation.blocks.0.attn.k_norm.weight",
        "latent_deliberation.blocks.0.attn.k_proj.weight",
        "latent_deliberation.blocks.0.attn.o_proj.weight",
        "latent_deliberation.blocks.0.attn.q_norm.weight",
        "latent_deliberation.blocks.0.attn.q_proj.weight",
        "latent_deliberation.blocks.0.attn.v_proj.weight",
        "latent_deliberation.blocks.0.ff.down.weight",
        "latent_deliberation.blocks.0.ff.gate.weight",
        "latent_deliberation.blocks.0.ff.up.weight",
        "latent_deliberation.blocks.0.ff_norm.weight",
        "latent_deliberation.blocks.0.norm.weight",
        "latent_deliberation.blocks.1.attn.k_norm.weight",
        "latent_deliberation.blocks.1.attn.k_proj.weight",
        "latent_deliberation.blocks.1.attn.o_proj.weight",
        "latent_deliberation.blocks.1.attn.q_norm.weight",
        "latent_deliberation.blocks.1.attn.q_proj.weight",
        "latent_deliberation.blocks.1.attn.v_proj.weight",
        "latent_deliberation.blocks.1.ff.down.weight",
        "latent_deliberation.blocks.1.ff.gate.weight",
        "latent_deliberation.blocks.1.ff.up.weight",
        "latent_deliberation.blocks.1.ff_norm.weight",
        "latent_deliberation.blocks.1.norm.weight",
        "latent_deliberation.blocks.2.attn.k_norm.weight",
        "latent_deliberation.blocks.2.attn.k_proj.weight",
        "latent_deliberation.blocks.2.attn.o_proj.weight",
        "latent_deliberation.blocks.2.attn.q_norm.weight",
        "latent_deliberation.blocks.2.attn.q_proj.weight",
        "latent_deliberation.blocks.2.attn.v_proj.weight",
        "latent_deliberation.blocks.2.ff.down.weight",
        "latent_deliberation.blocks.2.ff.gate.weight",
        "latent_deliberation.blocks.2.ff.up.weight",
        "latent_deliberation.blocks.2.ff_norm.weight",
        "latent_deliberation.blocks.2.norm.weight",
        "latent_deliberation.blocks.3.attn.k_norm.weight",
        "latent_deliberation.blocks.3.attn.k_proj.weight",
        "latent_deliberation.blocks.3.attn.o_proj.weight",
        "latent_deliberation.blocks.3.attn.q_norm.weight",
        "latent_deliberation.blocks.3.attn.q_proj.weight",
        "latent_deliberation.blocks.3.attn.v_proj.weight",
        "latent_deliberation.blocks.3.ff.down.weight",
        "latent_deliberation.blocks.3.ff.gate.weight",
        "latent_deliberation.blocks.3.ff.up.weight",
        "latent_deliberation.blocks.3.ff_norm.weight",
        "latent_deliberation.blocks.3.norm.weight",
        "latent_deliberation.experience_in.weight",
        "latent_deliberation.output_norm.weight",
        "latent_deliberation.persistent_memory_bus.alpha",
        "latent_deliberation.reason_embed.weight",
        "latent_deliberation.trajectory.cell.a_log",
        "latent_deliberation.trajectory.cell.b_proj.weight",
        "latent_deliberation.trajectory.cell.dt_bias",
        "latent_deliberation.trajectory.cell.f_proj.down.weight",
        "latent_deliberation.trajectory.cell.f_proj.up.weight",
        "latent_deliberation.trajectory.cell.g_proj.down.weight",
        "latent_deliberation.trajectory.cell.g_proj.up.weight",
        "latent_deliberation.trajectory.cell.k_proj.weight",
        "latent_deliberation.trajectory.cell.o_proj.weight",
        "latent_deliberation.trajectory.cell.q_proj.weight",
        "latent_deliberation.trajectory.cell.v_proj.weight",
        "latent_deliberation.trajectory.cell.w_proj.weight",
        "latent_deliberation.trajectory.persistent.a_log",
        "latent_deliberation.trajectory.persistent.b_proj.weight",
        "latent_deliberation.trajectory.persistent.dt_bias",
        "latent_deliberation.trajectory.persistent.f_proj.down.weight",
        "latent_deliberation.trajectory.persistent.f_proj.up.weight",
        "latent_deliberation.trajectory.persistent.g_proj.down.weight",
        "latent_deliberation.trajectory.persistent.g_proj.up.weight",
        "latent_deliberation.trajectory.persistent.k_proj.weight",
        "latent_deliberation.trajectory.persistent.o_proj.weight",
        "latent_deliberation.trajectory.persistent.q_proj.weight",
        "latent_deliberation.trajectory.persistent.v_proj.weight",
        "latent_deliberation.trajectory.persistent.w_proj.weight",
        "latent_deliberation.trajectory.probe_embed.weight",
        "latent_deliberation.trajectory.row.a_log",
        "latent_deliberation.trajectory.row.b_proj.weight",
        "latent_deliberation.trajectory.row.dt_bias",
        "latent_deliberation.trajectory.row.f_proj.down.weight",
        "latent_deliberation.trajectory.row.f_proj.up.weight",
        "latent_deliberation.trajectory.row.g_proj.down.weight",
        "latent_deliberation.trajectory.row.g_proj.up.weight",
        "latent_deliberation.trajectory.row.k_proj.weight",
        "latent_deliberation.trajectory.row.o_proj.weight",
        "latent_deliberation.trajectory.row.q_proj.weight",
        "latent_deliberation.trajectory.row.v_proj.weight",
        "latent_deliberation.trajectory.row.w_proj.weight",
        "latent_deliberation.working_memory_bus.address_norm.weight",
        "latent_deliberation.working_memory_bus.alpha",
        "latent_deliberation.working_memory_bus.k_proj.weight",
        "latent_deliberation.working_memory_bus.memory_norm.weight",
        "latent_deliberation.working_memory_bus.o_proj.0.weight",
        "latent_deliberation.working_memory_bus.o_proj.1.weight",
        "latent_deliberation.working_memory_bus.o_proj.2.weight",
        "latent_deliberation.working_memory_bus.o_proj.3.weight",
        "latent_deliberation.working_memory_bus.o_proj.4.weight",
        "latent_deliberation.working_memory_bus.q_norm.weight",
        "latent_deliberation.working_memory_bus.q_proj.0.weight",
        "latent_deliberation.working_memory_bus.q_proj.1.weight",
        "latent_deliberation.working_memory_bus.q_proj.2.weight",
        "latent_deliberation.working_memory_bus.q_proj.3.weight",
        "latent_deliberation.working_memory_bus.q_proj.4.weight",
        "latent_deliberation.working_memory_bus.rel_bias",
        "latent_deliberation.working_memory_bus.v_proj.weight"
      ]
    },
    {
      "file": "model-00002.safetensors",
      "bytes": 1991115265,
      "sha256": "3b11844cc3f237a2b8008c1a7df7ffe2d6e78b4e4789af9e8dc756d619c72cf6",
      "tensors": [
        "model.decoder.embed_tokens.weight",
        "model.decoder.layers.0.experts.down_proj.linear.weight",
        "model.decoder.layers.0.experts.down_proj.lora_a",
        "model.decoder.layers.0.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00003.safetensors",
      "bytes": 1645281286,
      "sha256": "e4ca40905b0e8da647a20a735659268fce793827a64ccdf8de747b06fb604313",
      "tensors": [
        "model.decoder.layers.0.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.0.experts.gate_up_proj.lora_a",
        "model.decoder.layers.0.experts.gate_up_proj.lora_b",
        "model.decoder.layers.0.input_layernorm.weight",
        "model.decoder.layers.0.layer_scalar",
        "model.decoder.layers.0.mlp.down_proj.linear.weight",
        "model.decoder.layers.0.mlp.down_proj.lora_a",
        "model.decoder.layers.0.mlp.down_proj.lora_b",
        "model.decoder.layers.0.mlp.gate_proj.linear.weight",
        "model.decoder.layers.0.mlp.gate_proj.lora_a",
        "model.decoder.layers.0.mlp.gate_proj.lora_b",
        "model.decoder.layers.0.mlp.up_proj.linear.weight",
        "model.decoder.layers.0.mlp.up_proj.lora_a",
        "model.decoder.layers.0.mlp.up_proj.lora_b",
        "model.decoder.layers.0.post_attention_layernorm.weight",
        "model.decoder.layers.0.post_feedforward_layernorm.weight",
        "model.decoder.layers.0.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.0.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.0.pre_feedforward_layernorm.weight",
        "model.decoder.layers.0.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.0.router.per_expert_scale",
        "model.decoder.layers.0.router.proj.weight",
        "model.decoder.layers.0.router.scale",
        "model.decoder.layers.0.self_attn.k_norm.weight",
        "model.decoder.layers.0.self_attn.k_proj.linear.weight",
        "model.decoder.layers.0.self_attn.k_proj.lora_a",
        "model.decoder.layers.0.self_attn.k_proj.lora_b",
        "model.decoder.layers.0.self_attn.o_proj.linear.weight",
        "model.decoder.layers.0.self_attn.o_proj.lora_a",
        "model.decoder.layers.0.self_attn.o_proj.lora_b",
        "model.decoder.layers.0.self_attn.q_norm.weight",
        "model.decoder.layers.0.self_attn.q_proj.linear.weight",
        "model.decoder.layers.0.self_attn.q_proj.lora_a",
        "model.decoder.layers.0.self_attn.q_proj.lora_b",
        "model.decoder.layers.0.self_attn.v_proj.linear.weight",
        "model.decoder.layers.0.self_attn.v_proj.lora_a",
        "model.decoder.layers.0.self_attn.v_proj.lora_b",
        "model.decoder.layers.1.experts.down_proj.linear.weight",
        "model.decoder.layers.1.experts.down_proj.lora_a",
        "model.decoder.layers.1.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00004.safetensors",
      "bytes": 1645281307,
      "sha256": "8909471307dfe90a1b5d908f89d62ee9456754aa3fc1f4c5a1d36f2fb7b0ca6f",
      "tensors": [
        "model.decoder.layers.1.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.1.experts.gate_up_proj.lora_a",
        "model.decoder.layers.1.experts.gate_up_proj.lora_b",
        "model.decoder.layers.1.input_layernorm.weight",
        "model.decoder.layers.1.layer_scalar",
        "model.decoder.layers.1.mlp.down_proj.linear.weight",
        "model.decoder.layers.1.mlp.down_proj.lora_a",
        "model.decoder.layers.1.mlp.down_proj.lora_b",
        "model.decoder.layers.1.mlp.gate_proj.linear.weight",
        "model.decoder.layers.1.mlp.gate_proj.lora_a",
        "model.decoder.layers.1.mlp.gate_proj.lora_b",
        "model.decoder.layers.1.mlp.up_proj.linear.weight",
        "model.decoder.layers.1.mlp.up_proj.lora_a",
        "model.decoder.layers.1.mlp.up_proj.lora_b",
        "model.decoder.layers.1.post_attention_layernorm.weight",
        "model.decoder.layers.1.post_feedforward_layernorm.weight",
        "model.decoder.layers.1.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.1.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.1.pre_feedforward_layernorm.weight",
        "model.decoder.layers.1.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.1.router.per_expert_scale",
        "model.decoder.layers.1.router.proj.weight",
        "model.decoder.layers.1.router.scale",
        "model.decoder.layers.1.self_attn.k_norm.weight",
        "model.decoder.layers.1.self_attn.k_proj.linear.weight",
        "model.decoder.layers.1.self_attn.k_proj.lora_a",
        "model.decoder.layers.1.self_attn.k_proj.lora_b",
        "model.decoder.layers.1.self_attn.o_proj.linear.weight",
        "model.decoder.layers.1.self_attn.o_proj.lora_a",
        "model.decoder.layers.1.self_attn.o_proj.lora_b",
        "model.decoder.layers.1.self_attn.q_norm.weight",
        "model.decoder.layers.1.self_attn.q_proj.linear.weight",
        "model.decoder.layers.1.self_attn.q_proj.lora_a",
        "model.decoder.layers.1.self_attn.q_proj.lora_b",
        "model.decoder.layers.1.self_attn.v_proj.linear.weight",
        "model.decoder.layers.1.self_attn.v_proj.lora_a",
        "model.decoder.layers.1.self_attn.v_proj.lora_b",
        "model.decoder.layers.10.experts.down_proj.linear.weight",
        "model.decoder.layers.10.experts.down_proj.lora_a",
        "model.decoder.layers.10.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00005.safetensors",
      "bytes": 1645281376,
      "sha256": "f213f94cb031abe72a258c325145aeb6c0a6fecf9c6d17e46298207141d27cc7",
      "tensors": [
        "model.decoder.layers.10.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.10.experts.gate_up_proj.lora_a",
        "model.decoder.layers.10.experts.gate_up_proj.lora_b",
        "model.decoder.layers.10.input_layernorm.weight",
        "model.decoder.layers.10.layer_scalar",
        "model.decoder.layers.10.mlp.down_proj.linear.weight",
        "model.decoder.layers.10.mlp.down_proj.lora_a",
        "model.decoder.layers.10.mlp.down_proj.lora_b",
        "model.decoder.layers.10.mlp.gate_proj.linear.weight",
        "model.decoder.layers.10.mlp.gate_proj.lora_a",
        "model.decoder.layers.10.mlp.gate_proj.lora_b",
        "model.decoder.layers.10.mlp.up_proj.linear.weight",
        "model.decoder.layers.10.mlp.up_proj.lora_a",
        "model.decoder.layers.10.mlp.up_proj.lora_b",
        "model.decoder.layers.10.post_attention_layernorm.weight",
        "model.decoder.layers.10.post_feedforward_layernorm.weight",
        "model.decoder.layers.10.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.10.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.10.pre_feedforward_layernorm.weight",
        "model.decoder.layers.10.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.10.router.per_expert_scale",
        "model.decoder.layers.10.router.proj.weight",
        "model.decoder.layers.10.router.scale",
        "model.decoder.layers.10.self_attn.k_norm.weight",
        "model.decoder.layers.10.self_attn.k_proj.linear.weight",
        "model.decoder.layers.10.self_attn.k_proj.lora_a",
        "model.decoder.layers.10.self_attn.k_proj.lora_b",
        "model.decoder.layers.10.self_attn.o_proj.linear.weight",
        "model.decoder.layers.10.self_attn.o_proj.lora_a",
        "model.decoder.layers.10.self_attn.o_proj.lora_b",
        "model.decoder.layers.10.self_attn.q_norm.weight",
        "model.decoder.layers.10.self_attn.q_proj.linear.weight",
        "model.decoder.layers.10.self_attn.q_proj.lora_a",
        "model.decoder.layers.10.self_attn.q_proj.lora_b",
        "model.decoder.layers.10.self_attn.v_proj.linear.weight",
        "model.decoder.layers.10.self_attn.v_proj.lora_a",
        "model.decoder.layers.10.self_attn.v_proj.lora_b",
        "model.decoder.layers.11.experts.down_proj.linear.weight",
        "model.decoder.layers.11.experts.down_proj.lora_a",
        "model.decoder.layers.11.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00006.safetensors",
      "bytes": 1674191579,
      "sha256": "7d4997e1fdc157712d55572f1f427400bdb176888e6ad420bbb8567d6e50fbde",
      "tensors": [
        "model.decoder.layers.11.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.11.experts.gate_up_proj.lora_a",
        "model.decoder.layers.11.experts.gate_up_proj.lora_b",
        "model.decoder.layers.11.input_layernorm.weight",
        "model.decoder.layers.11.layer_scalar",
        "model.decoder.layers.11.mlp.down_proj.linear.weight",
        "model.decoder.layers.11.mlp.down_proj.lora_a",
        "model.decoder.layers.11.mlp.down_proj.lora_b",
        "model.decoder.layers.11.mlp.gate_proj.linear.weight",
        "model.decoder.layers.11.mlp.gate_proj.lora_a",
        "model.decoder.layers.11.mlp.gate_proj.lora_b",
        "model.decoder.layers.11.mlp.up_proj.linear.weight",
        "model.decoder.layers.11.mlp.up_proj.lora_a",
        "model.decoder.layers.11.mlp.up_proj.lora_b",
        "model.decoder.layers.11.post_attention_layernorm.weight",
        "model.decoder.layers.11.post_feedforward_layernorm.weight",
        "model.decoder.layers.11.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.11.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.11.pre_feedforward_layernorm.weight",
        "model.decoder.layers.11.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.11.router.per_expert_scale",
        "model.decoder.layers.11.router.proj.weight",
        "model.decoder.layers.11.router.scale",
        "model.decoder.layers.11.self_attn.k_norm.weight",
        "model.decoder.layers.11.self_attn.k_proj.linear.weight",
        "model.decoder.layers.11.self_attn.k_proj.lora_a",
        "model.decoder.layers.11.self_attn.k_proj.lora_b",
        "model.decoder.layers.11.self_attn.o_proj.linear.weight",
        "model.decoder.layers.11.self_attn.o_proj.lora_a",
        "model.decoder.layers.11.self_attn.o_proj.lora_b",
        "model.decoder.layers.11.self_attn.q_norm.weight",
        "model.decoder.layers.11.self_attn.q_proj.linear.weight",
        "model.decoder.layers.11.self_attn.q_proj.lora_a",
        "model.decoder.layers.11.self_attn.q_proj.lora_b",
        "model.decoder.layers.12.experts.down_proj.linear.weight",
        "model.decoder.layers.12.experts.down_proj.lora_a",
        "model.decoder.layers.12.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00007.safetensors",
      "bytes": 1645281438,
      "sha256": "ec97a3868a1d77d35ff51e38e718db99d353ef44e0d320136966ec437f3376da",
      "tensors": [
        "model.decoder.layers.12.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.12.experts.gate_up_proj.lora_a",
        "model.decoder.layers.12.experts.gate_up_proj.lora_b",
        "model.decoder.layers.12.input_layernorm.weight",
        "model.decoder.layers.12.layer_scalar",
        "model.decoder.layers.12.mlp.down_proj.linear.weight",
        "model.decoder.layers.12.mlp.down_proj.lora_a",
        "model.decoder.layers.12.mlp.down_proj.lora_b",
        "model.decoder.layers.12.mlp.gate_proj.linear.weight",
        "model.decoder.layers.12.mlp.gate_proj.lora_a",
        "model.decoder.layers.12.mlp.gate_proj.lora_b",
        "model.decoder.layers.12.mlp.up_proj.linear.weight",
        "model.decoder.layers.12.mlp.up_proj.lora_a",
        "model.decoder.layers.12.mlp.up_proj.lora_b",
        "model.decoder.layers.12.post_attention_layernorm.weight",
        "model.decoder.layers.12.post_feedforward_layernorm.weight",
        "model.decoder.layers.12.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.12.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.12.pre_feedforward_layernorm.weight",
        "model.decoder.layers.12.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.12.router.per_expert_scale",
        "model.decoder.layers.12.router.proj.weight",
        "model.decoder.layers.12.router.scale",
        "model.decoder.layers.12.self_attn.k_norm.weight",
        "model.decoder.layers.12.self_attn.k_proj.linear.weight",
        "model.decoder.layers.12.self_attn.k_proj.lora_a",
        "model.decoder.layers.12.self_attn.k_proj.lora_b",
        "model.decoder.layers.12.self_attn.o_proj.linear.weight",
        "model.decoder.layers.12.self_attn.o_proj.lora_a",
        "model.decoder.layers.12.self_attn.o_proj.lora_b",
        "model.decoder.layers.12.self_attn.q_norm.weight",
        "model.decoder.layers.12.self_attn.q_proj.linear.weight",
        "model.decoder.layers.12.self_attn.q_proj.lora_a",
        "model.decoder.layers.12.self_attn.q_proj.lora_b",
        "model.decoder.layers.12.self_attn.v_proj.linear.weight",
        "model.decoder.layers.12.self_attn.v_proj.lora_a",
        "model.decoder.layers.12.self_attn.v_proj.lora_b",
        "model.decoder.layers.13.experts.down_proj.linear.weight",
        "model.decoder.layers.13.experts.down_proj.lora_a",
        "model.decoder.layers.13.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00008.safetensors",
      "bytes": 1645281390,
      "sha256": "c8aeeb93591a35251e6669fcb56d10101db3714468b5f4d1eecb27ae69a7cb98",
      "tensors": [
        "model.decoder.layers.13.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.13.experts.gate_up_proj.lora_a",
        "model.decoder.layers.13.experts.gate_up_proj.lora_b",
        "model.decoder.layers.13.input_layernorm.weight",
        "model.decoder.layers.13.layer_scalar",
        "model.decoder.layers.13.mlp.down_proj.linear.weight",
        "model.decoder.layers.13.mlp.down_proj.lora_a",
        "model.decoder.layers.13.mlp.down_proj.lora_b",
        "model.decoder.layers.13.mlp.gate_proj.linear.weight",
        "model.decoder.layers.13.mlp.gate_proj.lora_a",
        "model.decoder.layers.13.mlp.gate_proj.lora_b",
        "model.decoder.layers.13.mlp.up_proj.linear.weight",
        "model.decoder.layers.13.mlp.up_proj.lora_a",
        "model.decoder.layers.13.mlp.up_proj.lora_b",
        "model.decoder.layers.13.post_attention_layernorm.weight",
        "model.decoder.layers.13.post_feedforward_layernorm.weight",
        "model.decoder.layers.13.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.13.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.13.pre_feedforward_layernorm.weight",
        "model.decoder.layers.13.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.13.router.per_expert_scale",
        "model.decoder.layers.13.router.proj.weight",
        "model.decoder.layers.13.router.scale",
        "model.decoder.layers.13.self_attn.k_norm.weight",
        "model.decoder.layers.13.self_attn.k_proj.linear.weight",
        "model.decoder.layers.13.self_attn.k_proj.lora_a",
        "model.decoder.layers.13.self_attn.k_proj.lora_b",
        "model.decoder.layers.13.self_attn.o_proj.linear.weight",
        "model.decoder.layers.13.self_attn.o_proj.lora_a",
        "model.decoder.layers.13.self_attn.o_proj.lora_b",
        "model.decoder.layers.13.self_attn.q_norm.weight",
        "model.decoder.layers.13.self_attn.q_proj.linear.weight",
        "model.decoder.layers.13.self_attn.q_proj.lora_a",
        "model.decoder.layers.13.self_attn.q_proj.lora_b",
        "model.decoder.layers.13.self_attn.v_proj.linear.weight",
        "model.decoder.layers.13.self_attn.v_proj.lora_a",
        "model.decoder.layers.13.self_attn.v_proj.lora_b",
        "model.decoder.layers.14.experts.down_proj.linear.weight",
        "model.decoder.layers.14.experts.down_proj.lora_a",
        "model.decoder.layers.14.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00009.safetensors",
      "bytes": 1645281404,
      "sha256": "43b686872a4680f4d021f2ddaf71a592601972645cc6f97136708a2b529dc4c2",
      "tensors": [
        "model.decoder.layers.14.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.14.experts.gate_up_proj.lora_a",
        "model.decoder.layers.14.experts.gate_up_proj.lora_b",
        "model.decoder.layers.14.input_layernorm.weight",
        "model.decoder.layers.14.layer_scalar",
        "model.decoder.layers.14.mlp.down_proj.linear.weight",
        "model.decoder.layers.14.mlp.down_proj.lora_a",
        "model.decoder.layers.14.mlp.down_proj.lora_b",
        "model.decoder.layers.14.mlp.gate_proj.linear.weight",
        "model.decoder.layers.14.mlp.gate_proj.lora_a",
        "model.decoder.layers.14.mlp.gate_proj.lora_b",
        "model.decoder.layers.14.mlp.up_proj.linear.weight",
        "model.decoder.layers.14.mlp.up_proj.lora_a",
        "model.decoder.layers.14.mlp.up_proj.lora_b",
        "model.decoder.layers.14.post_attention_layernorm.weight",
        "model.decoder.layers.14.post_feedforward_layernorm.weight",
        "model.decoder.layers.14.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.14.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.14.pre_feedforward_layernorm.weight",
        "model.decoder.layers.14.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.14.router.per_expert_scale",
        "model.decoder.layers.14.router.proj.weight",
        "model.decoder.layers.14.router.scale",
        "model.decoder.layers.14.self_attn.k_norm.weight",
        "model.decoder.layers.14.self_attn.k_proj.linear.weight",
        "model.decoder.layers.14.self_attn.k_proj.lora_a",
        "model.decoder.layers.14.self_attn.k_proj.lora_b",
        "model.decoder.layers.14.self_attn.o_proj.linear.weight",
        "model.decoder.layers.14.self_attn.o_proj.lora_a",
        "model.decoder.layers.14.self_attn.o_proj.lora_b",
        "model.decoder.layers.14.self_attn.q_norm.weight",
        "model.decoder.layers.14.self_attn.q_proj.linear.weight",
        "model.decoder.layers.14.self_attn.q_proj.lora_a",
        "model.decoder.layers.14.self_attn.q_proj.lora_b",
        "model.decoder.layers.14.self_attn.v_proj.linear.weight",
        "model.decoder.layers.14.self_attn.v_proj.lora_a",
        "model.decoder.layers.14.self_attn.v_proj.lora_b",
        "model.decoder.layers.15.experts.down_proj.linear.weight",
        "model.decoder.layers.15.experts.down_proj.lora_a",
        "model.decoder.layers.15.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00010.safetensors",
      "bytes": 1645281370,
      "sha256": "946aa5129578785476e9e241757b64a0f4ec5777c873b576b25bbdc67d2f5f08",
      "tensors": [
        "model.decoder.layers.15.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.15.experts.gate_up_proj.lora_a",
        "model.decoder.layers.15.experts.gate_up_proj.lora_b",
        "model.decoder.layers.15.input_layernorm.weight",
        "model.decoder.layers.15.layer_scalar",
        "model.decoder.layers.15.mlp.down_proj.linear.weight",
        "model.decoder.layers.15.mlp.down_proj.lora_a",
        "model.decoder.layers.15.mlp.down_proj.lora_b",
        "model.decoder.layers.15.mlp.gate_proj.linear.weight",
        "model.decoder.layers.15.mlp.gate_proj.lora_a",
        "model.decoder.layers.15.mlp.gate_proj.lora_b",
        "model.decoder.layers.15.mlp.up_proj.linear.weight",
        "model.decoder.layers.15.mlp.up_proj.lora_a",
        "model.decoder.layers.15.mlp.up_proj.lora_b",
        "model.decoder.layers.15.post_attention_layernorm.weight",
        "model.decoder.layers.15.post_feedforward_layernorm.weight",
        "model.decoder.layers.15.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.15.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.15.pre_feedforward_layernorm.weight",
        "model.decoder.layers.15.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.15.router.per_expert_scale",
        "model.decoder.layers.15.router.proj.weight",
        "model.decoder.layers.15.router.scale",
        "model.decoder.layers.15.self_attn.k_norm.weight",
        "model.decoder.layers.15.self_attn.k_proj.linear.weight",
        "model.decoder.layers.15.self_attn.k_proj.lora_a",
        "model.decoder.layers.15.self_attn.k_proj.lora_b",
        "model.decoder.layers.15.self_attn.o_proj.linear.weight",
        "model.decoder.layers.15.self_attn.o_proj.lora_a",
        "model.decoder.layers.15.self_attn.o_proj.lora_b",
        "model.decoder.layers.15.self_attn.q_norm.weight",
        "model.decoder.layers.15.self_attn.q_proj.linear.weight",
        "model.decoder.layers.15.self_attn.q_proj.lora_a",
        "model.decoder.layers.15.self_attn.q_proj.lora_b",
        "model.decoder.layers.15.self_attn.v_proj.linear.weight",
        "model.decoder.layers.15.self_attn.v_proj.lora_a",
        "model.decoder.layers.15.self_attn.v_proj.lora_b",
        "model.decoder.layers.16.experts.down_proj.linear.weight",
        "model.decoder.layers.16.experts.down_proj.lora_a",
        "model.decoder.layers.16.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00011.safetensors",
      "bytes": 1645281424,
      "sha256": "adb4df6d408098ec73a1ed3a6f4fbad0202fd93d4af150828996720341c141ec",
      "tensors": [
        "model.decoder.layers.16.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.16.experts.gate_up_proj.lora_a",
        "model.decoder.layers.16.experts.gate_up_proj.lora_b",
        "model.decoder.layers.16.input_layernorm.weight",
        "model.decoder.layers.16.layer_scalar",
        "model.decoder.layers.16.mlp.down_proj.linear.weight",
        "model.decoder.layers.16.mlp.down_proj.lora_a",
        "model.decoder.layers.16.mlp.down_proj.lora_b",
        "model.decoder.layers.16.mlp.gate_proj.linear.weight",
        "model.decoder.layers.16.mlp.gate_proj.lora_a",
        "model.decoder.layers.16.mlp.gate_proj.lora_b",
        "model.decoder.layers.16.mlp.up_proj.linear.weight",
        "model.decoder.layers.16.mlp.up_proj.lora_a",
        "model.decoder.layers.16.mlp.up_proj.lora_b",
        "model.decoder.layers.16.post_attention_layernorm.weight",
        "model.decoder.layers.16.post_feedforward_layernorm.weight",
        "model.decoder.layers.16.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.16.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.16.pre_feedforward_layernorm.weight",
        "model.decoder.layers.16.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.16.router.per_expert_scale",
        "model.decoder.layers.16.router.proj.weight",
        "model.decoder.layers.16.router.scale",
        "model.decoder.layers.16.self_attn.k_norm.weight",
        "model.decoder.layers.16.self_attn.k_proj.linear.weight",
        "model.decoder.layers.16.self_attn.k_proj.lora_a",
        "model.decoder.layers.16.self_attn.k_proj.lora_b",
        "model.decoder.layers.16.self_attn.o_proj.linear.weight",
        "model.decoder.layers.16.self_attn.o_proj.lora_a",
        "model.decoder.layers.16.self_attn.o_proj.lora_b",
        "model.decoder.layers.16.self_attn.q_norm.weight",
        "model.decoder.layers.16.self_attn.q_proj.linear.weight",
        "model.decoder.layers.16.self_attn.q_proj.lora_a",
        "model.decoder.layers.16.self_attn.q_proj.lora_b",
        "model.decoder.layers.16.self_attn.v_proj.linear.weight",
        "model.decoder.layers.16.self_attn.v_proj.lora_a",
        "model.decoder.layers.16.self_attn.v_proj.lora_b",
        "model.decoder.layers.17.experts.down_proj.linear.weight",
        "model.decoder.layers.17.experts.down_proj.lora_a",
        "model.decoder.layers.17.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00012.safetensors",
      "bytes": 1674191611,
      "sha256": "86c9845b281f09b894ae62b3dde3574cd779417a8906b52a471b601c6ae19890",
      "tensors": [
        "model.decoder.layers.17.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.17.experts.gate_up_proj.lora_a",
        "model.decoder.layers.17.experts.gate_up_proj.lora_b",
        "model.decoder.layers.17.input_layernorm.weight",
        "model.decoder.layers.17.layer_scalar",
        "model.decoder.layers.17.mlp.down_proj.linear.weight",
        "model.decoder.layers.17.mlp.down_proj.lora_a",
        "model.decoder.layers.17.mlp.down_proj.lora_b",
        "model.decoder.layers.17.mlp.gate_proj.linear.weight",
        "model.decoder.layers.17.mlp.gate_proj.lora_a",
        "model.decoder.layers.17.mlp.gate_proj.lora_b",
        "model.decoder.layers.17.mlp.up_proj.linear.weight",
        "model.decoder.layers.17.mlp.up_proj.lora_a",
        "model.decoder.layers.17.mlp.up_proj.lora_b",
        "model.decoder.layers.17.post_attention_layernorm.weight",
        "model.decoder.layers.17.post_feedforward_layernorm.weight",
        "model.decoder.layers.17.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.17.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.17.pre_feedforward_layernorm.weight",
        "model.decoder.layers.17.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.17.router.per_expert_scale",
        "model.decoder.layers.17.router.proj.weight",
        "model.decoder.layers.17.router.scale",
        "model.decoder.layers.17.self_attn.k_norm.weight",
        "model.decoder.layers.17.self_attn.k_proj.linear.weight",
        "model.decoder.layers.17.self_attn.k_proj.lora_a",
        "model.decoder.layers.17.self_attn.k_proj.lora_b",
        "model.decoder.layers.17.self_attn.o_proj.linear.weight",
        "model.decoder.layers.17.self_attn.o_proj.lora_a",
        "model.decoder.layers.17.self_attn.o_proj.lora_b",
        "model.decoder.layers.17.self_attn.q_norm.weight",
        "model.decoder.layers.17.self_attn.q_proj.linear.weight",
        "model.decoder.layers.17.self_attn.q_proj.lora_a",
        "model.decoder.layers.17.self_attn.q_proj.lora_b",
        "model.decoder.layers.18.experts.down_proj.linear.weight",
        "model.decoder.layers.18.experts.down_proj.lora_a",
        "model.decoder.layers.18.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00013.safetensors",
      "bytes": 1645281386,
      "sha256": "b2042c1d4bdb49b8a4ed0423a86c4694f2c0cf345e6d247022bc919b818db9e4",
      "tensors": [
        "model.decoder.layers.18.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.18.experts.gate_up_proj.lora_a",
        "model.decoder.layers.18.experts.gate_up_proj.lora_b",
        "model.decoder.layers.18.input_layernorm.weight",
        "model.decoder.layers.18.layer_scalar",
        "model.decoder.layers.18.mlp.down_proj.linear.weight",
        "model.decoder.layers.18.mlp.down_proj.lora_a",
        "model.decoder.layers.18.mlp.down_proj.lora_b",
        "model.decoder.layers.18.mlp.gate_proj.linear.weight",
        "model.decoder.layers.18.mlp.gate_proj.lora_a",
        "model.decoder.layers.18.mlp.gate_proj.lora_b",
        "model.decoder.layers.18.mlp.up_proj.linear.weight",
        "model.decoder.layers.18.mlp.up_proj.lora_a",
        "model.decoder.layers.18.mlp.up_proj.lora_b",
        "model.decoder.layers.18.post_attention_layernorm.weight",
        "model.decoder.layers.18.post_feedforward_layernorm.weight",
        "model.decoder.layers.18.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.18.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.18.pre_feedforward_layernorm.weight",
        "model.decoder.layers.18.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.18.router.per_expert_scale",
        "model.decoder.layers.18.router.proj.weight",
        "model.decoder.layers.18.router.scale",
        "model.decoder.layers.18.self_attn.k_norm.weight",
        "model.decoder.layers.18.self_attn.k_proj.linear.weight",
        "model.decoder.layers.18.self_attn.k_proj.lora_a",
        "model.decoder.layers.18.self_attn.k_proj.lora_b",
        "model.decoder.layers.18.self_attn.o_proj.linear.weight",
        "model.decoder.layers.18.self_attn.o_proj.lora_a",
        "model.decoder.layers.18.self_attn.o_proj.lora_b",
        "model.decoder.layers.18.self_attn.q_norm.weight",
        "model.decoder.layers.18.self_attn.q_proj.linear.weight",
        "model.decoder.layers.18.self_attn.q_proj.lora_a",
        "model.decoder.layers.18.self_attn.q_proj.lora_b",
        "model.decoder.layers.18.self_attn.v_proj.linear.weight",
        "model.decoder.layers.18.self_attn.v_proj.lora_a",
        "model.decoder.layers.18.self_attn.v_proj.lora_b",
        "model.decoder.layers.19.experts.down_proj.linear.weight",
        "model.decoder.layers.19.experts.down_proj.lora_a",
        "model.decoder.layers.19.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00014.safetensors",
      "bytes": 1645281393,
      "sha256": "0233e8d5a91a79edd030479e287621af3bd962f080d7b1717dd03ab6fb417c97",
      "tensors": [
        "model.decoder.layers.19.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.19.experts.gate_up_proj.lora_a",
        "model.decoder.layers.19.experts.gate_up_proj.lora_b",
        "model.decoder.layers.19.input_layernorm.weight",
        "model.decoder.layers.19.layer_scalar",
        "model.decoder.layers.19.mlp.down_proj.linear.weight",
        "model.decoder.layers.19.mlp.down_proj.lora_a",
        "model.decoder.layers.19.mlp.down_proj.lora_b",
        "model.decoder.layers.19.mlp.gate_proj.linear.weight",
        "model.decoder.layers.19.mlp.gate_proj.lora_a",
        "model.decoder.layers.19.mlp.gate_proj.lora_b",
        "model.decoder.layers.19.mlp.up_proj.linear.weight",
        "model.decoder.layers.19.mlp.up_proj.lora_a",
        "model.decoder.layers.19.mlp.up_proj.lora_b",
        "model.decoder.layers.19.post_attention_layernorm.weight",
        "model.decoder.layers.19.post_feedforward_layernorm.weight",
        "model.decoder.layers.19.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.19.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.19.pre_feedforward_layernorm.weight",
        "model.decoder.layers.19.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.19.router.per_expert_scale",
        "model.decoder.layers.19.router.proj.weight",
        "model.decoder.layers.19.router.scale",
        "model.decoder.layers.19.self_attn.k_norm.weight",
        "model.decoder.layers.19.self_attn.k_proj.linear.weight",
        "model.decoder.layers.19.self_attn.k_proj.lora_a",
        "model.decoder.layers.19.self_attn.k_proj.lora_b",
        "model.decoder.layers.19.self_attn.o_proj.linear.weight",
        "model.decoder.layers.19.self_attn.o_proj.lora_a",
        "model.decoder.layers.19.self_attn.o_proj.lora_b",
        "model.decoder.layers.19.self_attn.q_norm.weight",
        "model.decoder.layers.19.self_attn.q_proj.linear.weight",
        "model.decoder.layers.19.self_attn.q_proj.lora_a",
        "model.decoder.layers.19.self_attn.q_proj.lora_b",
        "model.decoder.layers.19.self_attn.v_proj.linear.weight",
        "model.decoder.layers.19.self_attn.v_proj.lora_a",
        "model.decoder.layers.19.self_attn.v_proj.lora_b",
        "model.decoder.layers.2.experts.down_proj.linear.weight",
        "model.decoder.layers.2.experts.down_proj.lora_a",
        "model.decoder.layers.2.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00015.safetensors",
      "bytes": 1645281363,
      "sha256": "75bab4540f1aa39d31271ec3d36e414e24fa500d3bce0dc5a00da0c1a8a25582",
      "tensors": [
        "model.decoder.layers.2.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.2.experts.gate_up_proj.lora_a",
        "model.decoder.layers.2.experts.gate_up_proj.lora_b",
        "model.decoder.layers.2.input_layernorm.weight",
        "model.decoder.layers.2.layer_scalar",
        "model.decoder.layers.2.mlp.down_proj.linear.weight",
        "model.decoder.layers.2.mlp.down_proj.lora_a",
        "model.decoder.layers.2.mlp.down_proj.lora_b",
        "model.decoder.layers.2.mlp.gate_proj.linear.weight",
        "model.decoder.layers.2.mlp.gate_proj.lora_a",
        "model.decoder.layers.2.mlp.gate_proj.lora_b",
        "model.decoder.layers.2.mlp.up_proj.linear.weight",
        "model.decoder.layers.2.mlp.up_proj.lora_a",
        "model.decoder.layers.2.mlp.up_proj.lora_b",
        "model.decoder.layers.2.post_attention_layernorm.weight",
        "model.decoder.layers.2.post_feedforward_layernorm.weight",
        "model.decoder.layers.2.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.2.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.2.pre_feedforward_layernorm.weight",
        "model.decoder.layers.2.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.2.router.per_expert_scale",
        "model.decoder.layers.2.router.proj.weight",
        "model.decoder.layers.2.router.scale",
        "model.decoder.layers.2.self_attn.k_norm.weight",
        "model.decoder.layers.2.self_attn.k_proj.linear.weight",
        "model.decoder.layers.2.self_attn.k_proj.lora_a",
        "model.decoder.layers.2.self_attn.k_proj.lora_b",
        "model.decoder.layers.2.self_attn.o_proj.linear.weight",
        "model.decoder.layers.2.self_attn.o_proj.lora_a",
        "model.decoder.layers.2.self_attn.o_proj.lora_b",
        "model.decoder.layers.2.self_attn.q_norm.weight",
        "model.decoder.layers.2.self_attn.q_proj.linear.weight",
        "model.decoder.layers.2.self_attn.q_proj.lora_a",
        "model.decoder.layers.2.self_attn.q_proj.lora_b",
        "model.decoder.layers.2.self_attn.v_proj.linear.weight",
        "model.decoder.layers.2.self_attn.v_proj.lora_a",
        "model.decoder.layers.2.self_attn.v_proj.lora_b",
        "model.decoder.layers.20.experts.down_proj.linear.weight",
        "model.decoder.layers.20.experts.down_proj.lora_a",
        "model.decoder.layers.20.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00016.safetensors",
      "bytes": 1645281392,
      "sha256": "37e7c3bbf2fc81bd14f95dd5e14d49e7e6cda481b8478d64562a8be0d2c989fa",
      "tensors": [
        "model.decoder.layers.20.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.20.experts.gate_up_proj.lora_a",
        "model.decoder.layers.20.experts.gate_up_proj.lora_b",
        "model.decoder.layers.20.input_layernorm.weight",
        "model.decoder.layers.20.layer_scalar",
        "model.decoder.layers.20.mlp.down_proj.linear.weight",
        "model.decoder.layers.20.mlp.down_proj.lora_a",
        "model.decoder.layers.20.mlp.down_proj.lora_b",
        "model.decoder.layers.20.mlp.gate_proj.linear.weight",
        "model.decoder.layers.20.mlp.gate_proj.lora_a",
        "model.decoder.layers.20.mlp.gate_proj.lora_b",
        "model.decoder.layers.20.mlp.up_proj.linear.weight",
        "model.decoder.layers.20.mlp.up_proj.lora_a",
        "model.decoder.layers.20.mlp.up_proj.lora_b",
        "model.decoder.layers.20.post_attention_layernorm.weight",
        "model.decoder.layers.20.post_feedforward_layernorm.weight",
        "model.decoder.layers.20.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.20.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.20.pre_feedforward_layernorm.weight",
        "model.decoder.layers.20.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.20.router.per_expert_scale",
        "model.decoder.layers.20.router.proj.weight",
        "model.decoder.layers.20.router.scale",
        "model.decoder.layers.20.self_attn.k_norm.weight",
        "model.decoder.layers.20.self_attn.k_proj.linear.weight",
        "model.decoder.layers.20.self_attn.k_proj.lora_a",
        "model.decoder.layers.20.self_attn.k_proj.lora_b",
        "model.decoder.layers.20.self_attn.o_proj.linear.weight",
        "model.decoder.layers.20.self_attn.o_proj.lora_a",
        "model.decoder.layers.20.self_attn.o_proj.lora_b",
        "model.decoder.layers.20.self_attn.q_norm.weight",
        "model.decoder.layers.20.self_attn.q_proj.linear.weight",
        "model.decoder.layers.20.self_attn.q_proj.lora_a",
        "model.decoder.layers.20.self_attn.q_proj.lora_b",
        "model.decoder.layers.20.self_attn.v_proj.linear.weight",
        "model.decoder.layers.20.self_attn.v_proj.lora_a",
        "model.decoder.layers.20.self_attn.v_proj.lora_b",
        "model.decoder.layers.21.experts.down_proj.linear.weight",
        "model.decoder.layers.21.experts.down_proj.lora_a",
        "model.decoder.layers.21.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00017.safetensors",
      "bytes": 1645281346,
      "sha256": "82dc6c75a7ca3a36ddb16733beead912507fcb5fbc236d06b9c249b9eb22ac09",
      "tensors": [
        "model.decoder.layers.21.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.21.experts.gate_up_proj.lora_a",
        "model.decoder.layers.21.experts.gate_up_proj.lora_b",
        "model.decoder.layers.21.input_layernorm.weight",
        "model.decoder.layers.21.layer_scalar",
        "model.decoder.layers.21.mlp.down_proj.linear.weight",
        "model.decoder.layers.21.mlp.down_proj.lora_a",
        "model.decoder.layers.21.mlp.down_proj.lora_b",
        "model.decoder.layers.21.mlp.gate_proj.linear.weight",
        "model.decoder.layers.21.mlp.gate_proj.lora_a",
        "model.decoder.layers.21.mlp.gate_proj.lora_b",
        "model.decoder.layers.21.mlp.up_proj.linear.weight",
        "model.decoder.layers.21.mlp.up_proj.lora_a",
        "model.decoder.layers.21.mlp.up_proj.lora_b",
        "model.decoder.layers.21.post_attention_layernorm.weight",
        "model.decoder.layers.21.post_feedforward_layernorm.weight",
        "model.decoder.layers.21.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.21.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.21.pre_feedforward_layernorm.weight",
        "model.decoder.layers.21.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.21.router.per_expert_scale",
        "model.decoder.layers.21.router.proj.weight",
        "model.decoder.layers.21.router.scale",
        "model.decoder.layers.21.self_attn.k_norm.weight",
        "model.decoder.layers.21.self_attn.k_proj.linear.weight",
        "model.decoder.layers.21.self_attn.k_proj.lora_a",
        "model.decoder.layers.21.self_attn.k_proj.lora_b",
        "model.decoder.layers.21.self_attn.o_proj.linear.weight",
        "model.decoder.layers.21.self_attn.o_proj.lora_a",
        "model.decoder.layers.21.self_attn.o_proj.lora_b",
        "model.decoder.layers.21.self_attn.q_norm.weight",
        "model.decoder.layers.21.self_attn.q_proj.linear.weight",
        "model.decoder.layers.21.self_attn.q_proj.lora_a",
        "model.decoder.layers.21.self_attn.q_proj.lora_b",
        "model.decoder.layers.21.self_attn.v_proj.linear.weight",
        "model.decoder.layers.21.self_attn.v_proj.lora_a",
        "model.decoder.layers.21.self_attn.v_proj.lora_b",
        "model.decoder.layers.22.experts.down_proj.linear.weight",
        "model.decoder.layers.22.experts.down_proj.lora_a",
        "model.decoder.layers.22.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00018.safetensors",
      "bytes": 1645281342,
      "sha256": "b49d0e174f0d8422d737d7023c283a555f359c994a07e593aae5d07eb291c0ae",
      "tensors": [
        "model.decoder.layers.22.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.22.experts.gate_up_proj.lora_a",
        "model.decoder.layers.22.experts.gate_up_proj.lora_b",
        "model.decoder.layers.22.input_layernorm.weight",
        "model.decoder.layers.22.layer_scalar",
        "model.decoder.layers.22.mlp.down_proj.linear.weight",
        "model.decoder.layers.22.mlp.down_proj.lora_a",
        "model.decoder.layers.22.mlp.down_proj.lora_b",
        "model.decoder.layers.22.mlp.gate_proj.linear.weight",
        "model.decoder.layers.22.mlp.gate_proj.lora_a",
        "model.decoder.layers.22.mlp.gate_proj.lora_b",
        "model.decoder.layers.22.mlp.up_proj.linear.weight",
        "model.decoder.layers.22.mlp.up_proj.lora_a",
        "model.decoder.layers.22.mlp.up_proj.lora_b",
        "model.decoder.layers.22.post_attention_layernorm.weight",
        "model.decoder.layers.22.post_feedforward_layernorm.weight",
        "model.decoder.layers.22.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.22.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.22.pre_feedforward_layernorm.weight",
        "model.decoder.layers.22.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.22.router.per_expert_scale",
        "model.decoder.layers.22.router.proj.weight",
        "model.decoder.layers.22.router.scale",
        "model.decoder.layers.22.self_attn.k_norm.weight",
        "model.decoder.layers.22.self_attn.k_proj.linear.weight",
        "model.decoder.layers.22.self_attn.k_proj.lora_a",
        "model.decoder.layers.22.self_attn.k_proj.lora_b",
        "model.decoder.layers.22.self_attn.o_proj.linear.weight",
        "model.decoder.layers.22.self_attn.o_proj.lora_a",
        "model.decoder.layers.22.self_attn.o_proj.lora_b",
        "model.decoder.layers.22.self_attn.q_norm.weight",
        "model.decoder.layers.22.self_attn.q_proj.linear.weight",
        "model.decoder.layers.22.self_attn.q_proj.lora_a",
        "model.decoder.layers.22.self_attn.q_proj.lora_b",
        "model.decoder.layers.22.self_attn.v_proj.linear.weight",
        "model.decoder.layers.22.self_attn.v_proj.lora_a",
        "model.decoder.layers.22.self_attn.v_proj.lora_b",
        "model.decoder.layers.23.experts.down_proj.linear.weight",
        "model.decoder.layers.23.experts.down_proj.lora_a",
        "model.decoder.layers.23.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00019.safetensors",
      "bytes": 1674191627,
      "sha256": "e8e66177d4ad899a85ca712eb15fce721a7db756121ce3123d13b16eb3c0b0a8",
      "tensors": [
        "model.decoder.layers.23.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.23.experts.gate_up_proj.lora_a",
        "model.decoder.layers.23.experts.gate_up_proj.lora_b",
        "model.decoder.layers.23.input_layernorm.weight",
        "model.decoder.layers.23.layer_scalar",
        "model.decoder.layers.23.mlp.down_proj.linear.weight",
        "model.decoder.layers.23.mlp.down_proj.lora_a",
        "model.decoder.layers.23.mlp.down_proj.lora_b",
        "model.decoder.layers.23.mlp.gate_proj.linear.weight",
        "model.decoder.layers.23.mlp.gate_proj.lora_a",
        "model.decoder.layers.23.mlp.gate_proj.lora_b",
        "model.decoder.layers.23.mlp.up_proj.linear.weight",
        "model.decoder.layers.23.mlp.up_proj.lora_a",
        "model.decoder.layers.23.mlp.up_proj.lora_b",
        "model.decoder.layers.23.post_attention_layernorm.weight",
        "model.decoder.layers.23.post_feedforward_layernorm.weight",
        "model.decoder.layers.23.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.23.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.23.pre_feedforward_layernorm.weight",
        "model.decoder.layers.23.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.23.router.per_expert_scale",
        "model.decoder.layers.23.router.proj.weight",
        "model.decoder.layers.23.router.scale",
        "model.decoder.layers.23.self_attn.k_norm.weight",
        "model.decoder.layers.23.self_attn.k_proj.linear.weight",
        "model.decoder.layers.23.self_attn.k_proj.lora_a",
        "model.decoder.layers.23.self_attn.k_proj.lora_b",
        "model.decoder.layers.23.self_attn.o_proj.linear.weight",
        "model.decoder.layers.23.self_attn.o_proj.lora_a",
        "model.decoder.layers.23.self_attn.o_proj.lora_b",
        "model.decoder.layers.23.self_attn.q_norm.weight",
        "model.decoder.layers.23.self_attn.q_proj.linear.weight",
        "model.decoder.layers.23.self_attn.q_proj.lora_a",
        "model.decoder.layers.23.self_attn.q_proj.lora_b",
        "model.decoder.layers.24.experts.down_proj.linear.weight",
        "model.decoder.layers.24.experts.down_proj.lora_a",
        "model.decoder.layers.24.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00020.safetensors",
      "bytes": 1645281442,
      "sha256": "d87e3d52c22cbec2e5681d7c048ddbe5109df7f8c7af99c713c00dfe351b2ae6",
      "tensors": [
        "model.decoder.layers.24.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.24.experts.gate_up_proj.lora_a",
        "model.decoder.layers.24.experts.gate_up_proj.lora_b",
        "model.decoder.layers.24.input_layernorm.weight",
        "model.decoder.layers.24.layer_scalar",
        "model.decoder.layers.24.mlp.down_proj.linear.weight",
        "model.decoder.layers.24.mlp.down_proj.lora_a",
        "model.decoder.layers.24.mlp.down_proj.lora_b",
        "model.decoder.layers.24.mlp.gate_proj.linear.weight",
        "model.decoder.layers.24.mlp.gate_proj.lora_a",
        "model.decoder.layers.24.mlp.gate_proj.lora_b",
        "model.decoder.layers.24.mlp.up_proj.linear.weight",
        "model.decoder.layers.24.mlp.up_proj.lora_a",
        "model.decoder.layers.24.mlp.up_proj.lora_b",
        "model.decoder.layers.24.post_attention_layernorm.weight",
        "model.decoder.layers.24.post_feedforward_layernorm.weight",
        "model.decoder.layers.24.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.24.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.24.pre_feedforward_layernorm.weight",
        "model.decoder.layers.24.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.24.router.per_expert_scale",
        "model.decoder.layers.24.router.proj.weight",
        "model.decoder.layers.24.router.scale",
        "model.decoder.layers.24.self_attn.k_norm.weight",
        "model.decoder.layers.24.self_attn.k_proj.linear.weight",
        "model.decoder.layers.24.self_attn.k_proj.lora_a",
        "model.decoder.layers.24.self_attn.k_proj.lora_b",
        "model.decoder.layers.24.self_attn.o_proj.linear.weight",
        "model.decoder.layers.24.self_attn.o_proj.lora_a",
        "model.decoder.layers.24.self_attn.o_proj.lora_b",
        "model.decoder.layers.24.self_attn.q_norm.weight",
        "model.decoder.layers.24.self_attn.q_proj.linear.weight",
        "model.decoder.layers.24.self_attn.q_proj.lora_a",
        "model.decoder.layers.24.self_attn.q_proj.lora_b",
        "model.decoder.layers.24.self_attn.v_proj.linear.weight",
        "model.decoder.layers.24.self_attn.v_proj.lora_a",
        "model.decoder.layers.24.self_attn.v_proj.lora_b",
        "model.decoder.layers.25.experts.down_proj.linear.weight",
        "model.decoder.layers.25.experts.down_proj.lora_a",
        "model.decoder.layers.25.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00021.safetensors",
      "bytes": 1645281412,
      "sha256": "44aecb5538ee528dcc90f49733268c67795a44a71b0fece8eb1b51b84cca59ea",
      "tensors": [
        "model.decoder.layers.25.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.25.experts.gate_up_proj.lora_a",
        "model.decoder.layers.25.experts.gate_up_proj.lora_b",
        "model.decoder.layers.25.input_layernorm.weight",
        "model.decoder.layers.25.layer_scalar",
        "model.decoder.layers.25.mlp.down_proj.linear.weight",
        "model.decoder.layers.25.mlp.down_proj.lora_a",
        "model.decoder.layers.25.mlp.down_proj.lora_b",
        "model.decoder.layers.25.mlp.gate_proj.linear.weight",
        "model.decoder.layers.25.mlp.gate_proj.lora_a",
        "model.decoder.layers.25.mlp.gate_proj.lora_b",
        "model.decoder.layers.25.mlp.up_proj.linear.weight",
        "model.decoder.layers.25.mlp.up_proj.lora_a",
        "model.decoder.layers.25.mlp.up_proj.lora_b",
        "model.decoder.layers.25.post_attention_layernorm.weight",
        "model.decoder.layers.25.post_feedforward_layernorm.weight",
        "model.decoder.layers.25.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.25.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.25.pre_feedforward_layernorm.weight",
        "model.decoder.layers.25.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.25.router.per_expert_scale",
        "model.decoder.layers.25.router.proj.weight",
        "model.decoder.layers.25.router.scale",
        "model.decoder.layers.25.self_attn.k_norm.weight",
        "model.decoder.layers.25.self_attn.k_proj.linear.weight",
        "model.decoder.layers.25.self_attn.k_proj.lora_a",
        "model.decoder.layers.25.self_attn.k_proj.lora_b",
        "model.decoder.layers.25.self_attn.o_proj.linear.weight",
        "model.decoder.layers.25.self_attn.o_proj.lora_a",
        "model.decoder.layers.25.self_attn.o_proj.lora_b",
        "model.decoder.layers.25.self_attn.q_norm.weight",
        "model.decoder.layers.25.self_attn.q_proj.linear.weight",
        "model.decoder.layers.25.self_attn.q_proj.lora_a",
        "model.decoder.layers.25.self_attn.q_proj.lora_b",
        "model.decoder.layers.25.self_attn.v_proj.linear.weight",
        "model.decoder.layers.25.self_attn.v_proj.lora_a",
        "model.decoder.layers.25.self_attn.v_proj.lora_b",
        "model.decoder.layers.26.experts.down_proj.linear.weight",
        "model.decoder.layers.26.experts.down_proj.lora_a",
        "model.decoder.layers.26.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00022.safetensors",
      "bytes": 1645281408,
      "sha256": "70f3124fee705d9cb5221936c59f39820ab9c37a2c70bad89176e309e1c1ca07",
      "tensors": [
        "model.decoder.layers.26.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.26.experts.gate_up_proj.lora_a",
        "model.decoder.layers.26.experts.gate_up_proj.lora_b",
        "model.decoder.layers.26.input_layernorm.weight",
        "model.decoder.layers.26.layer_scalar",
        "model.decoder.layers.26.mlp.down_proj.linear.weight",
        "model.decoder.layers.26.mlp.down_proj.lora_a",
        "model.decoder.layers.26.mlp.down_proj.lora_b",
        "model.decoder.layers.26.mlp.gate_proj.linear.weight",
        "model.decoder.layers.26.mlp.gate_proj.lora_a",
        "model.decoder.layers.26.mlp.gate_proj.lora_b",
        "model.decoder.layers.26.mlp.up_proj.linear.weight",
        "model.decoder.layers.26.mlp.up_proj.lora_a",
        "model.decoder.layers.26.mlp.up_proj.lora_b",
        "model.decoder.layers.26.post_attention_layernorm.weight",
        "model.decoder.layers.26.post_feedforward_layernorm.weight",
        "model.decoder.layers.26.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.26.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.26.pre_feedforward_layernorm.weight",
        "model.decoder.layers.26.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.26.router.per_expert_scale",
        "model.decoder.layers.26.router.proj.weight",
        "model.decoder.layers.26.router.scale",
        "model.decoder.layers.26.self_attn.k_norm.weight",
        "model.decoder.layers.26.self_attn.k_proj.linear.weight",
        "model.decoder.layers.26.self_attn.k_proj.lora_a",
        "model.decoder.layers.26.self_attn.k_proj.lora_b",
        "model.decoder.layers.26.self_attn.o_proj.linear.weight",
        "model.decoder.layers.26.self_attn.o_proj.lora_a",
        "model.decoder.layers.26.self_attn.o_proj.lora_b",
        "model.decoder.layers.26.self_attn.q_norm.weight",
        "model.decoder.layers.26.self_attn.q_proj.linear.weight",
        "model.decoder.layers.26.self_attn.q_proj.lora_a",
        "model.decoder.layers.26.self_attn.q_proj.lora_b",
        "model.decoder.layers.26.self_attn.v_proj.linear.weight",
        "model.decoder.layers.26.self_attn.v_proj.lora_a",
        "model.decoder.layers.26.self_attn.v_proj.lora_b",
        "model.decoder.layers.27.experts.down_proj.linear.weight",
        "model.decoder.layers.27.experts.down_proj.lora_a",
        "model.decoder.layers.27.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00023.safetensors",
      "bytes": 1645281368,
      "sha256": "499f422a0073dce797799000ab7ea6df9e77f845aad9cff999ac9d404a47b8a0",
      "tensors": [
        "model.decoder.layers.27.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.27.experts.gate_up_proj.lora_a",
        "model.decoder.layers.27.experts.gate_up_proj.lora_b",
        "model.decoder.layers.27.input_layernorm.weight",
        "model.decoder.layers.27.layer_scalar",
        "model.decoder.layers.27.mlp.down_proj.linear.weight",
        "model.decoder.layers.27.mlp.down_proj.lora_a",
        "model.decoder.layers.27.mlp.down_proj.lora_b",
        "model.decoder.layers.27.mlp.gate_proj.linear.weight",
        "model.decoder.layers.27.mlp.gate_proj.lora_a",
        "model.decoder.layers.27.mlp.gate_proj.lora_b",
        "model.decoder.layers.27.mlp.up_proj.linear.weight",
        "model.decoder.layers.27.mlp.up_proj.lora_a",
        "model.decoder.layers.27.mlp.up_proj.lora_b",
        "model.decoder.layers.27.post_attention_layernorm.weight",
        "model.decoder.layers.27.post_feedforward_layernorm.weight",
        "model.decoder.layers.27.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.27.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.27.pre_feedforward_layernorm.weight",
        "model.decoder.layers.27.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.27.router.per_expert_scale",
        "model.decoder.layers.27.router.proj.weight",
        "model.decoder.layers.27.router.scale",
        "model.decoder.layers.27.self_attn.k_norm.weight",
        "model.decoder.layers.27.self_attn.k_proj.linear.weight",
        "model.decoder.layers.27.self_attn.k_proj.lora_a",
        "model.decoder.layers.27.self_attn.k_proj.lora_b",
        "model.decoder.layers.27.self_attn.o_proj.linear.weight",
        "model.decoder.layers.27.self_attn.o_proj.lora_a",
        "model.decoder.layers.27.self_attn.o_proj.lora_b",
        "model.decoder.layers.27.self_attn.q_norm.weight",
        "model.decoder.layers.27.self_attn.q_proj.linear.weight",
        "model.decoder.layers.27.self_attn.q_proj.lora_a",
        "model.decoder.layers.27.self_attn.q_proj.lora_b",
        "model.decoder.layers.27.self_attn.v_proj.linear.weight",
        "model.decoder.layers.27.self_attn.v_proj.lora_a",
        "model.decoder.layers.27.self_attn.v_proj.lora_b",
        "model.decoder.layers.28.experts.down_proj.linear.weight",
        "model.decoder.layers.28.experts.down_proj.lora_a",
        "model.decoder.layers.28.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00024.safetensors",
      "bytes": 1645281350,
      "sha256": "e7cd9ffad7914610f2cf54be55ac46a5d2082d5faee7f3d7d2854fa46315d46e",
      "tensors": [
        "model.decoder.layers.28.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.28.experts.gate_up_proj.lora_a",
        "model.decoder.layers.28.experts.gate_up_proj.lora_b",
        "model.decoder.layers.28.input_layernorm.weight",
        "model.decoder.layers.28.layer_scalar",
        "model.decoder.layers.28.mlp.down_proj.linear.weight",
        "model.decoder.layers.28.mlp.down_proj.lora_a",
        "model.decoder.layers.28.mlp.down_proj.lora_b",
        "model.decoder.layers.28.mlp.gate_proj.linear.weight",
        "model.decoder.layers.28.mlp.gate_proj.lora_a",
        "model.decoder.layers.28.mlp.gate_proj.lora_b",
        "model.decoder.layers.28.mlp.up_proj.linear.weight",
        "model.decoder.layers.28.mlp.up_proj.lora_a",
        "model.decoder.layers.28.mlp.up_proj.lora_b",
        "model.decoder.layers.28.post_attention_layernorm.weight",
        "model.decoder.layers.28.post_feedforward_layernorm.weight",
        "model.decoder.layers.28.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.28.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.28.pre_feedforward_layernorm.weight",
        "model.decoder.layers.28.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.28.router.per_expert_scale",
        "model.decoder.layers.28.router.proj.weight",
        "model.decoder.layers.28.router.scale",
        "model.decoder.layers.28.self_attn.k_norm.weight",
        "model.decoder.layers.28.self_attn.k_proj.linear.weight",
        "model.decoder.layers.28.self_attn.k_proj.lora_a",
        "model.decoder.layers.28.self_attn.k_proj.lora_b",
        "model.decoder.layers.28.self_attn.o_proj.linear.weight",
        "model.decoder.layers.28.self_attn.o_proj.lora_a",
        "model.decoder.layers.28.self_attn.o_proj.lora_b",
        "model.decoder.layers.28.self_attn.q_norm.weight",
        "model.decoder.layers.28.self_attn.q_proj.linear.weight",
        "model.decoder.layers.28.self_attn.q_proj.lora_a",
        "model.decoder.layers.28.self_attn.q_proj.lora_b",
        "model.decoder.layers.28.self_attn.v_proj.linear.weight",
        "model.decoder.layers.28.self_attn.v_proj.lora_a",
        "model.decoder.layers.28.self_attn.v_proj.lora_b",
        "model.decoder.layers.29.experts.down_proj.linear.weight",
        "model.decoder.layers.29.experts.down_proj.lora_a",
        "model.decoder.layers.29.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00025.safetensors",
      "bytes": 1674191592,
      "sha256": "234e835f2a3abdcd98cdbba245f057948573dd480301c94c00921f6d5c91bd96",
      "tensors": [
        "model.decoder.layers.29.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.29.experts.gate_up_proj.lora_a",
        "model.decoder.layers.29.experts.gate_up_proj.lora_b",
        "model.decoder.layers.29.input_layernorm.weight",
        "model.decoder.layers.29.layer_scalar",
        "model.decoder.layers.29.mlp.down_proj.linear.weight",
        "model.decoder.layers.29.mlp.down_proj.lora_a",
        "model.decoder.layers.29.mlp.down_proj.lora_b",
        "model.decoder.layers.29.mlp.gate_proj.linear.weight",
        "model.decoder.layers.29.mlp.gate_proj.lora_a",
        "model.decoder.layers.29.mlp.gate_proj.lora_b",
        "model.decoder.layers.29.mlp.up_proj.linear.weight",
        "model.decoder.layers.29.mlp.up_proj.lora_a",
        "model.decoder.layers.29.mlp.up_proj.lora_b",
        "model.decoder.layers.29.post_attention_layernorm.weight",
        "model.decoder.layers.29.post_feedforward_layernorm.weight",
        "model.decoder.layers.29.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.29.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.29.pre_feedforward_layernorm.weight",
        "model.decoder.layers.29.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.29.router.per_expert_scale",
        "model.decoder.layers.29.router.proj.weight",
        "model.decoder.layers.29.router.scale",
        "model.decoder.layers.29.self_attn.k_norm.weight",
        "model.decoder.layers.29.self_attn.k_proj.linear.weight",
        "model.decoder.layers.29.self_attn.k_proj.lora_a",
        "model.decoder.layers.29.self_attn.k_proj.lora_b",
        "model.decoder.layers.29.self_attn.o_proj.linear.weight",
        "model.decoder.layers.29.self_attn.o_proj.lora_a",
        "model.decoder.layers.29.self_attn.o_proj.lora_b",
        "model.decoder.layers.29.self_attn.q_norm.weight",
        "model.decoder.layers.29.self_attn.q_proj.linear.weight",
        "model.decoder.layers.29.self_attn.q_proj.lora_a",
        "model.decoder.layers.29.self_attn.q_proj.lora_b",
        "model.decoder.layers.3.experts.down_proj.linear.weight",
        "model.decoder.layers.3.experts.down_proj.lora_a",
        "model.decoder.layers.3.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00026.safetensors",
      "bytes": 1645281350,
      "sha256": "57581d8a8c3efc5f19ac487469272cbf8d9fa68cb978c5d8f605701f3dca7fa2",
      "tensors": [
        "model.decoder.layers.3.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.3.experts.gate_up_proj.lora_a",
        "model.decoder.layers.3.experts.gate_up_proj.lora_b",
        "model.decoder.layers.3.input_layernorm.weight",
        "model.decoder.layers.3.layer_scalar",
        "model.decoder.layers.3.mlp.down_proj.linear.weight",
        "model.decoder.layers.3.mlp.down_proj.lora_a",
        "model.decoder.layers.3.mlp.down_proj.lora_b",
        "model.decoder.layers.3.mlp.gate_proj.linear.weight",
        "model.decoder.layers.3.mlp.gate_proj.lora_a",
        "model.decoder.layers.3.mlp.gate_proj.lora_b",
        "model.decoder.layers.3.mlp.up_proj.linear.weight",
        "model.decoder.layers.3.mlp.up_proj.lora_a",
        "model.decoder.layers.3.mlp.up_proj.lora_b",
        "model.decoder.layers.3.post_attention_layernorm.weight",
        "model.decoder.layers.3.post_feedforward_layernorm.weight",
        "model.decoder.layers.3.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.3.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.3.pre_feedforward_layernorm.weight",
        "model.decoder.layers.3.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.3.router.per_expert_scale",
        "model.decoder.layers.3.router.proj.weight",
        "model.decoder.layers.3.router.scale",
        "model.decoder.layers.3.self_attn.k_norm.weight",
        "model.decoder.layers.3.self_attn.k_proj.linear.weight",
        "model.decoder.layers.3.self_attn.k_proj.lora_a",
        "model.decoder.layers.3.self_attn.k_proj.lora_b",
        "model.decoder.layers.3.self_attn.o_proj.linear.weight",
        "model.decoder.layers.3.self_attn.o_proj.lora_a",
        "model.decoder.layers.3.self_attn.o_proj.lora_b",
        "model.decoder.layers.3.self_attn.q_norm.weight",
        "model.decoder.layers.3.self_attn.q_proj.linear.weight",
        "model.decoder.layers.3.self_attn.q_proj.lora_a",
        "model.decoder.layers.3.self_attn.q_proj.lora_b",
        "model.decoder.layers.3.self_attn.v_proj.linear.weight",
        "model.decoder.layers.3.self_attn.v_proj.lora_a",
        "model.decoder.layers.3.self_attn.v_proj.lora_b",
        "model.decoder.layers.4.experts.down_proj.linear.weight",
        "model.decoder.layers.4.experts.down_proj.lora_a",
        "model.decoder.layers.4.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00027.safetensors",
      "bytes": 1645281304,
      "sha256": "a2d814a172600dd0169ceac390fde8d8683926d72be9bc025c368c9c9bf5e5e9",
      "tensors": [
        "model.decoder.layers.4.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.4.experts.gate_up_proj.lora_a",
        "model.decoder.layers.4.experts.gate_up_proj.lora_b",
        "model.decoder.layers.4.input_layernorm.weight",
        "model.decoder.layers.4.layer_scalar",
        "model.decoder.layers.4.mlp.down_proj.linear.weight",
        "model.decoder.layers.4.mlp.down_proj.lora_a",
        "model.decoder.layers.4.mlp.down_proj.lora_b",
        "model.decoder.layers.4.mlp.gate_proj.linear.weight",
        "model.decoder.layers.4.mlp.gate_proj.lora_a",
        "model.decoder.layers.4.mlp.gate_proj.lora_b",
        "model.decoder.layers.4.mlp.up_proj.linear.weight",
        "model.decoder.layers.4.mlp.up_proj.lora_a",
        "model.decoder.layers.4.mlp.up_proj.lora_b",
        "model.decoder.layers.4.post_attention_layernorm.weight",
        "model.decoder.layers.4.post_feedforward_layernorm.weight",
        "model.decoder.layers.4.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.4.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.4.pre_feedforward_layernorm.weight",
        "model.decoder.layers.4.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.4.router.per_expert_scale",
        "model.decoder.layers.4.router.proj.weight",
        "model.decoder.layers.4.router.scale",
        "model.decoder.layers.4.self_attn.k_norm.weight",
        "model.decoder.layers.4.self_attn.k_proj.linear.weight",
        "model.decoder.layers.4.self_attn.k_proj.lora_a",
        "model.decoder.layers.4.self_attn.k_proj.lora_b",
        "model.decoder.layers.4.self_attn.o_proj.linear.weight",
        "model.decoder.layers.4.self_attn.o_proj.lora_a",
        "model.decoder.layers.4.self_attn.o_proj.lora_b",
        "model.decoder.layers.4.self_attn.q_norm.weight",
        "model.decoder.layers.4.self_attn.q_proj.linear.weight",
        "model.decoder.layers.4.self_attn.q_proj.lora_a",
        "model.decoder.layers.4.self_attn.q_proj.lora_b",
        "model.decoder.layers.4.self_attn.v_proj.linear.weight",
        "model.decoder.layers.4.self_attn.v_proj.lora_a",
        "model.decoder.layers.4.self_attn.v_proj.lora_b",
        "model.decoder.layers.5.experts.down_proj.linear.weight",
        "model.decoder.layers.5.experts.down_proj.lora_a",
        "model.decoder.layers.5.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00028.safetensors",
      "bytes": 1674191574,
      "sha256": "a7ee3b8626e6b76a82e0fbd78508a66151504ad02e5983f2895c735f3c41e704",
      "tensors": [
        "model.decoder.layers.5.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.5.experts.gate_up_proj.lora_a",
        "model.decoder.layers.5.experts.gate_up_proj.lora_b",
        "model.decoder.layers.5.input_layernorm.weight",
        "model.decoder.layers.5.layer_scalar",
        "model.decoder.layers.5.mlp.down_proj.linear.weight",
        "model.decoder.layers.5.mlp.down_proj.lora_a",
        "model.decoder.layers.5.mlp.down_proj.lora_b",
        "model.decoder.layers.5.mlp.gate_proj.linear.weight",
        "model.decoder.layers.5.mlp.gate_proj.lora_a",
        "model.decoder.layers.5.mlp.gate_proj.lora_b",
        "model.decoder.layers.5.mlp.up_proj.linear.weight",
        "model.decoder.layers.5.mlp.up_proj.lora_a",
        "model.decoder.layers.5.mlp.up_proj.lora_b",
        "model.decoder.layers.5.post_attention_layernorm.weight",
        "model.decoder.layers.5.post_feedforward_layernorm.weight",
        "model.decoder.layers.5.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.5.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.5.pre_feedforward_layernorm.weight",
        "model.decoder.layers.5.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.5.router.per_expert_scale",
        "model.decoder.layers.5.router.proj.weight",
        "model.decoder.layers.5.router.scale",
        "model.decoder.layers.5.self_attn.k_norm.weight",
        "model.decoder.layers.5.self_attn.k_proj.linear.weight",
        "model.decoder.layers.5.self_attn.k_proj.lora_a",
        "model.decoder.layers.5.self_attn.k_proj.lora_b",
        "model.decoder.layers.5.self_attn.o_proj.linear.weight",
        "model.decoder.layers.5.self_attn.o_proj.lora_a",
        "model.decoder.layers.5.self_attn.o_proj.lora_b",
        "model.decoder.layers.5.self_attn.q_norm.weight",
        "model.decoder.layers.5.self_attn.q_proj.linear.weight",
        "model.decoder.layers.5.self_attn.q_proj.lora_a",
        "model.decoder.layers.5.self_attn.q_proj.lora_b",
        "model.decoder.layers.6.experts.down_proj.linear.weight",
        "model.decoder.layers.6.experts.down_proj.lora_a",
        "model.decoder.layers.6.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00029.safetensors",
      "bytes": 1645281378,
      "sha256": "51e757bfbf302e147a926faf4ad7ab2baa97355b5e8ffd688b06a62f74b95f6b",
      "tensors": [
        "model.decoder.layers.6.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.6.experts.gate_up_proj.lora_a",
        "model.decoder.layers.6.experts.gate_up_proj.lora_b",
        "model.decoder.layers.6.input_layernorm.weight",
        "model.decoder.layers.6.layer_scalar",
        "model.decoder.layers.6.mlp.down_proj.linear.weight",
        "model.decoder.layers.6.mlp.down_proj.lora_a",
        "model.decoder.layers.6.mlp.down_proj.lora_b",
        "model.decoder.layers.6.mlp.gate_proj.linear.weight",
        "model.decoder.layers.6.mlp.gate_proj.lora_a",
        "model.decoder.layers.6.mlp.gate_proj.lora_b",
        "model.decoder.layers.6.mlp.up_proj.linear.weight",
        "model.decoder.layers.6.mlp.up_proj.lora_a",
        "model.decoder.layers.6.mlp.up_proj.lora_b",
        "model.decoder.layers.6.post_attention_layernorm.weight",
        "model.decoder.layers.6.post_feedforward_layernorm.weight",
        "model.decoder.layers.6.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.6.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.6.pre_feedforward_layernorm.weight",
        "model.decoder.layers.6.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.6.router.per_expert_scale",
        "model.decoder.layers.6.router.proj.weight",
        "model.decoder.layers.6.router.scale",
        "model.decoder.layers.6.self_attn.k_norm.weight",
        "model.decoder.layers.6.self_attn.k_proj.linear.weight",
        "model.decoder.layers.6.self_attn.k_proj.lora_a",
        "model.decoder.layers.6.self_attn.k_proj.lora_b",
        "model.decoder.layers.6.self_attn.o_proj.linear.weight",
        "model.decoder.layers.6.self_attn.o_proj.lora_a",
        "model.decoder.layers.6.self_attn.o_proj.lora_b",
        "model.decoder.layers.6.self_attn.q_norm.weight",
        "model.decoder.layers.6.self_attn.q_proj.linear.weight",
        "model.decoder.layers.6.self_attn.q_proj.lora_a",
        "model.decoder.layers.6.self_attn.q_proj.lora_b",
        "model.decoder.layers.6.self_attn.v_proj.linear.weight",
        "model.decoder.layers.6.self_attn.v_proj.lora_a",
        "model.decoder.layers.6.self_attn.v_proj.lora_b",
        "model.decoder.layers.7.experts.down_proj.linear.weight",
        "model.decoder.layers.7.experts.down_proj.lora_a",
        "model.decoder.layers.7.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00030.safetensors",
      "bytes": 1645281282,
      "sha256": "2dcc2427d1c1bd6d143283f4d2cede73940db3c093b59b45c1c1b879b5406631",
      "tensors": [
        "model.decoder.layers.7.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.7.experts.gate_up_proj.lora_a",
        "model.decoder.layers.7.experts.gate_up_proj.lora_b",
        "model.decoder.layers.7.input_layernorm.weight",
        "model.decoder.layers.7.layer_scalar",
        "model.decoder.layers.7.mlp.down_proj.linear.weight",
        "model.decoder.layers.7.mlp.down_proj.lora_a",
        "model.decoder.layers.7.mlp.down_proj.lora_b",
        "model.decoder.layers.7.mlp.gate_proj.linear.weight",
        "model.decoder.layers.7.mlp.gate_proj.lora_a",
        "model.decoder.layers.7.mlp.gate_proj.lora_b",
        "model.decoder.layers.7.mlp.up_proj.linear.weight",
        "model.decoder.layers.7.mlp.up_proj.lora_a",
        "model.decoder.layers.7.mlp.up_proj.lora_b",
        "model.decoder.layers.7.post_attention_layernorm.weight",
        "model.decoder.layers.7.post_feedforward_layernorm.weight",
        "model.decoder.layers.7.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.7.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.7.pre_feedforward_layernorm.weight",
        "model.decoder.layers.7.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.7.router.per_expert_scale",
        "model.decoder.layers.7.router.proj.weight",
        "model.decoder.layers.7.router.scale",
        "model.decoder.layers.7.self_attn.k_norm.weight",
        "model.decoder.layers.7.self_attn.k_proj.linear.weight",
        "model.decoder.layers.7.self_attn.k_proj.lora_a",
        "model.decoder.layers.7.self_attn.k_proj.lora_b",
        "model.decoder.layers.7.self_attn.o_proj.linear.weight",
        "model.decoder.layers.7.self_attn.o_proj.lora_a",
        "model.decoder.layers.7.self_attn.o_proj.lora_b",
        "model.decoder.layers.7.self_attn.q_norm.weight",
        "model.decoder.layers.7.self_attn.q_proj.linear.weight",
        "model.decoder.layers.7.self_attn.q_proj.lora_a",
        "model.decoder.layers.7.self_attn.q_proj.lora_b",
        "model.decoder.layers.7.self_attn.v_proj.linear.weight",
        "model.decoder.layers.7.self_attn.v_proj.lora_a",
        "model.decoder.layers.7.self_attn.v_proj.lora_b",
        "model.decoder.layers.8.experts.down_proj.linear.weight",
        "model.decoder.layers.8.experts.down_proj.lora_a",
        "model.decoder.layers.8.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00031.safetensors",
      "bytes": 1645281334,
      "sha256": "df89b8419ce01b7d78f761aac8e324df22004cfeba4c9ae81c627dde94a73535",
      "tensors": [
        "model.decoder.layers.8.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.8.experts.gate_up_proj.lora_a",
        "model.decoder.layers.8.experts.gate_up_proj.lora_b",
        "model.decoder.layers.8.input_layernorm.weight",
        "model.decoder.layers.8.layer_scalar",
        "model.decoder.layers.8.mlp.down_proj.linear.weight",
        "model.decoder.layers.8.mlp.down_proj.lora_a",
        "model.decoder.layers.8.mlp.down_proj.lora_b",
        "model.decoder.layers.8.mlp.gate_proj.linear.weight",
        "model.decoder.layers.8.mlp.gate_proj.lora_a",
        "model.decoder.layers.8.mlp.gate_proj.lora_b",
        "model.decoder.layers.8.mlp.up_proj.linear.weight",
        "model.decoder.layers.8.mlp.up_proj.lora_a",
        "model.decoder.layers.8.mlp.up_proj.lora_b",
        "model.decoder.layers.8.post_attention_layernorm.weight",
        "model.decoder.layers.8.post_feedforward_layernorm.weight",
        "model.decoder.layers.8.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.8.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.8.pre_feedforward_layernorm.weight",
        "model.decoder.layers.8.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.8.router.per_expert_scale",
        "model.decoder.layers.8.router.proj.weight",
        "model.decoder.layers.8.router.scale",
        "model.decoder.layers.8.self_attn.k_norm.weight",
        "model.decoder.layers.8.self_attn.k_proj.linear.weight",
        "model.decoder.layers.8.self_attn.k_proj.lora_a",
        "model.decoder.layers.8.self_attn.k_proj.lora_b",
        "model.decoder.layers.8.self_attn.o_proj.linear.weight",
        "model.decoder.layers.8.self_attn.o_proj.lora_a",
        "model.decoder.layers.8.self_attn.o_proj.lora_b",
        "model.decoder.layers.8.self_attn.q_norm.weight",
        "model.decoder.layers.8.self_attn.q_proj.linear.weight",
        "model.decoder.layers.8.self_attn.q_proj.lora_a",
        "model.decoder.layers.8.self_attn.q_proj.lora_b",
        "model.decoder.layers.8.self_attn.v_proj.linear.weight",
        "model.decoder.layers.8.self_attn.v_proj.lora_a",
        "model.decoder.layers.8.self_attn.v_proj.lora_b",
        "model.decoder.layers.9.experts.down_proj.linear.weight",
        "model.decoder.layers.9.experts.down_proj.lora_a",
        "model.decoder.layers.9.experts.down_proj.lora_b"
      ]
    },
    {
      "file": "model-00032.safetensors",
      "bytes": 1166260864,
      "sha256": "c62b07e428849f2db92bf286a58d2657c685deb2128692ef0de49d5df071f226",
      "tensors": [
        "model.decoder.layers.9.experts.gate_up_proj.linear.weight",
        "model.decoder.layers.9.experts.gate_up_proj.lora_a",
        "model.decoder.layers.9.experts.gate_up_proj.lora_b",
        "model.decoder.layers.9.input_layernorm.weight",
        "model.decoder.layers.9.layer_scalar",
        "model.decoder.layers.9.mlp.down_proj.linear.weight",
        "model.decoder.layers.9.mlp.down_proj.lora_a",
        "model.decoder.layers.9.mlp.down_proj.lora_b",
        "model.decoder.layers.9.mlp.gate_proj.linear.weight",
        "model.decoder.layers.9.mlp.gate_proj.lora_a",
        "model.decoder.layers.9.mlp.gate_proj.lora_b",
        "model.decoder.layers.9.mlp.up_proj.linear.weight",
        "model.decoder.layers.9.mlp.up_proj.lora_a",
        "model.decoder.layers.9.mlp.up_proj.lora_b",
        "model.decoder.layers.9.post_attention_layernorm.weight",
        "model.decoder.layers.9.post_feedforward_layernorm.weight",
        "model.decoder.layers.9.post_feedforward_layernorm_1.weight",
        "model.decoder.layers.9.post_feedforward_layernorm_2.weight",
        "model.decoder.layers.9.pre_feedforward_layernorm.weight",
        "model.decoder.layers.9.pre_feedforward_layernorm_2.weight",
        "model.decoder.layers.9.router.per_expert_scale",
        "model.decoder.layers.9.router.proj.weight",
        "model.decoder.layers.9.router.scale",
        "model.decoder.layers.9.self_attn.k_norm.weight",
        "model.decoder.layers.9.self_attn.k_proj.linear.weight",
        "model.decoder.layers.9.self_attn.k_proj.lora_a",
        "model.decoder.layers.9.self_attn.k_proj.lora_b",
        "model.decoder.layers.9.self_attn.o_proj.linear.weight",
        "model.decoder.layers.9.self_attn.o_proj.lora_a",
        "model.decoder.layers.9.self_attn.o_proj.lora_b",
        "model.decoder.layers.9.self_attn.q_norm.weight",
        "model.decoder.layers.9.self_attn.q_proj.linear.weight",
        "model.decoder.layers.9.self_attn.q_proj.lora_a",
        "model.decoder.layers.9.self_attn.q_proj.lora_b",
        "model.decoder.layers.9.self_attn.v_proj.linear.weight",
        "model.decoder.layers.9.self_attn.v_proj.lora_a",
        "model.decoder.layers.9.self_attn.v_proj.lora_b",
        "model.decoder.norm.weight",
        "model.decoder.self_conditioning.down_proj.weight",
        "model.decoder.self_conditioning.gate_proj.weight",
        "model.decoder.self_conditioning.pre_norm.weight",
        "model.decoder.self_conditioning.up_proj.weight",
        "model.encoder.language_model.layers.0.layer_scalar",
        "model.encoder.language_model.layers.1.layer_scalar",
        "model.encoder.language_model.layers.10.layer_scalar",
        "model.encoder.language_model.layers.11.layer_scalar",
        "model.encoder.language_model.layers.12.layer_scalar",
        "model.encoder.language_model.layers.13.layer_scalar",
        "model.encoder.language_model.layers.14.layer_scalar",
        "model.encoder.language_model.layers.15.layer_scalar",
        "model.encoder.language_model.layers.16.layer_scalar",
        "model.encoder.language_model.layers.17.layer_scalar",
        "model.encoder.language_model.layers.18.layer_scalar",
        "model.encoder.language_model.layers.19.layer_scalar",
        "model.encoder.language_model.layers.2.layer_scalar",
        "model.encoder.language_model.layers.20.layer_scalar",
        "model.encoder.language_model.layers.21.layer_scalar",
        "model.encoder.language_model.layers.22.layer_scalar",
        "model.encoder.language_model.layers.23.layer_scalar",
        "model.encoder.language_model.layers.24.layer_scalar",
        "model.encoder.language_model.layers.25.layer_scalar",
        "model.encoder.language_model.layers.26.layer_scalar",
        "model.encoder.language_model.layers.27.layer_scalar",
        "model.encoder.language_model.layers.28.layer_scalar",
        "model.encoder.language_model.layers.29.layer_scalar",
        "model.encoder.language_model.layers.3.layer_scalar",
        "model.encoder.language_model.layers.4.layer_scalar",
        "model.encoder.language_model.layers.5.layer_scalar",
        "model.encoder.language_model.layers.6.layer_scalar",
        "model.encoder.language_model.layers.7.layer_scalar",
        "model.encoder.language_model.layers.8.layer_scalar",
        "model.encoder.language_model.layers.9.layer_scalar"
      ]
    }
  ],
  "dependencies": {
    "mlx": "0.32.2",
    "mlx-lm": "0.31.3",
    "mlx-vlm": "0.7.2",
    "transformers": "5.14.1",
    "huggingface-hub": "1.20.1",
    "safetensors": "0.8.0",
    "numpy": "2.5.0"
  }
}