periodic checkpoint @ step 200

README.md

@@ -0,0 +1,15 @@
+---
+language: en
+library_name: transformers
+tags:
+- kimi-k3
+- luau
+- experimental-checkpoint
+---
+
+# d
+
+In-training periodic checkpoint (step 200). Architecture:
+Kimi K3 mini port (`KimiLinearForCausalLM`).
+
+**Not a finished model** -- intermediate weights uploaded during training for safekeeping/diffing.

config.json

@@ -0,0 +1,84 @@
+{
+  "architectures": [
+    "KimiLinearForCausalLM"
+  ],
+  "model_type": "kimi_linear",
+  "auto_map": {
+    "AutoConfig": "configuration_kimi_k3.KimiLinearConfig",
+    "AutoModelForCausalLM": "modeling_kimi_linear.KimiLinearForCausalLM"
+  },
+  "vocab_size": 32768,
+  "hidden_size": 2560,
+  "num_hidden_layers": 32,
+  "num_attention_heads": 20,
+  "num_key_value_heads": 20,
+  "max_position_embeddings": 1048576,
+  "rms_norm_eps": 1e-06,
+  "tie_word_embeddings": true,
+  "linear_attn_config": {
+    "full_attn_layers": [
+      4,
+      8,
+      12,
+      16,
+      20,
+      24,
+      28,
+      32
+    ],
+    "kda_layers": [
+      1,
+      2,
+      3,
+      5,
+      6,
+      7,
+      9,
+      10,
+      11,
+      13,
+      14,
+      15,
+      17,
+      18,
+      19,
+      21,
+      22,
+      23,
+      25,
+      26,
+      27,
+      29,
+      30,
+      31
+    ],
+    "num_heads": 20,
+    "head_dim": 128,
+    "short_conv_kernel_size": 4,
+    "gate_lower_bound": -5.0,
+    "use_full_rank_gate": true
+  },
+  "q_lora_rank": 1280,
+  "kv_lora_rank": 640,
+  "qk_nope_head_dim": 128,
+  "qk_rope_head_dim": 0,
+  "v_head_dim": 128,
+  "mla_use_nope": true,
+  "mla_use_output_gate": true,
+  "num_experts": 0,
+  "num_experts_per_tok": 0,
+  "num_shared_experts": 0,
+  "moe_intermediate_size": 0,
+  "routed_expert_hidden_size": 0,
+  "first_k_dense_replace": 32,
+  "routed_scaling_factor": 1.0,
+  "moe_renormalize": false,
+  "moe_router_activation_func": "sigmoid",
+  "dense_ffn_hidden": 6144,
+  "intermediate_size": 6144,
+  "hidden_act": "situ",
+  "activation_situ_beta": 4.0,
+  "activation_situ_linear_beta": 25.0,
+  "attn_res_block_size": 4,
+  "initializer_range": 0.02
+}

metrics.jsonl

@@ -0,0 +1,23 @@
+{"wall": 88.274, "eval_loss": 10.890216708183289, "eval_ppl": 53648.9296875, "step": 0}
+{"wall": 411.813, "step": 10, "loss": 10.885039567947388, "lm_loss": 10.885039567947388, "mtp_loss": 0.0, "lr": 0.00013560344827586205, "grad_norm": 33.359554290771484, "tokens": 2949120, "adamw_lr": 4.172413793103448e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.169921875, "loss_smoothed": 10.88545929590861, "seq_len": 4096}
+{"wall": 616.746, "step": 20, "loss": 10.876587311426798, "lm_loss": 10.876587311426798, "mtp_loss": 0.0, "lr": 0.00014120689655172414, "grad_norm": 33.370948791503906, "tokens": 5898240, "adamw_lr": 4.344827586206897e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.169921875, "loss_smoothed": 10.87702712615331, "seq_len": 4096}
+{"wall": 821.709, "step": 30, "loss": 10.862957080205282, "lm_loss": 10.862957080205282, "mtp_loss": 0.0, "lr": 0.0001468103448275862, "grad_norm": 33.463016510009766, "tokens": 8847360, "adamw_lr": 4.5172413793103454e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.169921875, "loss_smoothed": 10.86825514237086, "seq_len": 4096}
+{"wall": 1026.873, "step": 40, "loss": 10.857197324434916, "lm_loss": 10.857197324434916, "mtp_loss": 0.0, "lr": 0.0001524137931034483, "grad_norm": 33.4448356628418, "tokens": 11796480, "adamw_lr": 4.689655172413794e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.169921875, "loss_smoothed": 10.862377226352692, "seq_len": 4096}
+{"wall": 1231.558, "step": 50, "loss": 10.852633595466614, "lm_loss": 10.852633595466614, "mtp_loss": 0.0, "lr": 0.00015801724137931035, "grad_norm": 33.3587532043457, "tokens": 14745600, "adamw_lr": 4.862068965517242e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.169921875, "loss_smoothed": 10.856609841187794, "seq_len": 4096}
+{"wall": 1436.412, "step": 60, "loss": 10.848609368006388, "lm_loss": 10.848609368006388, "mtp_loss": 0.0, "lr": 0.00016362068965517242, "grad_norm": 33.31964111328125, "tokens": 17694720, "adamw_lr": 5.03448275862069e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.169921875, "loss_smoothed": 10.849376634756725, "seq_len": 4096}
+{"wall": 1641.593, "step": 70, "loss": 10.840846498807272, "lm_loss": 10.840846498807272, "mtp_loss": 0.0, "lr": 0.00016922413793103448, "grad_norm": 33.22914505004883, "tokens": 20643840, "adamw_lr": 5.2068965517241384e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.169921875, "loss_smoothed": 10.843668512503307, "seq_len": 4096}
+{"wall": 1846.552, "step": 80, "loss": 10.835049510002136, "lm_loss": 10.835049510002136, "mtp_loss": 0.0, "lr": 0.00017482758620689654, "grad_norm": 33.283485412597656, "tokens": 23592960, "adamw_lr": 5.379310344827586e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.169921875, "loss_smoothed": 10.83760959704717, "seq_len": 4096}
+{"wall": 2051.419, "step": 90, "loss": 10.829344550768534, "lm_loss": 10.829344550768534, "mtp_loss": 0.0, "lr": 0.0001804310344827586, "grad_norm": 33.317237854003906, "tokens": 26542080, "adamw_lr": 5.551724137931035e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.169921875, "loss_smoothed": 10.830694985389709, "seq_len": 4096}
+{"wall": 2258.685, "eval_loss": 10.822476506233215, "eval_ppl": 50135.0859375, "step": 100}
+{"wall": 2258.687, "step": 100, "loss": 10.818859140078226, "lm_loss": 10.818859140078226, "mtp_loss": 0.0, "lr": 0.0001860344827586207, "grad_norm": 33.2758903503418, "tokens": 29491200, "adamw_lr": 5.7241379310344835e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.169921875, "loss_smoothed": 10.822945741812386, "seq_len": 4096}
+{"wall": 2479.188, "step": 110, "loss": 10.814252654711405, "lm_loss": 10.814252654711405, "mtp_loss": 0.0, "lr": 0.00019163793103448278, "grad_norm": 33.27556610107422, "tokens": 32440320, "adamw_lr": 5.896551724137932e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.072265625, "loss_smoothed": 10.818115150928497, "seq_len": 4096}
+{"wall": 2684.107, "step": 120, "loss": 10.803292949994406, "lm_loss": 10.803292949994406, "mtp_loss": 0.0, "lr": 0.00019724137931034482, "grad_norm": 33.32582092285156, "tokens": 35389440, "adamw_lr": 6.068965517241379e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.072265625, "loss_smoothed": 10.810300902525585, "seq_len": 4096}
+{"wall": 2889.22, "step": 130, "loss": 10.799176494280497, "lm_loss": 10.799176494280497, "mtp_loss": 0.0, "lr": 0.0002028448275862069, "grad_norm": 33.18770980834961, "tokens": 38338560, "adamw_lr": 6.241379310344829e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.072265625, "loss_smoothed": 10.804276450475056, "seq_len": 4096}
+{"wall": 3094.181, "step": 140, "loss": 10.793241739273071, "lm_loss": 10.793241739273071, "mtp_loss": 0.0, "lr": 0.00020844827586206894, "grad_norm": 33.08427810668945, "tokens": 41287680, "adamw_lr": 6.413793103448276e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.072265625, "loss_smoothed": 10.798630730311077, "seq_len": 4096}
+{"wall": 3299.102, "step": 150, "loss": 10.787952423095703, "lm_loss": 10.787952423095703, "mtp_loss": 0.0, "lr": 0.00021405172413793106, "grad_norm": 33.30586624145508, "tokens": 44236800, "adamw_lr": 6.586206896551726e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.072265625, "loss_smoothed": 10.791370856761933, "seq_len": 4096}
+{"wall": 3503.945, "step": 160, "loss": 10.781545042991638, "lm_loss": 10.781545042991638, "mtp_loss": 0.0, "lr": 0.0002196551724137931, "grad_norm": 33.21318435668945, "tokens": 47185920, "adamw_lr": 6.758620689655173e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.072265625, "loss_smoothed": 10.783225440979004, "seq_len": 4096}
+{"wall": 3709.172, "step": 170, "loss": 10.767958879470825, "lm_loss": 10.767958879470825, "mtp_loss": 0.0, "lr": 0.00022525862068965515, "grad_norm": 65.31437683105469, "tokens": 50135040, "adamw_lr": 6.93103448275862e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.072265625, "loss_smoothed": 10.774782085418702, "seq_len": 4096}
+{"wall": 3913.611, "step": 180, "loss": 10.770054618517557, "lm_loss": 10.770054618517557, "mtp_loss": 0.0, "lr": 0.00023086206896551727, "grad_norm": 37.112770080566406, "tokens": 53084160, "adamw_lr": 7.10344827586207e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.072265625, "loss_smoothed": 10.767992003758748, "seq_len": 4096}
+{"wall": 4118.728, "step": 190, "loss": 10.752914865811666, "lm_loss": 10.752914865811666, "mtp_loss": 0.0, "lr": 0.0002364655172413793, "grad_norm": 33.216190338134766, "tokens": 56033280, "adamw_lr": 7.275862068965517e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.072265625, "loss_smoothed": 10.75819280942281, "seq_len": 4096}
+{"wall": 4325.819, "eval_loss": 10.744163990020752, "eval_ppl": 46358.6640625, "step": 200}
+{"wall": 4325.821, "step": 200, "loss": 10.74438444773356, "lm_loss": 10.74438444773356, "mtp_loss": 0.0, "lr": 0.0002420689655172414, "grad_norm": 33.22344207763672, "tokens": 58982400, "adamw_lr": 7.448275862068966e-05, "mem_alloc_gb": 14.685915470123291, "mem_reserv_gb": 79.072265625, "loss_smoothed": 10.748678056399028, "seq_len": 4096}