step 398 shard 24

README.md

@@ -1,10 +1,5 @@
----
-license: apache-2.0
-library_name: pytorch
----
-
 # lobotomy-1b
 
-21-layer ~1B student from Qwen3.8-27B (64→21, d=5120→1536).
+21-layer ~1B student (Qwen3.8-27B pooled 64→21, downproj 5120→1536).
 
-step=366
+step=398  pass=

config.json

@@ -1,6 +1,5 @@
 {
-  "step": 366,
-  "cfg": {
+  "model": {
     "vocab_size": 248320,
     "hidden_size": 1536,
     "n_layers": 21,
@@ -19,10 +18,56 @@
     "logit_softcap": 15.0,
     "mtp": 1
   },
-  "progress": {
-    "next_shard_i": 23,
-    "step": 367,
-    "last_shard": "shard_00022.safetensors",
-    "pass": "in_progress"
-  }
+  "train": {
+    "seq_len": 256,
+    "batch_size": 1,
+    "max_steps": 8000,
+    "warmup_steps": 200,
+    "cooldown_frac": 0.45,
+    "decay_steps": 40,
+    "lr": 0.0003,
+    "muon_lr": 0.02,
+    "embed_lr": 0.0006,
+    "head_lr": 0.0003,
+    "muon_momentum_warmup": 200,
+    "weight_decay": 0.1,
+    "seq_schedule": [],
+    "grad_clip": 1.0,
+    "optimizer": "muon",
+    "bf16": true,
+    "compile": false,
+    "seed": 0,
+    "log_every": 10,
+    "save_every": 200,
+    "out_dir": "checkpoints/1b"
+  },
+  "distill": {
+    "enabled": true,
+    "teacher": "Qwen/Qwen3.8-27B",
+    "mode": "hub",
+    "cache_dir": "data/teacher",
+    "load_in_4bit": true,
+    "weight": 0.0,
+    "ce_weight": 0.1,
+    "logit_weight": 1.0,
+    "loss": "cosine",
+    "layers": "grouped",
+    "pool": "last",
+    "pca_batches": 0,
+    "subsample": 0,
+    "max_seq": 256,
+    "temperature": 2.0,
+    "trust_remote_code": true,
+    "hub_repo": "avewright/qwen-code-soft-logits",
+    "hub_model": "avewright/lobotomy-1b",
+    "hub_private": true,
+    "shard_seqs": 16,
+    "max_local_gb": 6.0,
+    "local_dir": "data/teacher_shards",
+    "max_passes": 1,
+    "upload_every_shards": 5
+  },
+  "step": 398,
+  "shard_i": 24,
+  "pass": ""
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa1d49b8327563738a76a8e5c270aa298d92df7552fd1571985d7028ef2ff082
+oid sha256:6b4e0e86e6f5a1adf5fc5e7c8288c294f7bea786e893f10b53471220ef9b231e
 size 2698072840