Add base checkpoint (step 2162)

README.md

@@ -0,0 +1,19 @@
+---
+license: mit
+---
+
+# lfm2-quantum-128m (base checkpoint, step 2162)
+
+LFM2-style hybrid quantum GPT, budget tier, trained with
+[Quantum-GPT](https://github.com) `runs/run_lfm2_quantum_mini.sh`.
+
+- 16 layers (10 gated short-conv + 6 GQA attention, conv-first)
+- n_embd=1024, 16 query heads / 8 KV heads (GQA), seq_len 1024
+- vocab_size 32768
+- Quantum feed-forward network in every block (4 qubits, depth 2)
+- RoPE theta 1e6, value embeddings disabled
+
+This is a **base** (pretrained, not instruction-tuned) checkpoint at step 2162
+(final step of pretraining). `meta_002162.json` has the full model/training config.
+`tokenizer/` contains the matching BPE tokenizer (`tokenizer.pkl`) and token-byte
+table (`token_bytes.pt`) needed to reproduce BPB evaluation.

meta_002162.json

@@ -0,0 +1,77 @@
+{
+  "step": 2162,
+  "val_bpb": 0.9221935981158367,
+  "model_config": {
+    "sequence_len": 1024,
+    "vocab_size": 32768,
+    "n_layer": 16,
+    "n_head": 16,
+    "n_kv_head": 8,
+    "n_embd": 1024,
+    "mlp_type": "quantum",
+    "quantum_num_qubits": 4,
+    "quantum_depth": 2,
+    "window_pattern": "L",
+    "mixer_pattern": "CCCCCCCCCCAAAAAA",
+    "conv_kernel": 3,
+    "rope_theta": 1000000.0,
+    "ffn_pattern": "",
+    "use_value_embeddings": false
+  },
+  "user_config": {
+    "run": "lfm2-quantum-128m",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 16,
+    "aspect_ratio": 64,
+    "head_dim": 64,
+    "max_seq_len": 1024,
+    "window_pattern": "L",
+    "mlp_type": "quantum",
+    "quantum_num_qubits": 4,
+    "quantum_depth": 2,
+    "mixer_pattern": "CCCCCCCCCCAAAAAA",
+    "conv_kernel": 3,
+    "rope_theta": 1000000.0,
+    "ffn_pattern": "",
+    "n_kv_head": 8,
+    "disable_value_embeds": true,
+    "gradient_checkpointing": true,
+    "num_iterations": -1,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12.0,
+    "device_batch_size": 16,
+    "total_batch_size": -1,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "muon_group_size": 8,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.2,
+    "final_lr_frac": 0.1,
+    "resume_from_step": -1,
+    "eval_every": 250,
+    "eval_tokens": 1048576,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 250,
+    "model_tag": "lfm2-quantum-128m"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 1024,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "pq_idx": 28,
+    "rg_idx": 32,
+    "epoch": 1
+  },
+  "loop_state": {
+    "min_val_bpb": 0.9221935981158367,
+    "smooth_train_loss": 3.03223018311102,
+    "total_training_time": 50648.47665858269
+  }
+}

model_002162.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:924c928581545cbd084392f689222d180af42870f91f08d572d147981c73b167
+size 445164484

tokenizer/token_bytes.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bae8cf2ec0dc77c382412f9eb4ed74681d38f6453b1f378d8251ea32192844d
+size 132649

tokenizer/tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae73c5f7a960edc56022dfd46a653df2b9b38d84456e5e9f48eb5e02a60c21c2
+size 412126