Add d270

d270/README.md

@@ -0,0 +1,28 @@
+# D270 (dense control)
+
+Part of the [Self-Drafting Latent Memory](../README.md) collection.
+
+No memory. Uniform d_ff = 2464, so its size (269.8M non-embedding) equals the memory models' active parameters per token. **Trained on 2B tokens** (3,815 steps), twice the memory models' budget.
+
+- **Architecture:** 24-layer Llama-style decoder, d = 1024, 24 query / 4 KV heads, SwiGLU, RoPE, tied embeddings, vocabulary 32,768, context 2048. Training config: `w3_dense270` in the repository.
+- **Validation loss:** 2.5147 (2B tokens); 2.7167 at 0.94B.
+- **Zero-shot (lm-eval 0.4.13):**
+
+  | HellaSwag | MMLU-cloze | SciQ | WinoGrande | PIQA | ARC-Easy |
+  |---|---|---|---|---|---|
+  | 31.0 | 25.0 | 68.1 | 52.6 | 61.5 | 41.2 |
+
+## Files
+
+- `model.pt`: `{"model": state_dict, "config": ..., "step": ..., "params": ...}`, weights only (no optimizer state).
+- `config.json`: the resolved training/model config.
+- `params.json`: parameter counts by component.
+
+## Load
+
+```python
+from ffnmem.infer import load_model   # code from the accompanying repository
+model, cfg = load_model("d270/model.pt", device="cuda")
+```
+
+Research checkpoint: not instruction-tuned, not safety-tuned, not for deployment.

d270/config.json

@@ -0,0 +1,121 @@
+{
+ "model": {
+  "d_model": 1024,
+  "n_layers": 24,
+  "n_heads": 24,
+  "n_kv_heads": 4,
+  "head_dim": 64,
+  "vocab": 32768,
+  "seq_len": 2048,
+  "d_ff": [
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464,
+   2464
+  ],
+  "memory_layers": [],
+  "memory": null,
+  "norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "init_std": 0.02,
+  "layers": [
+   0,
+   1,
+   2,
+   3,
+   4,
+   5,
+   6,
+   7,
+   8,
+   9,
+   10,
+   11,
+   12,
+   13,
+   14,
+   15,
+   16,
+   17,
+   18,
+   19,
+   20,
+   21,
+   22,
+   23
+  ],
+  "memory_overrides": {},
+  "draft": null,
+  "ssm": null
+ },
+ "train": {
+  "seed": 1234,
+  "data_seed": 1234,
+  "val_tokens": 20000000,
+  "global_batch": 256,
+  "micro_batch": 2,
+  "tokens_total": 2000000000,
+  "lr": 0.0006,
+  "min_lr_ratio": 0.1,
+  "warmup_frac": 0.02,
+  "warmup_min_steps": 100,
+  "betas": [
+   0.9,
+   0.95
+  ],
+  "eps": 1e-08,
+  "weight_decay": 0.1,
+  "grad_clip": 1.0,
+  "table_lr_mult": 4.0,
+  "precision": "fp16",
+  "act_ckpt": true,
+  "attn_backend": "sdpa",
+  "zero1": true,
+  "ckpt_every": 200,
+  "keep_last": 3,
+  "keep_every": 1000,
+  "eval_every": 200,
+  "eval_max_batches": null,
+  "diag_batches": 4,
+  "log_every": 10,
+  "scaler_min": 1.0,
+  "scaler_patience": 100,
+  "nan_patience": 20,
+  "save_steps": [
+   1000,
+   2000,
+   3815
+  ],
+  "strip_old_optimizer": true,
+  "seq_len": 2048
+ },
+ "name": "w3_dense270",
+ "params": {
+  "expected_non_embedding": null,
+  "reference": false,
+  "match_reference": false
+ },
+ "wandb": {
+  "project": "morbius-wave3"
+ }
+}

d270/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b0158c8807023d6b75364dc8f35aa0b3fb13d6c2572a81a5e32daa4037d2db4
+size 1213468811

d270/params.json

@@ -0,0 +1,13 @@
+{
+ "embedding": 33554432,
+ "attention": 88080384,
+ "ffn": 181665792,
+ "norms": 50176,
+ "memory_tables": 0,
+ "memory_readout": 0,
+ "draft": 0,
+ "ssm": 0,
+ "total": 303350784,
+ "non_embedding": 269796352,
+ "active_non_embedding_per_token": 269796352
+}