export step 4000 (13.88%)

README.md

@@ -13,11 +13,11 @@ tags:
 
 # Oleander-150M
 
-**Training checkpoint** (`export`) — step **3,500 / 28,820** · **12.14%** of the planned run.
+**Training checkpoint** (`export`) — step **4,000 / 28,820** · **13.88%** of the planned run.
 
 ```text
-steps    ███░░░░░░░░░░░░░░░░░░░░░░░░░  12.14%   3,500 / 28,820
-tokens   ████░░░░░░░░░░░░░░░░░░░░░░░░  12.52%   1.89B / 15.11B compute
+steps    ████░░░░░░░░░░░░░░░░░░░░░░░░  13.88%   4,000 / 28,820
+tokens   ████░░░░░░░░░░░░░░░░░░░░░░░░  14.25%   2.15B / 15.11B compute
 phase    tst   (TST bag training ends at step 8,646)
 ```
 
@@ -27,17 +27,17 @@ Trained with the Oleander marimo trainer (molab): hybrid **GDN2** (gated delta-r
 
 | | |
 |---|---|
-| Step | **3,500** / 28,820 |
-| % complete (steps) | **12.14%** |
-| Compute tokens seen | **1.89B** / 15.11B (12.52%) |
-| Raw tokens seen (TST bags unpacked) | 11.35B / 37.77B |
-| Cluster tokens (all 1 worker(s)) | 1.89B / 15.11B compute |
+| Step | **4,000** / 28,820 |
+| % complete (steps) | **13.88%** |
+| Compute tokens seen | **2.15B** / 15.11B (14.25%) |
+| Raw tokens seen (TST bags unpacked) | 12.92B / 37.77B |
+| Cluster tokens (all 1 worker(s)) | 2.15B / 15.11B compute |
 | Phase | `tst` |
-| Val loss / ppl | **5.8579** / **349.98** |
-| Train loss (latest) | 5.3518 |
-| Throughput | 76,995 compute tok/s · 0.146 steps/s |
-| Session elapsed | 0d 05h 30m · ETA 2d 00h 17m |
-| Checkpoint written | 2026-11-12T15:51:00Z |
+| Val loss / ppl | **5.8692** / **353.98** |
+| Train loss (latest) | 5.3699 |
+| Throughput | 77,102 compute tok/s · 0.146 steps/s |
+| Session elapsed | 0d 06h 28m · ETA 1d 23h 18m |
+| Checkpoint written | 2026-11-12T16:48:07Z |
 
 ## How to load
 
@@ -98,6 +98,6 @@ Oleander-150M (28L = 7 x [GDN2 · GDN2 · GDN2 · GQA] · d=576 · SwiGLU(1600)
 
 ## Notes
 
-- This is a **work-in-progress** checkpoint at 12.14% of the planned run, not a finished model: expect repetition and incoherence, especially with few-shot prompts.
+- This is a **work-in-progress** checkpoint at 13.88% of the planned run, not a finished model: expect repetition and incoherence, especially with few-shot prompts.
 - The tokenizer comes from the base model the run started from; the vocabulary is a plain BPE model and is not extended during training.
 - `training_meta.json` is regenerated on every push and is the authoritative record of how far training got.

config.json

@@ -3,10 +3,6 @@
     "OleanderForCausalLM"
   ],
   "model_type": "oleander",
-  "auto_map": {
-    "AutoConfig": "configuration_oleander.OleanderConfig",
-    "AutoModelForCausalLM": "modeling_oleander.OleanderForCausalLM"
-  },
   "vocab_size": 32770,
   "hidden_size": 576,
   "num_hidden_layers": 28,
@@ -71,7 +67,7 @@
     "full_attention"
   ],
   "intermediate_size": 1600,
-  "max_position_embeddings": 2048,
+  "max_position_embeddings": 1024,
   "rms_norm_eps": 1e-06,
   "num_attention_heads": 9,
   "num_key_value_heads": 3,
@@ -91,7 +87,7 @@
       "gqa"
     ],
     "mlp_hidden": 1600,
-    "max_seq_len": 2048,
+    "max_seq_len": 1024,
     "norm_eps": 1e-06,
     "init_std": 0.02,
     "tie_embeddings": true,
@@ -115,5 +111,15 @@
       "value_residual": true
     }
   },
-  "model_name": "Oleander-150M"
+  "model_name": "Oleander-150M",
+  "training": {
+    "step": 4000,
+    "kind": "export",
+    "max_steps": 28820,
+    "percent_complete": 13.879,
+    "tokens_seen": 12922650624,
+    "compute_tokens_seen": 2153775104
+  },
+  "transformers_version": "5.14.1",
+  "torch_dtype": "float32"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b9ab55c38570e42aa66a8a54aebb22e0b52dce596903891843e4d1bba0e1535
+oid sha256:7c99dc7195c5b595080777800f6ddd682e8167ae98461cc6e4c77fd666ae9be4
 size 599970860

progress.json

@@ -1,15 +1,15 @@
 {
-  "step": 3500,
+  "step": 4000,
   "max_steps": 28820,
-  "percent_complete": 12.144,
-  "tokens_seen": 11349786624,
-  "compute_tokens_seen": 1891631104,
+  "percent_complete": 13.879,
+  "tokens_seen": 12922650624,
+  "compute_tokens_seen": 2153775104,
   "compute_tokens_total": 15109980160,
-  "percent_tokens": 12.519,
-  "val_loss": 5.857882604002953,
-  "val_ppl": 349.98230777900153,
+  "percent_tokens": 14.254,
+  "val_loss": 5.86922912299633,
+  "val_ppl": 353.97600310473035,
   "params_M": 149.976,
-  "pushed_at": "2026-11-12T15:51:00Z",
+  "pushed_at": "2026-11-12T16:48:07Z",
   "architecture": "Oleander-150M",
   "features": [
     "GDN2 (gated delta net 2) x21: 9h x 64, conv4",

training_meta.json

@@ -1,10 +1,10 @@
 {
   "format": "oleander-training-meta/1",
-  "generated_at": "2026-11-12T15:51:00Z",
+  "generated_at": "2026-11-12T16:48:07Z",
   "kind": "export",
-  "step": 3500,
-  "tokens_seen": 11349786624,
-  "pushed_at": "2026-11-12T15:51:00Z",
+  "step": 4000,
+  "tokens_seen": 12922650624,
+  "pushed_at": "2026-11-12T16:48:07Z",
   "params_M": 149.976,
   "architecture": "Oleander-150M",
   "features": [
@@ -20,55 +20,55 @@
     "TST (token superposition training, bag 6)"
   ],
   "optimizer": "muon_adamw",
-  "val_loss": 5.857882604002953,
-  "val_ppl": 349.98230777900153,
+  "val_loss": 5.86922912299633,
+  "val_ppl": 353.97600310473035,
   "run": {
     "model_name": "Oleander-150M",
     "architecture_detail": "Oleander-150M (28L = 7 x [GDN2 \u00b7 GDN2 \u00b7 GDN2 \u00b7 GQA] \u00b7 d=576 \u00b7 SwiGLU(1600) \u00b7 tied 32,770 vocab)",
     "session_id": "20261112_101934_502ed6",
     "repo_id": "CyanMonkey/Test-Subject-3",
     "started_at": "2026-11-12T10:20:00Z",
-    "session_elapsed_s": 19859.3,
-    "last_update_at": "2026-11-12T15:50:58Z",
-    "steps_per_second": 0.14565,
-    "compute_tokens_per_second": 76994.8,
-    "eta": "2d 00h 17m",
+    "session_elapsed_s": 23287.0,
+    "last_update_at": "2026-11-12T16:48:02Z",
+    "steps_per_second": 0.14571,
+    "compute_tokens_per_second": 77102.5,
+    "eta": "1d 23h 18m",
     "checkpoint_kind": "export"
   },
   "progress": {
-    "step": 3500,
+    "step": 4000,
     "max_steps": 28820,
-    "step_fraction": 0.121443,
-    "percent_complete": 12.144,
-    "tokens_fraction": 0.125191,
-    "percent_tokens": 12.519,
+    "step_fraction": 0.138793,
+    "percent_complete": 13.879,
+    "tokens_fraction": 0.14254,
+    "percent_tokens": 14.254,
     "phase": "tst",
-    "phase_fraction": 0.404811,
+    "phase_fraction": 0.462642,
     "tst_switch_step": 8646,
     "tst_switch_applied": false
   },
   "tokens": {
-    "compute_tokens_seen": 1891631104,
+    "compute_tokens_seen": 2153775104,
     "compute_tokens_total": 15109980160,
-    "raw_tokens_seen": 11349786624,
+    "raw_tokens_seen": 12922650624,
     "raw_tokens_total": 37774950400,
     "raw_over_compute_ratio": 6.0,
-    "samples_seen": 10752000,
-    "rows_consumed": 10752000,
+    "samples_seen": 12288000,
+    "rows_consumed": 12288000,
     "compute_tokens_per_step": 524288,
     "cluster": {
       "n_workers": 1,
-      "compute_tokens_seen_all_workers": 1891631104,
-      "raw_tokens_seen_all_workers": 11349786624,
+      "compute_tokens_seen_all_workers": 2153775104,
+      "raw_tokens_seen_all_workers": 12922650624,
       "compute_tokens_total_planned_all_workers": 15109980160,
       "raw_tokens_total_planned_all_workers": 37774950400
     }
   },
   "metrics": {
-    "train_loss": 5.351828575134277,
-    "val_loss": 5.857882604002953,
-    "val_ppl": 349.98230777900153,
-    "grad_norm": 0.2005428969860077,
+    "train_loss": 5.369927883148193,
+    "val_loss": 5.86922912299633,
+    "val_ppl": 353.97600310473035,
+    "grad_norm": 0.2886804938316345,
     "lr_mult": 1.0
   },
   "optimization": {
@@ -151,7 +151,7 @@
     "doc_masking": "auto",
     "shuffle_block_rows": 192,
     "seed": 42,
-    "epochs_consumed_this_worker": 0.7288
+    "epochs_consumed_this_worker": 0.8329
   },
   "hardware": {
     "device": "cuda",