CyanMonkey commited on
Commit
a10c7e3
·
verified ·
1 Parent(s): 91a3ef9

round 1 step 500 (1.75%)

Browse files
Files changed (6) hide show
  1. README.md +15 -15
  2. config.json +6 -6
  3. model.safetensors +1 -1
  4. progress.json +11 -11
  5. train_config.json +2 -2
  6. training_meta.json +53 -44
README.md CHANGED
@@ -13,12 +13,12 @@ tags:
13
 
14
  # Oleander-150M
15
 
16
- **Training checkpoint** (`export`) — step **6,500 / 28,820** · **22.55%** of the planned run.
17
 
18
  ```text
19
- steps ██████░░░░░░░░░░░░░░░░░░░░░░ 22.55% 6,500 / 28,820
20
- tokens ██████░░░░░░░░░░░░░░░░░░░░░░ 22.93% 3.46B / 15.11B compute
21
- phase tst (TST bag training ends at step 8,646)
22
  ```
23
 
24
  Trained with the Oleander marimo trainer (molab): hybrid **GDN2** (gated delta-rule, gated-delta-net-2) and **gated GQA** layers with **block attention residuals**, MuonEq-R + AdamW, WSD schedule.
@@ -27,17 +27,17 @@ Trained with the Oleander marimo trainer (molab): hybrid **GDN2** (gated delta-r
27
 
28
  | | |
29
  |---|---|
30
- | Step | **6,500** / 28,820 |
31
- | % complete (steps) | **22.55%** |
32
- | Compute tokens seen | **3.46B** / 15.11B (22.93%) |
33
- | Raw tokens seen (TST bags unpacked) | 20.79B / 37.77B |
34
- | Cluster tokens (all 1 worker(s)) | 3.46B / 15.11B compute |
35
  | Phase | `tst` |
36
- | Val loss / ppl | **5.7545** / **315.62** |
37
- | Train loss (latest) | 5.2196 |
38
- | Throughput | 77,156 compute tok/s · 0.145 steps/s |
39
- | Session elapsed | 0d 11h 23m · ETA 1d 18h 51m |
40
- | Checkpoint written | 2026-11-14T17:23:50Z |
41
 
42
  ## How to load
43
 
@@ -98,6 +98,6 @@ Oleander-150M (28L = 7 x [GDN2 · GDN2 · GDN2 · GQA] · d=576 · SwiGLU(1600)
98
 
99
  ## Notes
100
 
101
- - This is a **work-in-progress** checkpoint at 22.55% of the planned run, not a finished model: expect repetition and incoherence, especially with few-shot prompts.
102
  - The tokenizer comes from the base model the run started from; the vocabulary is a plain BPE model and is not extended during training.
103
  - `training_meta.json` is regenerated on every push and is the authoritative record of how far training got.
 
13
 
14
  # Oleander-150M
15
 
16
+ **Training checkpoint** (`round 1`) — step **500 / 28,500** · **1.75%** of the planned run.
17
 
18
  ```text
19
+ steps ░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 1.75% 500 / 28,500
20
+ tokens ░░░░░░░░░░░░░░░░░░░░░░░░░░░░ 1.75% 262.14M / 14.94B compute
21
+ phase tst (TST bag training ends at step 9,000)
22
  ```
23
 
24
  Trained with the Oleander marimo trainer (molab): hybrid **GDN2** (gated delta-rule, gated-delta-net-2) and **gated GQA** layers with **block attention residuals**, MuonEq-R + AdamW, WSD schedule.
 
27
 
28
  | | |
29
  |---|---|
30
+ | Step | **500** / 28,500 |
31
+ | % complete (steps) | **1.75%** |
32
+ | Compute tokens seen | **262.14M** / 14.94B (1.75%) |
33
+ | Raw tokens seen (TST bags unpacked) | 1.57B / 38.54B |
34
+ | Cluster tokens (all 1 worker(s)) | 262.14M / 14.94B compute |
35
  | Phase | `tst` |
36
+ | Val loss / ppl | **6.3458** / **570.11** |
37
+ | Train loss (latest) | 5.7024 |
38
+ | Throughput | 76,417 compute tok/s · 0.128 steps/s |
39
+ | Session elapsed | 0d 01h 10m · ETA 2d 12h 38m |
40
+ | Checkpoint written | 2026-11-14T19:44:14Z |
41
 
42
  ## How to load
43
 
 
98
 
99
  ## Notes
100
 
101
+ - This is a **work-in-progress** checkpoint at 1.75% of the planned run, not a finished model: expect repetition and incoherence, especially with few-shot prompts.
102
  - The tokenizer comes from the base model the run started from; the vocabulary is a plain BPE model and is not extended during training.
103
  - `training_meta.json` is regenerated on every push and is the authoritative record of how far training got.
config.json CHANGED
@@ -113,12 +113,12 @@
113
  },
114
  "model_name": "Oleander-150M",
115
  "training": {
116
- "step": 6500,
117
- "kind": "export",
118
- "max_steps": 28820,
119
- "percent_complete": 22.554,
120
- "tokens_seen": 20786970624,
121
- "compute_tokens_seen": 3464495104
122
  },
123
  "transformers_version": "5.14.1",
124
  "torch_dtype": "float32"
 
113
  },
114
  "model_name": "Oleander-150M",
115
  "training": {
116
+ "step": 500,
117
+ "kind": "round 1",
118
+ "max_steps": 28500,
119
+ "percent_complete": 1.754,
120
+ "tokens_seen": 1572864000,
121
+ "compute_tokens_seen": 262144000
122
  },
123
  "transformers_version": "5.14.1",
124
  "torch_dtype": "float32"
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:82e501f73d4ea04ee8d6ef32331c79d7c1b1c6b094e9bc314e141100572d7810
3
  size 599970860
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:26885fc07d17789bef17e9d5bf7386ca750bd6d550ba418423272159432fe842
3
  size 599970860
progress.json CHANGED
@@ -1,15 +1,15 @@
1
  {
2
- "step": 6500,
3
- "max_steps": 28820,
4
- "percent_complete": 22.554,
5
- "tokens_seen": 20786970624,
6
- "compute_tokens_seen": 3464495104,
7
- "compute_tokens_total": 15109980160,
8
- "percent_tokens": 22.929,
9
- "val_loss": 5.754532158374786,
10
- "val_ppl": 315.6178538085929,
11
  "params_M": 149.976,
12
- "pushed_at": "2026-11-14T17:23:50Z",
13
  "architecture": "Oleander-150M",
14
  "features": [
15
  "GDN2 (gated delta net 2) x21: 9h x 64, conv4",
@@ -25,5 +25,5 @@
25
  ],
26
  "optimizer": "muon_adamw",
27
  "phase": "tst",
28
- "kind": "export"
29
  }
 
1
  {
2
+ "step": 500,
3
+ "max_steps": 28500,
4
+ "percent_complete": 1.754,
5
+ "tokens_seen": 1572864000,
6
+ "compute_tokens_seen": 262144000,
7
+ "compute_tokens_total": 14942208000,
8
+ "percent_tokens": 1.754,
9
+ "val_loss": 6.345822848379612,
10
+ "val_ppl": 570.1063078163132,
11
  "params_M": 149.976,
12
+ "pushed_at": "2026-11-14T19:44:14Z",
13
  "architecture": "Oleander-150M",
14
  "features": [
15
  "GDN2 (gated delta net 2) x21: 9h x 64, conv4",
 
25
  ],
26
  "optimizer": "muon_adamw",
27
  "phase": "tst",
28
+ "kind": "round 1"
29
  }
train_config.json CHANGED
@@ -155,10 +155,10 @@
155
  "profile": true
156
  },
157
  "diloco": {
158
- "enabled": false,
159
  "is_leader": true,
160
  "worker_id": 0,
161
- "n_workers": 2,
162
  "inner_steps": 500,
163
  "outer_lr": 0.7,
164
  "outer_momentum": 0.9,
 
155
  "profile": true
156
  },
157
  "diloco": {
158
+ "enabled": true,
159
  "is_leader": true,
160
  "worker_id": 0,
161
+ "n_workers": 1,
162
  "inner_steps": 500,
163
  "outer_lr": 0.7,
164
  "outer_momentum": 0.9,
training_meta.json CHANGED
@@ -1,10 +1,10 @@
1
  {
2
  "format": "oleander-training-meta/1",
3
- "generated_at": "2026-11-14T17:23:50Z",
4
- "kind": "export",
5
- "step": 6500,
6
- "tokens_seen": 20786970624,
7
- "pushed_at": "2026-11-14T17:23:50Z",
8
  "params_M": 149.976,
9
  "architecture": "Oleander-150M",
10
  "features": [
@@ -20,55 +20,55 @@
20
  "TST (token superposition training, bag 6)"
21
  ],
22
  "optimizer": "muon_adamw",
23
- "val_loss": 5.754532158374786,
24
- "val_ppl": 315.6178538085929,
25
  "run": {
26
  "model_name": "Oleander-150M",
27
  "architecture_detail": "Oleander-150M (28L = 7 x [GDN2 \u00b7 GDN2 \u00b7 GDN2 \u00b7 GQA] \u00b7 d=576 \u00b7 SwiGLU(1600) \u00b7 tied 32,770 vocab)",
28
- "session_id": "20261114_055200_5e72cc",
29
  "repo_id": "CyanMonkey/Test-Subject-3",
30
- "started_at": "2026-11-14T05:59:52Z",
31
- "session_elapsed_s": 41038.1,
32
- "last_update_at": "2026-11-14T17:23:45Z",
33
- "steps_per_second": 0.14465,
34
- "compute_tokens_per_second": 77156.3,
35
- "eta": "1d 18h 51m",
36
- "checkpoint_kind": "export"
37
  },
38
  "progress": {
39
- "step": 6500,
40
- "max_steps": 28820,
41
- "step_fraction": 0.225538,
42
- "percent_complete": 22.554,
43
- "tokens_fraction": 0.229285,
44
- "percent_tokens": 22.929,
45
  "phase": "tst",
46
- "phase_fraction": 0.751793,
47
- "tst_switch_step": 8646,
48
  "tst_switch_applied": false
49
  },
50
  "tokens": {
51
- "compute_tokens_seen": 3464495104,
52
- "compute_tokens_total": 15109980160,
53
- "raw_tokens_seen": 20786970624,
54
- "raw_tokens_total": 37774950400,
55
  "raw_over_compute_ratio": 6.0,
56
- "samples_seen": 19968000,
57
- "rows_consumed": 19968000,
58
  "compute_tokens_per_step": 524288,
59
  "cluster": {
60
  "n_workers": 1,
61
- "compute_tokens_seen_all_workers": 3464495104,
62
- "raw_tokens_seen_all_workers": 20786970624,
63
- "compute_tokens_total_planned_all_workers": 15109980160,
64
- "raw_tokens_total_planned_all_workers": 37774950400
65
  }
66
  },
67
  "metrics": {
68
- "train_loss": 5.219560623168945,
69
- "val_loss": 5.754532158374786,
70
- "val_ppl": 315.6178538085929,
71
- "grad_norm": 0.15032941102981567,
72
  "lr_mult": 1.0
73
  },
74
  "optimization": {
@@ -98,13 +98,13 @@
98
  "stable_until": 0.825,
99
  "decay_shape": "1-sqrt",
100
  "min_lr_ratio": 0.0,
101
- "decay_start_step": 23776
102
  },
103
  "tst": {
104
  "enabled": true,
105
  "bag_size": 6,
106
  "phase_fraction": 0.3,
107
- "switch_step": 8646
108
  }
109
  },
110
  "model": {
@@ -151,7 +151,7 @@
151
  "doc_masking": "auto",
152
  "shuffle_block_rows": 192,
153
  "seed": 42,
154
- "epochs_consumed_this_worker": 1.3534
155
  },
156
  "hardware": {
157
  "device": "cuda",
@@ -162,10 +162,10 @@
162
  "transformers": "5.14.1",
163
  "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
164
  "vram_gb": 101.98,
165
- "peak_gb": 38.76
166
  },
167
  "checkpoint": {
168
- "kind": "export",
169
  "weights_file": "model.safetensors",
170
  "dtype": "float32",
171
  "config_file": "config.json",
@@ -186,7 +186,16 @@
186
  ]
187
  },
188
  "diloco": {
189
- "enabled": false
 
 
 
 
 
 
 
190
  },
191
- "extra": {}
 
 
192
  }
 
1
  {
2
  "format": "oleander-training-meta/1",
3
+ "generated_at": "2026-11-14T19:44:14Z",
4
+ "kind": "round 1",
5
+ "step": 500,
6
+ "tokens_seen": 1572864000,
7
+ "pushed_at": "2026-11-14T19:44:14Z",
8
  "params_M": 149.976,
9
  "architecture": "Oleander-150M",
10
  "features": [
 
20
  "TST (token superposition training, bag 6)"
21
  ],
22
  "optimizer": "muon_adamw",
23
+ "val_loss": 6.345822848379612,
24
+ "val_ppl": 570.1063078163132,
25
  "run": {
26
  "model_name": "Oleander-150M",
27
  "architecture_detail": "Oleander-150M (28L = 7 x [GDN2 \u00b7 GDN2 \u00b7 GDN2 \u00b7 GQA] \u00b7 d=576 \u00b7 SwiGLU(1600) \u00b7 tied 32,770 vocab)",
28
+ "session_id": "20261114_181817_2de909",
29
  "repo_id": "CyanMonkey/Test-Subject-3",
30
+ "started_at": "2026-11-14T18:33:29Z",
31
+ "session_elapsed_s": 4245.2,
32
+ "last_update_at": "2026-11-14T19:44:13Z",
33
+ "steps_per_second": 0.12827,
34
+ "compute_tokens_per_second": 76416.7,
35
+ "eta": "2d 12h 38m",
36
+ "checkpoint_kind": "round 1"
37
  },
38
  "progress": {
39
+ "step": 500,
40
+ "max_steps": 28500,
41
+ "step_fraction": 0.017544,
42
+ "percent_complete": 1.754,
43
+ "tokens_fraction": 0.017544,
44
+ "percent_tokens": 1.754,
45
  "phase": "tst",
46
+ "phase_fraction": 0.055556,
47
+ "tst_switch_step": 9000,
48
  "tst_switch_applied": false
49
  },
50
  "tokens": {
51
+ "compute_tokens_seen": 262144000,
52
+ "compute_tokens_total": 14942208000,
53
+ "raw_tokens_seen": 1572864000,
54
+ "raw_tokens_total": 38535168000,
55
  "raw_over_compute_ratio": 6.0,
56
+ "samples_seen": 1536000,
57
+ "rows_consumed": 1536000,
58
  "compute_tokens_per_step": 524288,
59
  "cluster": {
60
  "n_workers": 1,
61
+ "compute_tokens_seen_all_workers": 262144000,
62
+ "raw_tokens_seen_all_workers": 1572864000,
63
+ "compute_tokens_total_planned_all_workers": 14942208000,
64
+ "raw_tokens_total_planned_all_workers": 38535168000
65
  }
66
  },
67
  "metrics": {
68
+ "train_loss": 5.702352523803711,
69
+ "val_loss": 6.345822848379612,
70
+ "val_ppl": 570.1063078163132,
71
+ "grad_norm": 0.34867212176322937,
72
  "lr_mult": 1.0
73
  },
74
  "optimization": {
 
98
  "stable_until": 0.825,
99
  "decay_shape": "1-sqrt",
100
  "min_lr_ratio": 0.0,
101
+ "decay_start_step": 23512
102
  },
103
  "tst": {
104
  "enabled": true,
105
  "bag_size": 6,
106
  "phase_fraction": 0.3,
107
+ "switch_step": 9000
108
  }
109
  },
110
  "model": {
 
151
  "doc_masking": "auto",
152
  "shuffle_block_rows": 192,
153
  "seed": 42,
154
+ "epochs_consumed_this_worker": 0.1041
155
  },
156
  "hardware": {
157
  "device": "cuda",
 
162
  "transformers": "5.14.1",
163
  "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
164
  "vram_gb": 101.98,
165
+ "peak_gb": 39.36
166
  },
167
  "checkpoint": {
168
+ "kind": "round 1",
169
  "weights_file": "model.safetensors",
170
  "dtype": "float32",
171
  "config_file": "config.json",
 
186
  ]
187
  },
188
  "diloco": {
189
+ "enabled": true,
190
+ "worker_id": 0,
191
+ "n_workers": 1,
192
+ "inner_steps": 500,
193
+ "outer_lr": 0.7,
194
+ "outer_momentum": 0.9,
195
+ "is_leader": true,
196
+ "support_repo_id": "CyanMonkey/TestSubject-3-support"
197
  },
198
+ "extra": {
199
+ "diloco_round": 1
200
+ }
201
  }