| { |
| "format": "oleander-training-meta/1", |
| "generated_at": "2026-11-13T07:27:27Z", |
| "kind": "export", |
| "step": 6500, |
| "tokens_seen": 20786970624, |
| "pushed_at": "2026-11-13T07:27:27Z", |
| "params_M": 149.976, |
| "architecture": "Oleander-150M", |
| "features": [ |
| "GDN2 (gated delta net 2) x21: 9h x 64, conv4", |
| "gated GQA x7: 9 Q / 3 KV heads, head_dim 64", |
| "QK-Norm (offset form)", |
| "partial RoPE (rotary_dim 32, theta 100000)", |
| "value residual (v_mix)", |
| "SwiGLU", |
| "RMSNorm", |
| "AttnRes (block attention residuals)", |
| "tied embeddings", |
| "TST (token superposition training, bag 6)" |
| ], |
| "optimizer": "muon_adamw", |
| "val_loss": 5.523073740303516, |
| "val_ppl": 250.40353093471919, |
| "run": { |
| "model_name": "Oleander-150M", |
| "architecture_detail": "Oleander-150M (28L = 7 x [GDN2 \u00b7 GDN2 \u00b7 GDN2 \u00b7 GQA] \u00b7 d=576 \u00b7 SwiGLU(1600) \u00b7 tied 32,770 vocab)", |
| "session_id": "20261112_195408_1a765b", |
| "repo_id": "CyanMonkey/Test-Subject-3", |
| "started_at": "2026-11-12T20:04:22Z", |
| "session_elapsed_s": 40985.1, |
| "last_update_at": "2026-11-13T07:27:25Z", |
| "steps_per_second": 0.14454, |
| "compute_tokens_per_second": 77099.8, |
| "eta": "1d 18h 53m", |
| "checkpoint_kind": "export" |
| }, |
| "progress": { |
| "step": 6500, |
| "max_steps": 28820, |
| "step_fraction": 0.225538, |
| "percent_complete": 22.554, |
| "tokens_fraction": 0.229285, |
| "percent_tokens": 22.929, |
| "phase": "tst", |
| "phase_fraction": 0.751793, |
| "tst_switch_step": 8646, |
| "tst_switch_applied": false |
| }, |
| "tokens": { |
| "compute_tokens_seen": 3464495104, |
| "compute_tokens_total": 15109980160, |
| "raw_tokens_seen": 20786970624, |
| "raw_tokens_total": 37774950400, |
| "raw_over_compute_ratio": 6.0, |
| "samples_seen": 19968000, |
| "rows_consumed": 19968000, |
| "compute_tokens_per_step": 524288, |
| "cluster": { |
| "n_workers": 1, |
| "compute_tokens_seen_all_workers": 3464495104, |
| "raw_tokens_seen_all_workers": 20786970624, |
| "compute_tokens_total_planned_all_workers": 15109980160, |
| "raw_tokens_total_planned_all_workers": 37774950400 |
| } |
| }, |
| "metrics": { |
| "train_loss": 5.220016002655029, |
| "val_loss": 5.523073740303516, |
| "val_ppl": 250.40353093471919, |
| "grad_norm": 0.3239007890224457, |
| "lr_mult": 1.0 |
| }, |
| "optimization": { |
| "name": "muon_adamw", |
| "description": "MuonEq-R (muoneq_r, lr 0.02, 5 NS steps, momentum 0.95) + AdamW (lr 0.0015, betas (0.9, 0.95))", |
| "muon": { |
| "variant": "muoneq_r", |
| "lr": 0.02, |
| "momentum": 0.95, |
| "nesterov": true, |
| "ns_steps": 5, |
| "weight_decay": 0.01 |
| }, |
| "adamw": { |
| "lr": 0.0015, |
| "betas": [ |
| 0.9, |
| 0.95 |
| ], |
| "eps": 1e-08, |
| "weight_decay": 0.0 |
| }, |
| "grad_clip": 1.0, |
| "schedule": { |
| "name": "WSD", |
| "warmup_steps": 100, |
| "stable_until": 0.825, |
| "decay_shape": "1-sqrt", |
| "min_lr_ratio": 0.0, |
| "decay_start_step": 23776 |
| }, |
| "tst": { |
| "enabled": true, |
| "bag_size": 6, |
| "phase_fraction": 0.3, |
| "switch_step": 8646 |
| } |
| }, |
| "model": { |
| "params": 149976139, |
| "params_M": 149.976, |
| "num_layers": 28, |
| "hidden_size": 576, |
| "group_pattern": [ |
| "gdn2", |
| "gdn2", |
| "gdn2", |
| "gqa" |
| ], |
| "num_groups": 7, |
| "mlp_hidden": 1600, |
| "vocab_size": 32770, |
| "max_seq_len": 1024, |
| "attn_res": true, |
| "residual_dtype": "bfloat16", |
| "tie_embeddings": true, |
| "features": [ |
| "GDN2 (gated delta net 2) x21: 9h x 64, conv4", |
| "gated GQA x7: 9 Q / 3 KV heads, head_dim 64", |
| "QK-Norm (offset form)", |
| "partial RoPE (rotary_dim 32, theta 100000)", |
| "value residual (v_mix)", |
| "SwiGLU", |
| "RMSNorm", |
| "AttnRes (block attention residuals)", |
| "tied embeddings", |
| "TST (token superposition training, bag 6)" |
| ], |
| "export_dtype": "float32" |
| }, |
| "data": { |
| "dataset_repo_id": "CyanMonkey/NextGen-1-32-1024", |
| "train_split": "train", |
| "tokenizer_id": "AxiomicLabs/GPT-X2.5-135M", |
| "n_train_rows": 14753933, |
| "n_val_rows": 2048, |
| "seq_len": 1024, |
| "micro_batch": 32, |
| "grad_accum_steps": 16, |
| "doc_masking": "auto", |
| "shuffle_block_rows": 192, |
| "seed": 42, |
| "epochs_consumed_this_worker": 1.3534 |
| }, |
| "hardware": { |
| "device": "cuda", |
| "torch": "2.11.0+cu130", |
| "amp_dtype": "torch.bfloat16", |
| "preset": "full", |
| "python": "3.13.11", |
| "transformers": "5.14.1", |
| "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", |
| "vram_gb": 101.98, |
| "peak_gb": 38.76 |
| }, |
| "checkpoint": { |
| "kind": "export", |
| "weights_file": "model.safetensors", |
| "dtype": "float32", |
| "config_file": "config.json", |
| "auto_map": { |
| "AutoConfig": "configuration_oleander.OleanderConfig", |
| "AutoModelForCausalLM": "modeling_oleander.OleanderForCausalLM" |
| }, |
| "modeling_files": [], |
| "tokenizer_files": [ |
| "tokenizer.json", |
| "tokenizer_config.json", |
| "special_tokens_map.json", |
| "vocab.json", |
| "merges.txt", |
| "added_tokens.json", |
| "chat_template.jinja", |
| "tokenizer.model" |
| ] |
| }, |
| "diloco": { |
| "enabled": false |
| }, |
| "extra": {} |
| } |
