{
  "format": "oleander-training-meta/1",
  "generated_at": "2026-11-13T07:27:27Z",
  "kind": "export",
  "step": 6500,
  "tokens_seen": 20786970624,
  "pushed_at": "2026-11-13T07:27:27Z",
  "params_M": 149.976,
  "architecture": "Oleander-150M",
  "features": [
    "GDN2 (gated delta net 2) x21: 9h x 64, conv4",
    "gated GQA x7: 9 Q / 3 KV heads, head_dim 64",
    "QK-Norm (offset form)",
    "partial RoPE (rotary_dim 32, theta 100000)",
    "value residual (v_mix)",
    "SwiGLU",
    "RMSNorm",
    "AttnRes (block attention residuals)",
    "tied embeddings",
    "TST (token superposition training, bag 6)"
  ],
  "optimizer": "muon_adamw",
  "val_loss": 5.523073740303516,
  "val_ppl": 250.40353093471919,
  "run": {
    "model_name": "Oleander-150M",
    "architecture_detail": "Oleander-150M (28L = 7 x [GDN2 \u00b7 GDN2 \u00b7 GDN2 \u00b7 GQA] \u00b7 d=576 \u00b7 SwiGLU(1600) \u00b7 tied 32,770 vocab)",
    "session_id": "20261112_195408_1a765b",
    "repo_id": "CyanMonkey/Test-Subject-3",
    "started_at": "2026-11-12T20:04:22Z",
    "session_elapsed_s": 40985.1,
    "last_update_at": "2026-11-13T07:27:25Z",
    "steps_per_second": 0.14454,
    "compute_tokens_per_second": 77099.8,
    "eta": "1d 18h 53m",
    "checkpoint_kind": "export"
  },
  "progress": {
    "step": 6500,
    "max_steps": 28820,
    "step_fraction": 0.225538,
    "percent_complete": 22.554,
    "tokens_fraction": 0.229285,
    "percent_tokens": 22.929,
    "phase": "tst",
    "phase_fraction": 0.751793,
    "tst_switch_step": 8646,
    "tst_switch_applied": false
  },
  "tokens": {
    "compute_tokens_seen": 3464495104,
    "compute_tokens_total": 15109980160,
    "raw_tokens_seen": 20786970624,
    "raw_tokens_total": 37774950400,
    "raw_over_compute_ratio": 6.0,
    "samples_seen": 19968000,
    "rows_consumed": 19968000,
    "compute_tokens_per_step": 524288,
    "cluster": {
      "n_workers": 1,
      "compute_tokens_seen_all_workers": 3464495104,
      "raw_tokens_seen_all_workers": 20786970624,
      "compute_tokens_total_planned_all_workers": 15109980160,
      "raw_tokens_total_planned_all_workers": 37774950400
    }
  },
  "metrics": {
    "train_loss": 5.220016002655029,
    "val_loss": 5.523073740303516,
    "val_ppl": 250.40353093471919,
    "grad_norm": 0.3239007890224457,
    "lr_mult": 1.0
  },
  "optimization": {
    "name": "muon_adamw",
    "description": "MuonEq-R (muoneq_r, lr 0.02, 5 NS steps, momentum 0.95) + AdamW (lr 0.0015, betas (0.9, 0.95))",
    "muon": {
      "variant": "muoneq_r",
      "lr": 0.02,
      "momentum": 0.95,
      "nesterov": true,
      "ns_steps": 5,
      "weight_decay": 0.01
    },
    "adamw": {
      "lr": 0.0015,
      "betas": [
        0.9,
        0.95
      ],
      "eps": 1e-08,
      "weight_decay": 0.0
    },
    "grad_clip": 1.0,
    "schedule": {
      "name": "WSD",
      "warmup_steps": 100,
      "stable_until": 0.825,
      "decay_shape": "1-sqrt",
      "min_lr_ratio": 0.0,
      "decay_start_step": 23776
    },
    "tst": {
      "enabled": true,
      "bag_size": 6,
      "phase_fraction": 0.3,
      "switch_step": 8646
    }
  },
  "model": {
    "params": 149976139,
    "params_M": 149.976,
    "num_layers": 28,
    "hidden_size": 576,
    "group_pattern": [
      "gdn2",
      "gdn2",
      "gdn2",
      "gqa"
    ],
    "num_groups": 7,
    "mlp_hidden": 1600,
    "vocab_size": 32770,
    "max_seq_len": 1024,
    "attn_res": true,
    "residual_dtype": "bfloat16",
    "tie_embeddings": true,
    "features": [
      "GDN2 (gated delta net 2) x21: 9h x 64, conv4",
      "gated GQA x7: 9 Q / 3 KV heads, head_dim 64",
      "QK-Norm (offset form)",
      "partial RoPE (rotary_dim 32, theta 100000)",
      "value residual (v_mix)",
      "SwiGLU",
      "RMSNorm",
      "AttnRes (block attention residuals)",
      "tied embeddings",
      "TST (token superposition training, bag 6)"
    ],
    "export_dtype": "float32"
  },
  "data": {
    "dataset_repo_id": "CyanMonkey/NextGen-1-32-1024",
    "train_split": "train",
    "tokenizer_id": "AxiomicLabs/GPT-X2.5-135M",
    "n_train_rows": 14753933,
    "n_val_rows": 2048,
    "seq_len": 1024,
    "micro_batch": 32,
    "grad_accum_steps": 16,
    "doc_masking": "auto",
    "shuffle_block_rows": 192,
    "seed": 42,
    "epochs_consumed_this_worker": 1.3534
  },
  "hardware": {
    "device": "cuda",
    "torch": "2.11.0+cu130",
    "amp_dtype": "torch.bfloat16",
    "preset": "full",
    "python": "3.13.11",
    "transformers": "5.14.1",
    "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
    "vram_gb": 101.98,
    "peak_gb": 38.76
  },
  "checkpoint": {
    "kind": "export",
    "weights_file": "model.safetensors",
    "dtype": "float32",
    "config_file": "config.json",
    "auto_map": {
      "AutoConfig": "configuration_oleander.OleanderConfig",
      "AutoModelForCausalLM": "modeling_oleander.OleanderForCausalLM"
    },
    "modeling_files": [],
    "tokenizer_files": [
      "tokenizer.json",
      "tokenizer_config.json",
      "special_tokens_map.json",
      "vocab.json",
      "merges.txt",
      "added_tokens.json",
      "chat_template.jinja",
      "tokenizer.model"
    ]
  },
  "diloco": {
    "enabled": false
  },
  "extra": {}
}