{
  "preset": "full",
  "mode": "train",
  "hf": {
    "token": "<redacted>",
    "output_repo_id": "CyanMonkey/Test-Subject-3",
    "private": false,
    "push": true,
    "resume_from_hub": true,
    "auto_map": true
  },
  "data": {
    "source": "hub",
    "dataset_repo_id": "CyanMonkey/NextGen-1-32-1024",
    "train_split": "train",
    "val_split": null,
    "val_rows": 2048,
    "max_train_rows": null,
    "tokenizer_id": "AxiomicLabs/GPT-X2.5-135M",
    "doc_masking": "auto",
    "shuffle_block_rows": 192,
    "prefetch": 6,
    "seed": 42
  },
  "model": {
    "arch_name": "OleanderForCausalLM",
    "vocab_size": 32770,
    "hidden_size": 576,
    "num_groups": 7,
    "group_pattern": [
      "gdn2",
      "gdn2",
      "gdn2",
      "gqa"
    ],
    "mlp_hidden": 1600,
    "max_seq_len": 1024,
    "norm_eps": 1e-06,
    "init_std": 0.02,
    "tie_embeddings": true,
    "attn_res": true,
    "residual_dtype": "bfloat16",
    "gdn2": {
      "heads": 9,
      "head_dim": 64,
      "conv_size": 4,
      "norm_eps": 1e-05,
      "allow_neg_eigval": false,
      "gate_in_kernel": false
    },
    "gqa": {
      "q_heads": 9,
      "kv_heads": 3,
      "head_dim": 64,
      "rotary_dim": 32,
      "rope_theta": 100000.0,
      "qk_norm": true,
      "value_residual": true
    }
  },
  "train": {
    "micro_batch": 32,
    "grad_accum_steps": 16,
    "max_steps": 28820,
    "grad_clip": 1.0,
    "seed": 42,
    "dtype": "auto"
  },
  "optim": {
    "muon_variant": "muoneq_r",
    "muon_lr": 0.02,
    "muon_momentum": 0.95,
    "muon_nesterov": true,
    "muon_ns_steps": 5,
    "muon_wd": 0.01,
    "skinny_to_adamw": false,
    "adamw_lr": 0.0015,
    "adamw_betas": [
      0.9,
      0.95
    ],
    "adamw_eps": 1e-08,
    "adamw_wd": 0.0
  },
  "schedule": {
    "warmup_steps": 100,
    "stable_until": 0.825,
    "decay_shape": "1-sqrt",
    "min_lr_ratio": 0.0
  },
  "tst": {
    "enabled": true,
    "phase_fraction": 0.3,
    "bag_size": 6,
    "alog_shift_at_switch": false,
    "reset_opt_at_switch": false
  },
  "loss": {
    "chunk_tokens": 8192,
    "z_loss": 0.0
  },
  "compile": {
    "enabled": true,
    "mode": "default",
    "precompile": true,
    "fallback_to_eager": true
  },
  "generation": {
    "enabled": true,
    "every_n_steps": 1000,
    "prompt": "Once upon a time",
    "max_new_tokens": 64,
    "temperature": 0.8,
    "top_k": 50
  },
  "save": {
    "work_dir": "./oleander_run",
    "log_steps": 20,
    "eval_steps": 500,
    "eval_batches": 64,
    "save_steps": 500,
    "keep_last_n": 2,
    "push_steps": 500,
    "resume_push_steps": 500,
    "export_dtype": "float32",
    "cpt_checkpoint": true
  },
  "export": {
    "model_name": "Oleander-150M",
    "license": "apache-2.0",
    "write_modeling_files": true,
    "write_tokenizer_files": true,
    "write_readme": true,
    "write_training_meta": true,
    "write_generation_config": true,
    "meta_extra": {},
    "tokenizer_files": [
      "tokenizer.json",
      "tokenizer_config.json",
      "special_tokens_map.json",
      "vocab.json",
      "merges.txt",
      "added_tokens.json",
      "chat_template.jinja",
      "tokenizer.model"
    ]
  },
  "bench": {
    "micro_batches": [
      16,
      32
    ],
    "iters": 8,
    "warmup": 3,
    "profile": true
  },
  "diloco": {
    "enabled": false,
    "is_leader": true,
    "worker_id": 0,
    "n_workers": 2,
    "inner_steps": 500,
    "outer_lr": 0.7,
    "outer_momentum": 0.9,
    "outer_nesterov": true,
    "delta_dtype": "bfloat16",
    "min_workers": 2,
    "quorum_grace_s": 600,
    "poll_s": 20,
    "outlier_factor": 3.0,
    "support_repo_id": "CyanMonkey/TestSubject-3-support",
    "transport": "hub",
    "local_dir": "./oleander_dl_local",
    "reset_support": false,
    "force_leader": false,
    "lease_ttl_s": 900,
    "lease_beat_s": 300,
    "keep_rounds": 2,
    "outer_state_rounds": 4,
    "opt_snapshot_rounds": 4,
    "ramp_steps": 50,
    "og_push_rounds": 1,
    "sample_every_rounds": 5,
    "squash_every_rounds": 25
  }
}