{
  "model": "amor_gdn",
  "scale": "1.5b",
  "backbone_config": {
    "d_model": 2048,
    "n_layer": 24,
    "d_ff": 4096,
    "d_state": 128,
    "expand": 2,
    "head_dim": 64,
    "d_conv": 4,
    "n_groups": 1,
    "peak_lr": 0.00015
  },
  "seq_len": 3072,
  "batch_size": 2,
  "grad_accum": 80,
  "world_size": 1,
  "total_tokens": 30748518400,
  "total_steps": 62558,
  "warmup_steps": 2000,
  "peak_lr": 0.00015,
  "min_lr": 1e-05,
  "chinchilla_multiplier": 20,
  "seed": 42,
  "vocab_size": 128256,
  "tokenizer": "teknium/Llama-3.1-AlternateTokenizer",
  "n_params": 1524025472,
  "amor": {
    "lr_alpha": 0.003,
    "init_alpha": 0.0,
    "init_threshold": 0.3,
    "gate_offset": 0.03,
    "offset_mode": "adaptive",
    "offset_k": 0.2
  }
}