training_config.json

{
  "model": "amor_gdn",
  "scale": "440m",
  "backbone_config": {
    "d_model": 1024,
    "n_layer": 24,
    "d_ff": 1984,
    "d_state": 128,
    "expand": 2,
    "head_dim": 64,
    "d_conv": 4,
    "n_groups": 1,
    "peak_lr": 0.0003
  },
  "seq_len": 3072,
  "batch_size": 4,
  "grad_accum": 40,
  "world_size": 1,
  "total_tokens": 8973009920,
  "total_steps": 18255,
  "warmup_steps": 2000,
  "peak_lr": 0.0003,
  "min_lr": 1e-05,
  "chinchilla_multiplier": 20,
  "seed": 42,
  "vocab_size": 128256,
  "tokenizer": "teknium/Llama-3.1-AlternateTokenizer",
  "n_params": 442133056,
  "amor": {
    "lr_alpha": 0.003,
    "init_alpha": 0.0,
    "init_threshold": 0.3,
    "gate_offset": 0.03,
    "offset_mode": "adaptive",
    "offset_k": 0.2
  }
}