{
  "eos_id": 50282,
  "expanded_width": 2624,
  "format_version": 1,
  "global_every": 3,
  "global_rope_base": 160000.0,
  "head_activation": "gelu",
  "heads": 16,
  "layers": 29,
  "local_rope_base": 160000.0,
  "local_window": 64,
  "maximum_length": 7999,
  "norm_epsilon": 1e-05,
  "pad_id": 50283,
  "storage_vocabulary_size": 65536,
  "vocabulary_size": 50368,
  "width": 1024
}