config.json

{
  "kind": "diffusion",
  "model": {
    "vocab_size": 50368,
    "max_seq_len": 2048,
    "d_model": 768,
    "n_layers": 22,
    "n_heads": 12,
    "d_ff": 1152,
    "dropout": 0.0,
    "embedding_dropout": 0.0,
    "mlp_dropout": 0.0,
    "attention_dropout": 0.0,
    "attention_bias": false,
    "mlp_bias": false,
    "norm_bias": false,
    "prediction_bias": false,
    "decoder_bias": true,
    "norm_eps": 1e-05,
    "local_attention": 128,
    "global_attn_every_n_layers": 3,
    "local_rope_theta": 160000.0,
    "global_rope_theta": 160000.0,
    "initializer_range": 0.02,
    "initializer_cutoff_factor": 2.0,
    "pad_token_id": 50283,
    "unk_token_id": 50280,
    "bos_token_id": 50281,
    "eos_token_id": 50282,
    "mask_token_id": 50284,
    "label_ignore_index": -100,
    "mlm_probability": 0.15,
    "diffusion_steps": 1000,
    "do_not_mask_token_ids": [
      50279,
      50280,
      50281,
      50282,
      50283,
      50284
    ]
  },
  "training": {
    "rope_fp32": true
  },
  "format": "pretrain",
  "tokens_seen": 56998518784
}