{
  "model": {
    "embedding_config": {
      "context_size": 1024,
      "vocab_size": 32000,
      "d_model": 768,
      "dropout_prob": 0.1
    },
    "transformer_config": {
      "mha_config": {
        "d_model": 768,
        "num_heads": 12,
        "dropout_prob": 0.1
      },
      "ff_expansion_factor": 4,
      "activation": "Gelu",
      "dropout_prob": 0.1
    },
    "num_transformer_layers": 12
  },
  "optimizer": {
    "grad_clipping": null,
    "beta_1": 0.9,
    "beta_2": 0.999,
    "epsilon": 0.00001,
    "weight_decay": 0.0001,
    "cautious_weight_decay": false,
    "amsgrad": false
  },
  "tokenizer_kind": "bpe",
  "tokenizer_path": "tokenizer.json",
  "num_epochs": 200,
  "batch_size": 6,
  "num_workers": 6,
  "seed": 123456,
  "learning_rate": 0.0001
}