default_model: m2

m2:
  arch: moe_2
  embed_dim: 128
  block: 4
  head: 4
  num_mha: 3
  num_ffn: 2
  steps: 2
  dropout: 0.001
  enable_mask: true
  model_save_path: "models/TF/M2_1.keras"
  batch_size: 512
  learning_rate: 2.0e-4
  label_smoothing: 0.00001

m2s:
  arch: moe_2
  embed_dim: 96
  block: 3
  head: 3
  num_mha: 2
  num_ffn: 2
  steps: 1
  dropout: 0.0001
  enable_mask: true
  model_save_path: "models/TF/M2s.keras"
  batch_size: 512
  learning_rate: 1.0e-4
  label_smoothing: 0.0001