File size: 500 Bytes
3f8f837 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 | default_model: m2
m2:
arch: moe_2
embed_dim: 128
block: 4
head: 4
num_mha: 3
num_ffn: 2
steps: 2
dropout: 0.001
enable_mask: true
model_save_path: "models/TF/M2_1.keras"
batch_size: 512
learning_rate: 2.0e-4
label_smoothing: 0.00001
m2s:
arch: moe_2
embed_dim: 96
block: 3
head: 3
num_mha: 2
num_ffn: 2
steps: 1
dropout: 0.0001
enable_mask: true
model_save_path: "models/TF/M2s.keras"
batch_size: 512
learning_rate: 1.0e-4
label_smoothing: 0.0001 |