| default_model: m2 | |
| m2: | |
| arch: moe_2 | |
| embed_dim: 128 | |
| block: 4 | |
| head: 4 | |
| num_mha: 3 | |
| num_ffn: 2 | |
| steps: 2 | |
| dropout: 0.001 | |
| enable_mask: true | |
| model_save_path: "models/TF/M2_1.keras" | |
| batch_size: 512 | |
| learning_rate: 2.0e-4 | |
| label_smoothing: 0.00001 | |
| m2s: | |
| arch: moe_2 | |
| embed_dim: 96 | |
| block: 3 | |
| head: 3 | |
| num_mha: 2 | |
| num_ffn: 2 | |
| steps: 1 | |
| dropout: 0.0001 | |
| enable_mask: true | |
| model_save_path: "models/TF/M2s.keras" | |
| batch_size: 512 | |
| learning_rate: 1.0e-4 | |
| label_smoothing: 0.0001 |