File size: 394 Bytes
ccf447d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | {
"activation": "gelu",
"bias_free_ln": false,
"cell_m": 4096,
"col_feature_group_size": 3,
"col_nhead": 8,
"col_num_blocks": 3,
"col_num_inds": 128,
"dropout": 0.0,
"embed_dim": 128,
"ff_factor": 2,
"icl_nhead": 8,
"icl_num_blocks": 12,
"max_classes": 10,
"row_nhead": 8,
"row_num_blocks": 3,
"row_num_cls": 4,
"row_rope_base": 100000,
"zero_init": false
}
|