File size: 1,100 Bytes
0f70a95 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | {
"variant": "Weevil-3M",
"source_checkpoint": "reports/latent-gdn2-sub3m-tst-8gpu-10b-20261023/step-00038147",
"source_model_sha256": "17715c39cc5fb56fd37bd655e98082991440cdc7d462160bfc2678bd0cd8263b",
"source_step": 38147,
"parameters": 2993576,
"model_spec": {
"name": "weevil_latent_gdn2_sub3m_v1",
"model_class": "WeevilSimpleParallel",
"layers_per_stack": 8,
"constructor": {
"vocab_size": 4096,
"embedding_dim": 128,
"intermediate_size": 432
},
"gdn2_config": {
"hidden_size": 144,
"num_heads": 3,
"key_head_dim": 48,
"value_head_dim": 48,
"qkv_rank": 48,
"conv_kernel": 4,
"norm_eps": 1e-06,
"compute_dtype": "bfloat16",
"mixer": "chunk",
"chunk_size": null,
"prefix_block_size": null
},
"parameter_budget": {
"exclusive_upper_limit": 3000000,
"expected_total": 2993576,
"embedding": 542720,
"mlp": 1492992,
"mixers": 956568,
"normalization": 1296,
"qkv_subset_of_mixers": 221184
}
},
"weights_dtype": "float32"
}
|