| { | |
| "variant": "Weevil-3M", | |
| "source_checkpoint": "reports/latent-gdn2-sub3m-tst-8gpu-10b-20261023/step-00038147", | |
| "source_model_sha256": "17715c39cc5fb56fd37bd655e98082991440cdc7d462160bfc2678bd0cd8263b", | |
| "source_step": 38147, | |
| "parameters": 2993576, | |
| "model_spec": { | |
| "name": "weevil_latent_gdn2_sub3m_v1", | |
| "model_class": "WeevilSimpleParallel", | |
| "layers_per_stack": 8, | |
| "constructor": { | |
| "vocab_size": 4096, | |
| "embedding_dim": 128, | |
| "intermediate_size": 432 | |
| }, | |
| "gdn2_config": { | |
| "hidden_size": 144, | |
| "num_heads": 3, | |
| "key_head_dim": 48, | |
| "value_head_dim": 48, | |
| "qkv_rank": 48, | |
| "conv_kernel": 4, | |
| "norm_eps": 1e-06, | |
| "compute_dtype": "bfloat16", | |
| "mixer": "chunk", | |
| "chunk_size": null, | |
| "prefix_block_size": null | |
| }, | |
| "parameter_budget": { | |
| "exclusive_upper_limit": 3000000, | |
| "expected_total": 2993576, | |
| "embedding": 542720, | |
| "mlp": 1492992, | |
| "mixers": 956568, | |
| "normalization": 1296, | |
| "qkv_subset_of_mixers": 221184 | |
| } | |
| }, | |
| "weights_dtype": "float32" | |
| } | |