| { | |
| "variant": "Weevil-3M-QA", | |
| "source_checkpoint": "reports/qa-latent-sub3m-90-10-wsd-8gpu-20261023/step-00014000", | |
| "source_model_sha256": "b42ea63e35e74c87718ef4465a7b5deb650a6e8a7ca328d6161cd35a3e15876b", | |
| "source_step": 14000, | |
| "parameters": 2993576, | |
| "model_spec": { | |
| "name": "weevil_latent_gdn2_sub3m_v1", | |
| "model_class": "WeevilSimpleParallel", | |
| "layers_per_stack": 8, | |
| "constructor": { | |
| "vocab_size": 4096, | |
| "embedding_dim": 128, | |
| "intermediate_size": 432 | |
| }, | |
| "gdn2_config": { | |
| "hidden_size": 144, | |
| "num_heads": 3, | |
| "key_head_dim": 48, | |
| "value_head_dim": 48, | |
| "qkv_rank": 48, | |
| "conv_kernel": 4, | |
| "norm_eps": 1e-06, | |
| "compute_dtype": "bfloat16", | |
| "mixer": "chunk", | |
| "chunk_size": null, | |
| "prefix_block_size": null | |
| }, | |
| "parameter_budget": { | |
| "exclusive_upper_limit": 3000000, | |
| "expected_total": 2993576, | |
| "embedding": 542720, | |
| "mlp": 1492992, | |
| "mixers": 956568, | |
| "normalization": 1296, | |
| "qkv_subset_of_mixers": 221184 | |
| } | |
| }, | |
| "weights_dtype": "float32" | |
| } | |