| { | |
| "model_type": "transwave_meyer", | |
| "architectures": ["TransWaveForCausalLM"], | |
| "auto_map": { | |
| "AutoConfig": "configuration_transwave.TransWaveConfig", | |
| "AutoModelForCausalLM": "modeling_transwave.TransWaveForCausalLM" | |
| }, | |
| "vocab_size": 256, | |
| "byte_dim": 96, | |
| "bytes_per_block": 8, | |
| "num_scales": 8, | |
| "hidden_size": 768, | |
| "max_position_embeddings": 32768, | |
| "block_context": 4096, | |
| "num_hidden_layers": 12, | |
| "num_groups": 12, | |
| "meyer_bands": 12, | |
| "meyer_controls_per_band": 8, | |
| "meyer_kernel_blocks": 512, | |
| "local_kernel_blocks": 7, | |
| "intermediate_size": 2816, | |
| "decoder_hidden_size": 256, | |
| "trained_context_length": 20480, | |
| "architecture_id": "causal_transwave_blocklifting_meyer_byte_lm_g4_v2", | |
| "training_dataset": "allenai/c4:en", | |
| "training_tokens": 4499988480, | |
| "training_steps": 785, | |
| "training_physical_batch": 280, | |
| "validation_nll": 2.228725892720205, | |
| "validation_ppl": 9.288024597671594, | |
| "source_commit": "5cae31b81e15160379ded79f32bcbb443d7db9f7", | |
| "tie_word_embeddings": true | |
| } |