File size: 1,060 Bytes
ea51a3d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | {
"model_type": "transwave_meyer",
"architectures": ["TransWaveForCausalLM"],
"auto_map": {
"AutoConfig": "configuration_transwave.TransWaveConfig",
"AutoModelForCausalLM": "modeling_transwave.TransWaveForCausalLM"
},
"vocab_size": 256,
"byte_dim": 96,
"bytes_per_block": 8,
"num_scales": 8,
"hidden_size": 768,
"max_position_embeddings": 32768,
"block_context": 4096,
"num_hidden_layers": 12,
"num_groups": 12,
"meyer_bands": 12,
"meyer_controls_per_band": 8,
"meyer_kernel_blocks": 512,
"local_kernel_blocks": 7,
"intermediate_size": 2816,
"decoder_hidden_size": 256,
"trained_context_length": 20480,
"architecture_id": "causal_transwave_blocklifting_meyer_byte_lm_g4_v2",
"training_dataset": "allenai/c4:en",
"training_tokens": 4499988480,
"training_steps": 785,
"training_physical_batch": 280,
"validation_nll": 2.228725892720205,
"validation_ppl": 9.288024597671594,
"source_commit": "5cae31b81e15160379ded79f32bcbb443d7db9f7",
"tie_word_embeddings": true
} |