KaiLu-3-Pro / config.json
KVC899's picture
KaiLu 3 Pro: open release (Apache-2.0), one GGUF + one safetensors
283500f verified
Raw History Blame Contribute Delete
1.42 kB
{
"architectures": [
"Lfm2ForCausalLM"
],
"block_auto_adjust_ff_dim": false,
"block_ffn_dim_multiplier": 1.0,
"block_multiple_of": 256,
"bos_token_id": null,
"conv_L_cache": 3,
"conv_bias": false,
"dtype": "bfloat16",
"eos_token_id": 0,
"full_attn_idxs": [
2,
5,
8,
11,
14,
17,
20,
23,
26,
29,
32,
34
],
"hidden_size": 1536,
"initializer_range": 0.02,
"intermediate_size": 4096,
"layer_types": [
"conv",
"conv",
"full_attention",
"conv",
"conv",
"full_attention",
"conv",
"conv",
"full_attention",
"conv",
"conv",
"full_attention",
"conv",
"conv",
"full_attention",
"conv",
"conv",
"full_attention",
"conv",
"conv",
"full_attention",
"conv",
"conv",
"full_attention",
"conv",
"conv",
"full_attention",
"conv",
"conv",
"full_attention",
"conv",
"conv",
"full_attention",
"conv",
"full_attention",
"conv"
],
"max_position_embeddings": 51200,
"model_type": "lfm2",
"norm_eps": 1e-05,
"num_attention_heads": 24,
"num_hidden_layers": 36,
"num_key_value_heads": 6,
"pad_token_id": 0,
"rope_parameters": {
"rope_theta": 1000000.0,
"rope_type": "default"
},
"tie_word_embeddings": true,
"transformers_version": "5.22.0",
"use_cache": true,
"vocab_size": 32768
}