Kestrel-34M-instruct / config.json
lillycham's picture
Upload Kestrel-34M-instruct
ff2813d verified
Raw History Blame Contribute Delete
1.27 kB
{
"attention_bias": false,
"attention_dropout": 0.0,
"attention_window": 256,
"bos_token_id": 1,
"coda_layers": 1,
"eos_token_id": 2,
"head_dim": 64,
"hidden_act": "silu",
"hidden_size": 512,
"initializer_range": 0.02,
"intermediate_size": 1408,
"layer_types": [
"full_attention",
"full_attention",
"full_attention",
"full_attention"
],
"max_position_embeddings": 1024,
"max_window_layers": 28,
"memory_chunk_size": 64,
"memory_forget_bias": 0.0,
"memory_heads": 4,
"memory_hidden_size": 512,
"memory_min_forget": 0.02,
"model_type": "kestrel",
"norm_attn_branch": true,
"num_attention_heads": 8,
"num_hidden_layers": 4,
"num_key_value_heads": 2,
"num_loops": 3,
"pad_token_id": 0,
"prelude_layers": 1,
"rms_norm_eps": 1e-06,
"rope_parameters": {
"rope_theta": 10000.0,
"rope_type": "default"
},
"sliding_window": null,
"tie_word_embeddings": true,
"transformers_version": "5.22.1",
"use_cache": false,
"use_sliding_window": false,
"vocab_size": 16384,
"architectures": [
"KestrelForCausalLM"
],
"dtype": "bfloat16",
"auto_map": {
"AutoConfig": "configuration_kestrel.KestrelConfig",
"AutoModelForCausalLM": "modeling_kestrel.KestrelForCausalLM"
}
}