File size: 699 Bytes
58f13e3 a22f353 58f13e3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 | {
"vocab_size": 4096,
"width": 768,
"heads": 24,
"kv_heads": 8,
"stored_blocks": 12,
"ffn_width": 3968,
"recurrent_start": 1,
"recurrent_blocks": 4,
"recurrent_passes": 2,
"tn_gram_rank": 299,
"max_ngram_order": 4,
"use_tn_gram": true,
"use_qk_norm": true,
"use_loop_conditioning": true,
"max_position_embeddings": 2048,
"rope_theta": 20000.0,
"model_type": "sol_pro",
"architectures": [
"SolProHFForCausalLM"
],
"auto_map": {
"AutoConfig": "configuration_sol_pro.SolProHFConfig",
"AutoModelForCausalLM": "modeling_sol_pro_hf.SolProHFForCausalLM"
},
"tie_word_embeddings": true,
"use_cache": false,
"eos_token_id": 2,
"pad_token_id": 3
} |