PP-Hydrogen / config.json
AdminHugging's picture
Upload folder using huggingface_hub
c8a9c5d verified
Raw History Blame Contribute Delete
797 Bytes
{
"architectures": [
"PPAIForCausalLM"
],
"auto_map": {
"AutoConfig": "configuration_ppai.PPAIConfig",
"AutoModelForCausalLM": "modeling_ppai.PPAIForCausalLM"
},
"bos_token_id": 1,
"d_model": 192,
"dtype": "float32",
"eos_token_id": 2,
"ffn_dim": 800,
"ffn_shared_dim": 256,
"gate_dim": 12,
"head_dim": 32,
"logit_softcap": 30.0,
"max_position_embeddings": 1024,
"model_type": "ppai",
"n_heads": 6,
"n_kv_heads": 2,
"n_layers": 10,
"ngram_buckets": 16384,
"ngram_dim": 12,
"ngram_seed": 1337,
"ngram_sizes": [
2,
3,
4
],
"num_experts": 20,
"pad_token_id": 0,
"rms_norm_eps": 1e-06,
"rope_theta": 10000.0,
"tie_word_embeddings": false,
"transformers_version": "5.20.2",
"use_cache": false,
"vocab_size": 24576
}