{ "architectures": [ "PPAIForCausalLM" ], "auto_map": { "AutoConfig": "configuration_ppai.PPAIConfig", "AutoModelForCausalLM": "modeling_ppai.PPAIForCausalLM" }, "bos_token_id": 1, "d_model": 192, "dtype": "float32", "eos_token_id": 2, "ffn_dim": 800, "ffn_shared_dim": 256, "gate_dim": 12, "head_dim": 32, "logit_softcap": 30.0, "max_position_embeddings": 1024, "model_type": "ppai", "n_heads": 6, "n_kv_heads": 2, "n_layers": 10, "ngram_buckets": 16384, "ngram_dim": 12, "ngram_seed": 1337, "ngram_sizes": [ 2, 3, 4 ], "num_experts": 20, "pad_token_id": 0, "rms_norm_eps": 1e-06, "rope_theta": 10000.0, "tie_word_embeddings": false, "transformers_version": "5.20.2", "use_cache": false, "vocab_size": 24576 }