NanoI1-155M-Instruct / config.json
anuj0456's picture
Upload folder using huggingface_hub
913fd49 verified
Raw History Blame Contribute Delete
976 Bytes
{
"model_type": "nanoi1",
"architectures": [
"NanoI1ForCausalLM"
],
"auto_map": {
"AutoConfig": "configuration_nanoi1.NanoI1Config",
"AutoModel": "modeling_nanoi1_hf.NanoI1Model",
"AutoModelForCausalLM": "modeling_nanoi1_hf.NanoI1ForCausalLM"
},
"vocab_size": 50304,
"hidden_size": 768,
"intermediate_size": 2048,
"num_hidden_layers": 12,
"num_attention_heads": 12,
"num_key_value_heads": 12,
"head_dim": 64,
"kv_latent_dim": 256,
"qk_norm": true,
"rms_norm_eps": 1e-06,
"rope_theta": 5000000.0,
"max_position_embeddings": 2048,
"tie_word_embeddings": false,
"torch_dtype": "bfloat16",
"embed_dim": 768,
"context_len": 131072,
"num_heads": 12,
"latent_dim": 256,
"ffn_hidden_dim": 2048,
"transformer_layers": 12,
"tie_embeddings": false,
"trained_context_len": 2048,
"num_parameters": 155293056,
"num_non_embedding_parameters": 116659584,
"bos_token_id": 0,
"eos_token_id": 4,
"pad_token_id": 1
}