ereniko's picture
Add Ivme-Conversate-v3.1-Base (CPT of v3 on 31.4B dense tokens)
07d8f8e verified
Raw History Blame Contribute Delete
605 Bytes
{
"architectures": [
"IvmeConversateForCausalLM"
],
"context_len": 1024,
"d_ffn": 864,
"d_model": 320,
"dtype": "float32",
"max_position_embeddings": 1024,
"model_type": "ivme_conversate",
"n_heads": 8,
"n_layers": 16,
"qknorm_eps": 1e-05,
"rmsnorm_eps": 1e-05,
"rope_theta": 10000.0,
"tie_embeddings": true,
"tie_word_embeddings": true,
"transformers_version": "5.20.2",
"vocab_size": 16000,
"auto_map": {
"AutoConfig": "configuration_ivme_conversate.IvmeConversateConfig",
"AutoModelForCausalLM": "modeling_ivme_conversate.IvmeConversateForCausalLM"
}
}