CryogenNet-151M / config.json
Sm0r3s's picture
Upload CryogenNet 151M
82fd153 verified
Raw History Blame Contribute Delete
913 Bytes
{
"model_type": "cryogennet",
"architecture": "CryogenNet",
"project": "CryogenAI",
"version": "0.3",
"parameter_count": 151005697,
"framework": "PyTorch",
"vocab_size": 24576,
"d_model": 1536,
"n_heads": 24,
"ffn_dim": 6144,
"n_shared_blocks": 2,
"n_coda_blocks": 1,
"max_context": 4096,
"base_training_context": 2048,
"training_recurrence_min": 2,
"training_recurrence_max": 4,
"normalization": "RMSNorm",
"activation": "SwiGLU",
"position_encoding": "RoPE",
"tied_embeddings": true,
"tokenizer": {
"type": "SentencePiece BPE",
"vocab_size": 24576,
"filename": "cryogen_spbpe24k.model",
"sha256": "6b776cb9a9d9760de07dfb670bf5fff19fa879e271301a7dbd493b024c938f34"
},
"base_training_tokens": 249429568,
"runtime_note": "CryogenNet is a custom recurrent architecture and requires the provided PyTorch implementation or another compatible backend."
}