File size: 1,078 Bytes
398cfcf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 | {
"model_name": "Sol Nano",
"model_type": "sol_nano",
"base_architecture": "Sol Lite",
"architecture_class": "SolForCausalLM",
"variant": "sol_nano_2p9m_tn_gram",
"parameter_budget": 2900000,
"expected_parameters": 2895188,
"vocab_size": 1024,
"width": 128,
"heads": 4,
"kv_heads": 2,
"stored_blocks": 10,
"ffn_width": 536,
"recurrent_start": 1,
"recurrent_blocks": 4,
"recurrent_passes": 2,
"effective_block_applications": 14,
"engram_entries": 0,
"use_engram": false,
"use_tn_gram": true,
"tn_rank": 10,
"tn_buckets": 4950,
"use_qk_norm": true,
"use_loop_conditioning": true,
"use_xsa": false,
"max_position_embeddings": 512,
"rope_theta": 20000.0,
"tie_word_embeddings": true,
"parameter_ledger": {
"token_embedding_and_tied_head": 131072,
"ten_transformer_blocks": 2552960,
"loop_embeddings_and_gates": 1280,
"final_norm": 128,
"tn_gram": 209748
},
"tokenizer_path": "models/Sol-Nano/tokenizer.json",
"tokenizer_sha256": "72a97c4c592c991733fcc6ad687734ce51e1ba25b0f7c2faf611ac30edc50231"
}
|