| { | |
| "model_name": "Hilbert", | |
| "model_family": "hilbert", | |
| "version": "1.0", | |
| "architecture": { | |
| "type": "decoder-only-transformer", | |
| "norm": "rmsnorm", | |
| "activation": "swiglu", | |
| "position_encoding": "rope", | |
| "attention": "grouped-query" | |
| }, | |
| "dimensions": { | |
| "d_model": 2048, | |
| "n_layers": 24, | |
| "n_heads": 16, | |
| "n_kv_heads": 4, | |
| "head_dim": 128, | |
| "d_ff": 5504, | |
| "vocab_size": 32000, | |
| "max_seq_len": 8192 | |
| }, | |
| "rope": { | |
| "theta": 10000.0, | |
| "dimension_count": 128, | |
| "scaling_type": "linear", | |
| "scaling_factor": 2.0, | |
| "original_context_length": 4096 | |
| }, | |
| "norm": { | |
| "type": "rmsnorm", | |
| "epsilon": 1e-5 | |
| }, | |
| "quantization": { | |
| "training": "bf16", | |
| "deployment": "Q4_K_M", | |
| "embeddings": "Q8_0", | |
| "norms": "F32" | |
| }, | |
| "parameters": { | |
| "total_estimate": "~4B", | |
| "embedding_params": "65536000", | |
| "per_layer_params": "~150M", | |
| "note": "Exact count depends on final vocab_size and tie_word_embeddings" | |
| } | |
| } | |