| { |
| "architectures": [ |
| "HetmanForCausalLM" |
| ], |
| "auto_map": { |
| "AutoConfig": "configuration_hetman.HetmanConfig", |
| "AutoModel": "modeling_hetman.HetmanModel", |
| "AutoModelForCausalLM": "modeling_hetman.HetmanForCausalLM" |
| }, |
| "model_type": "hetman", |
| "vocab_size": 65536, |
| "bos_token_id": 1, |
| "eos_token_id": 5, |
| "pad_token_id": 3, |
| "hidden_size": 1536, |
| "intermediate_size": 6144, |
| "num_hidden_layers": 34, |
| "num_deltanet_layers": 28, |
| "num_global_layers": 6, |
| "num_attention_heads": 16, |
| "deltanet_head_dim": 64, |
| "deltanet_conv_kernel": 4, |
| "global_head_dim": 128, |
| "global_attention_heads": 12, |
| "global_kv_heads": 2, |
| "hopfield_num_groups": 16, |
| "hopfield_codebook_size": 1024, |
| "hopfield_val_dim": 32, |
| "hopfield_expanded_dim": 128, |
| "hopfield_topk_1": 8, |
| "hopfield_topk_2": 4, |
| "hopfield_shared_attractors": 1, |
| "hopfield_subquery_dim": 2048, |
| "hopfield_beta": 0.1767767, |
| "hopfield_pid_ki": 0.01, |
| "hopfield_pid_kd": 0.005, |
| "hadamard_block_size": 512, |
| "swiglu_limit": 10.0, |
| "logit_softcap": 30.0, |
| "max_position_embeddings": 262144, |
| "qk_norm_bound": 11.3137, |
| "rms_norm_eps": 1e-06, |
| "rope_theta": 10000000.0, |
| "ternary_bits": 1.58, |
| "use_hadamard_rotation": true, |
| "torch_dtype": "bfloat16", |
| "transformers_version": "4.45.0" |
| } |
|
|
