| { |
| "architectures": [ |
| "Inkling2ForConditionalGeneration" |
| ], |
| "model_type": "inkling2", |
| "eos_token_id": 200006, |
| "text_config": { |
| "model_max_length": 1048576, |
| "torch_dtype": "bfloat16", |
| "hidden_size": 4608, |
| "num_hidden_layers": 48, |
| "vocab_size": 201024, |
| "num_attention_heads": 48, |
| "num_key_value_heads": 8, |
| "head_dim": 128, |
| "d_rel": 16, |
| "rel_extent": 1024, |
| "q_bias": false, |
| "o_bias": false, |
| "log_scaling_n_floor": 128000, |
| "log_scaling_alpha": 0.1, |
| "rms_norm_eps": 1e-06, |
| "pre_norm_pre_affine": false, |
| "final_norm_pre_affine": false, |
| "use_value_norm_no_gamma": true, |
| "qk_l2_clip_threshold": 22.627416997969522, |
| "rel_l2_clip_threshold": 8.0, |
| "local_layer_ids": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 30, |
| 31, |
| 32, |
| 33, |
| 34, |
| 36, |
| 37, |
| 38, |
| 39, |
| 40, |
| 42, |
| 43, |
| 44, |
| 45, |
| 46 |
| ], |
| "sconv_kernel_size": 4, |
| "unpadded_vocab_size": 200058, |
| "final_logit_softcapping": 30.0, |
| "swa_head_dim": 128, |
| "swa_num_attention_heads": 48, |
| "swa_num_key_value_heads": 8, |
| "sliding_window_size": 1024, |
| "n_layers_per_attn_res_block": 6, |
| "attn_res_normalize_values": true, |
| "n_routed_experts": 384, |
| "num_experts_per_tok": 8, |
| "n_shared_experts": 0, |
| "intermediate_size": 2304, |
| "mlp_gate_softcap": 4.0, |
| "mlp_up_softcap": 25.0, |
| "route_scale": 8.0, |
| "use_gate_bias": true |
| }, |
| "vision_config": { |
| "temporal_patch_size": 2, |
| "native_patch_size": 20, |
| "adapter_merge_size": 2, |
| "projection_hidden_dim": 4096, |
| "projector_bias": false, |
| "hidden_size": 1024, |
| "num_hidden_layers": 23, |
| "num_attention_heads": 16, |
| "mlp_ratio": 4.0, |
| "posemb_grid_size": 32, |
| "layer_norm_eps": 1e-05 |
| }, |
| "mtp_config": { |
| "hidden_size": 4608, |
| "num_hidden_layers": 5, |
| "num_attention_heads": 48, |
| "num_key_value_heads": 8, |
| "head_dim": 128, |
| "intermediate_size": 18432, |
| "max_position_embeddings": 1048576, |
| "torch_dtype": "bfloat16", |
| "q_bias": false, |
| "o_bias": false, |
| "rms_norm_eps": 1e-06, |
| "rope_theta": 1000000.0, |
| "local_layer_ids": [ |
| 0, |
| 1, |
| 3, |
| 4 |
| ], |
| "sliding_window_size": 1024, |
| "block_size": 8, |
| "markov_rank": 256, |
| "markov_head_type": "rnn_hidden", |
| "target_layer_ids": [ |
| 5, |
| 11, |
| 17, |
| 23, |
| 29, |
| 35, |
| 41, |
| 48 |
| ] |
| } |
| } |
|
|
