{ "architectures": [ "Inkling2LoopForConditionalGeneration" ], "dtype": "bfloat16", "eos_token_id": 200006, "image_token_id": 200054, "model_type": "inkling2_loop", "mtp_config": { "block_size": 8, "head_dim": 128, "hidden_size": 2304, "intermediate_size": 9216, "local_layer_ids": [ 0, 2 ], "markov_head_type": "rnn_hidden", "markov_rank": 256, "max_position_embeddings": 1048576, "num_attention_heads": 24, "num_hidden_layers": 3, "num_key_value_heads": 8, "o_bias": false, "q_bias": false, "rms_norm_eps": 1e-06, "rope_theta": 1000000.0, "sliding_window_size": 1024, "target_layer_ids": [ 5, 23, 41, 48 ], "torch_dtype": "bfloat16" }, "text_config": { "attention_dropout": 0.0, "attn_res_normalize_values": true, "bos_token_id": null, "conv_kernel_size": 4, "d_rel": 4, "dtype": "bfloat16", "eos_token_id": 200006, "final_logit_softcapping": 30.0, "head_dim": 8, "hidden_act": "silu", "hidden_size": 32, "initializer_range": 0.02, "layer_types": [ "hybrid_sliding", "hybrid", "hybrid_sliding", "hybrid", "hybrid_sliding", "hybrid", "hybrid_sliding", "hybrid" ], "local_layer_ids": [ 0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22 ], "log_scaling_alpha": 0.1, "log_scaling_n_floor": 8192, "loop_config": { "core_end": 6, "core_start": 2, "gate_rank": 8, "is_varloop": false, "num_passes": 3 }, "loop_gate_rank": 8, "loop_layer_index_end": 6, "loop_layer_index_start": 2, "loop_num_passes": 3, "max_position_embeddings": 1048576, "mlp_gate_softcap": 4.0, "mlp_up_softcap": 25.0, "model_type": "inkling2_loop_text", "moe_intermediate_size": 32, "n_layers_per_attn_res_block": 2, "n_routed_experts": 8, "n_shared_experts": 0, "num_attention_heads": 4, "num_experts_per_tok": 2, "num_hidden_layers": 8, "num_key_value_heads": 2, "o_bias": false, "pad_token_id": null, "q_bias": false, "qk_l2_clip_threshold": 22.627416997969522, "rel_extent": 32, "rel_l2_clip_threshold": 8.0, "rms_norm_eps": 1e-06, "route_scale": 8.0, "scoring_func": "sigmoid", "sliding_window_size": 16, "swa_head_dim": 8, "swa_num_attention_heads": 4, "swa_num_key_value_heads": 2, "unpadded_vocab_size": 200058, "use_cache": true, "use_gate_bias": true, "use_value_norm_no_gamma": true, "vocab_size": 201024 }, "transformers_version": "5.24.0.dev0", "vision_config": { "attention_dropout": 0.0, "hidden_act": "gelu", "hidden_size": 32, "initializer_range": 0.02, "intermediate_size": 4096, "layer_norm_eps": 1e-05, "layer_scale_init_value": 0.1, "mlp_ratio": 4.0, "model_type": "inkling2_vision", "num_attention_heads": 2, "num_channels": 3, "num_hidden_layers": 2, "patch_size": 20, "posemb_grid_size": 32, "projection_hidden_dim": 64, "projector_bias": false, "rope_parameters": { "rope_theta": 10000.0, "rope_type": "axial" }, "spatial_merge_size": 2, "temporal_patch_size": 2, "text_hidden_size": 32 } }