Inkling2ForConditionalGeneration( (model): Inkling2Model( (language_model): Inkling2TextModel( (embed_tokens): Inkling2NormedEmbedding( 201024, 128 (embed_norm): Inkling2RMSNorm((128,), eps=1e-06) ) (layers): ModuleList( (0): Inkling2SlidingDecoderLayer( (mlp_input): Inkling2BlockAttentionResidual( (norm): Inkling2UnweightedRMSNorm() ) (self_attn): Inkling2SlidingAttention( (q_proj): Linear(in_features=128, out_features=1024, bias=False) (k_proj): Linear(in_features=128, out_features=512, bias=False) (v_proj): Linear(in_features=128, out_features=512, bias=False) (r_proj): Linear(in_features=128, out_features=128, bias=False) (o_proj): Linear(in_features=1024, out_features=128, bias=False) (k_sconv): Inkling2ShortConvolution( (conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False) ) (v_sconv): Inkling2ShortConvolution( (conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False) ) (rel_logits_proj): Inkling2RelativeLogits() (gate_proj): Linear(in_features=128, out_features=1024, bias=False) (sink_proj): Linear(in_features=128, out_features=8, bias=False) (q_l2_clip): Inkling2ScaledL2Clip() (k_l2_clip): Inkling2L2Clip() (rel_l2_clip): Inkling2ScaledL2Clip() (v_norm): Inkling2UnweightedRMSNorm() ) (mlp): Inkling2SparseMoeBlock( (experts): Inkling2Experts( (act_fn): Inkling2SoftcappedSwiGLU( (gate_softcap): Inkling2Softcap() (up_softcap): Inkling2Softcap() ) ) (gate): Inkling2TopKRouter( (score_fn): Sigmoid() ) ) (input_layernorm): Inkling2RMSNorm((128,), eps=1e-06) (post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06) (post_attention_convolution): Inkling2ShortConvolution( (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False) ) (post_mlp_convolution): Inkling2ShortConvolution( (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False) ) ) (1-4): 4 x Inkling2SlidingDecoderLayer( (attention_input): Inkling2BlockAttentionResidual( (norm): Inkling2UnweightedRMSNorm() ) (mlp_input): Inkling2BlockAttentionResidual( (norm): Inkling2UnweightedRMSNorm() ) (self_attn): Inkling2SlidingAttention( (q_proj): Linear(in_features=128, out_features=1024, bias=False) (k_proj): Linear(in_features=128, out_features=512, bias=False) (v_proj): Linear(in_features=128, out_features=512, bias=False) (r_proj): Linear(in_features=128, out_features=128, bias=False) (o_proj): Linear(in_features=1024, out_features=128, bias=False) (k_sconv): Inkling2ShortConvolution( (conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False) ) (v_sconv): Inkling2ShortConvolution( (conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False) ) (rel_logits_proj): Inkling2RelativeLogits() (gate_proj): Linear(in_features=128, out_features=1024, bias=False) (sink_proj): Linear(in_features=128, out_features=8, bias=False) (q_l2_clip): Inkling2ScaledL2Clip() (k_l2_clip): Inkling2L2Clip() (rel_l2_clip): Inkling2ScaledL2Clip() (v_norm): Inkling2UnweightedRMSNorm() ) (mlp): Inkling2SparseMoeBlock( (experts): Inkling2Experts( (act_fn): Inkling2SoftcappedSwiGLU( (gate_softcap): Inkling2Softcap() (up_softcap): Inkling2Softcap() ) ) (gate): Inkling2TopKRouter( (score_fn): Sigmoid() ) ) (input_layernorm): Inkling2RMSNorm((128,), eps=1e-06) (post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06) (post_attention_convolution): Inkling2ShortConvolution( (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False) ) (post_mlp_convolution): Inkling2ShortConvolution( (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False) ) ) (5): Inkling2FullDecoderLayer( (attention_input): Inkling2BlockAttentionResidual( (norm): Inkling2UnweightedRMSNorm() ) (mlp_input): Inkling2BlockAttentionResidual( (norm): Inkling2UnweightedRMSNorm() ) (self_attn): Inkling2FullAttention( (q_proj): Linear(in_features=128, out_features=1024, bias=False) (k_proj): Linear(in_features=128, out_features=512, bias=False) (v_proj): Linear(in_features=128, out_features=512, bias=False) (r_proj): Linear(in_features=128, out_features=128, bias=False) (o_proj): Linear(in_features=1024, out_features=128, bias=False) (k_sconv): Inkling2DynamicShortConvolution( (weight_proj): Linear(in_features=128, out_features=16, bias=False) ) (v_sconv): Inkling2DynamicShortConvolution( (weight_proj): Linear(in_features=128, out_features=16, bias=False) ) (rel_logits_proj): Inkling2RelativeLogits() (gate_proj): Linear(in_features=128, out_features=1024, bias=False) (sink_proj): Linear(in_features=128, out_features=8, bias=False) (q_l2_clip): Inkling2ScaledL2Clip() (k_l2_clip): Inkling2L2Clip() (rel_l2_clip): Inkling2ScaledL2Clip() (v_norm): Inkling2UnweightedRMSNorm() ) (mlp): Inkling2SparseMoeBlock( (experts): Inkling2Experts( (act_fn): Inkling2SoftcappedSwiGLU( (gate_softcap): Inkling2Softcap() (up_softcap): Inkling2Softcap() ) ) (gate): Inkling2TopKRouter( (score_fn): Sigmoid() ) ) (q_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06) (k_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06) (v_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06) (post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06) (post_attention_convolution): Inkling2ShortConvolution( (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False) ) (post_mlp_convolution): Inkling2ShortConvolution( (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False) ) ) (6-10): 5 x Inkling2SlidingDecoderLayer( (attention_input): Inkling2BlockAttentionResidual( (norm): Inkling2UnweightedRMSNorm() ) (mlp_input): Inkling2BlockAttentionResidual( (norm): Inkling2UnweightedRMSNorm() ) (self_attn): Inkling2SlidingAttention( (q_proj): Linear(in_features=128, out_features=1024, bias=False) (k_proj): Linear(in_features=128, out_features=512, bias=False) (v_proj): Linear(in_features=128, out_features=512, bias=False) (r_proj): Linear(in_features=128, out_features=128, bias=False) (o_proj): Linear(in_features=1024, out_features=128, bias=False) (k_sconv): Inkling2ShortConvolution( (conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False) ) (v_sconv): Inkling2ShortConvolution( (conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False) ) (rel_logits_proj): Inkling2RelativeLogits() (gate_proj): Linear(in_features=128, out_features=1024, bias=False) (sink_proj): Linear(in_features=128, out_features=8, bias=False) (q_l2_clip): Inkling2ScaledL2Clip() (k_l2_clip): Inkling2L2Clip() (rel_l2_clip): Inkling2ScaledL2Clip() (v_norm): Inkling2UnweightedRMSNorm() ) (mlp): Inkling2SparseMoeBlock( (experts): Inkling2Experts( (act_fn): Inkling2SoftcappedSwiGLU( (gate_softcap): Inkling2Softcap() (up_softcap): Inkling2Softcap() ) ) (gate): Inkling2TopKRouter( (score_fn): Sigmoid() ) ) (input_layernorm): Inkling2RMSNorm((128,), eps=1e-06) (post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06) (post_attention_convolution): Inkling2ShortConvolution( (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False) ) (post_mlp_convolution): Inkling2ShortConvolution( (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False) ) ) (11): Inkling2FullDecoderLayer( (attention_input): Inkling2BlockAttentionResidual( (norm): Inkling2UnweightedRMSNorm() ) (mlp_input): Inkling2BlockAttentionResidual( (norm): Inkling2UnweightedRMSNorm() ) (self_attn): Inkling2FullAttention( (q_proj): Linear(in_features=128, out_features=1024, bias=False) (k_proj): Linear(in_features=128, out_features=512, bias=False) (v_proj): Linear(in_features=128, out_features=512, bias=False) (r_proj): Linear(in_features=128, out_features=128, bias=False) (o_proj): Linear(in_features=1024, out_features=128, bias=False) (k_sconv): Inkling2DynamicShortConvolution( (weight_proj): Linear(in_features=128, out_features=16, bias=False) ) (v_sconv): Inkling2DynamicShortConvolution( (weight_proj): Linear(in_features=128, out_features=16, bias=False) ) (rel_logits_proj): Inkling2RelativeLogits() (gate_proj): Linear(in_features=128, out_features=1024, bias=False) (sink_proj): Linear(in_features=128, out_features=8, bias=False) (q_l2_clip): Inkling2ScaledL2Clip() (k_l2_clip): Inkling2L2Clip() (rel_l2_clip): Inkling2ScaledL2Clip() (v_norm): Inkling2UnweightedRMSNorm() ) (mlp): Inkling2SparseMoeBlock( (experts): Inkling2Experts( (act_fn): Inkling2SoftcappedSwiGLU( (gate_softcap): Inkling2Softcap() (up_softcap): Inkling2Softcap() ) ) (gate): Inkling2TopKRouter( (score_fn): Sigmoid() ) ) (q_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06) (k_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06) (v_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06) (post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06) (post_attention_convolution): Inkling2ShortConvolution( (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False) ) (post_mlp_convolution): Inkling2ShortConvolution( (conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False) ) ) ) (norm): Inkling2RMSNorm((128,), eps=1e-06) (final_input): Inkling2BlockAttentionResidual( (norm): Inkling2UnweightedRMSNorm() ) ) (vision_tower): Inkling2VisionModel( (embeddings): Inkling2VisionEmbeddings( (patch_embedding): Linear(in_features=1200, out_features=32, bias=False) (position_embedding): Embedding(1024, 32) ) (pre_layernorm): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True) (rotary_emb): Inkling2VisionRotaryEmbedding() (layers): ModuleList( (0-3): 4 x Inkling2VisionEncoderLayer( (self_attn): Inkling2VisionAttention( (k_proj): Linear(in_features=32, out_features=32, bias=True) (v_proj): Linear(in_features=32, out_features=32, bias=True) (q_proj): Linear(in_features=32, out_features=32, bias=True) (out_proj): Linear(in_features=32, out_features=32, bias=True) ) (mlp): Inkling2VisionMLP( (activation_fn): GELUActivation() (fc1): Linear(in_features=32, out_features=128, bias=True) (fc2): Linear(in_features=128, out_features=32, bias=True) ) (layernorm_before): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True) (layernorm_after): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True) ) ) (merger): Inkling2VisionMerger( (linear_1): Linear(in_features=128, out_features=32, bias=False) (act): GELUActivation() (linear_2): Linear(in_features=32, out_features=128, bias=False) ) ) ) (lm_head): Linear(in_features=128, out_features=201024, bias=False) (logit_softcap): Inkling2Softcap() (mtp): Module( (layers): Module( (0): Module( (dflare_norm): Module() (input_layernorm): Module() (mlp): Module( (down_proj): Module() (gate_proj): Module() (up_proj): Module() ) (post_attention_layernorm): Module() (self_attn): Module( (k_norm): Module() (k_norm_ctx): Module() (k_proj): Module() (k_proj_ctx): Module() (o_proj): Module() (q_norm): Module() (q_proj): Module() (v_proj): Module() (v_proj_ctx): Module() ) ) (1): Module( (dflare_norm): Module() (input_layernorm): Module() (mlp): Module( (down_proj): Module() (gate_proj): Module() (up_proj): Module() ) (post_attention_layernorm): Module() (self_attn): Module( (k_norm): Module() (k_norm_ctx): Module() (k_proj): Module() (k_proj_ctx): Module() (o_proj): Module() (q_norm): Module() (q_proj): Module() (v_proj): Module() (v_proj_ctx): Module() ) ) (2): Module( (dflare_norm): Module() (input_layernorm): Module() (mlp): Module( (down_proj): Module() (gate_proj): Module() (up_proj): Module() ) (post_attention_layernorm): Module() (self_attn): Module( (k_norm): Module() (k_norm_ctx): Module() (k_proj): Module() (k_proj_ctx): Module() (o_proj): Module() (q_norm): Module() (q_proj): Module() (v_proj): Module() (v_proj_ctx): Module() ) ) (3): Module( (dflare_norm): Module() (input_layernorm): Module() (mlp): Module( (down_proj): Module() (gate_proj): Module() (up_proj): Module() ) (post_attention_layernorm): Module() (self_attn): Module( (k_norm): Module() (k_norm_ctx): Module() (k_proj): Module() (k_proj_ctx): Module() (o_proj): Module() (q_norm): Module() (q_proj): Module() (v_proj): Module() (v_proj_ctx): Module() ) ) (4): Module( (dflare_norm): Module() (input_layernorm): Module() (mlp): Module( (down_proj): Module() (gate_proj): Module() (up_proj): Module() ) (post_attention_layernorm): Module() (self_attn): Module( (k_norm): Module() (k_norm_ctx): Module() (k_proj): Module() (k_proj_ctx): Module() (o_proj): Module() (q_norm): Module() (q_proj): Module() (v_proj): Module() (v_proj_ctx): Module() ) ) ) (markov_head): Module( (dflare_norm): Module() (hidden_out): Module() (markov_w1): Module() (rnn_proj): Module() ) (norm): Module() ) )