inkling-2 / .meta /model.log
yujiepan's picture
Preserve native mixed NVFP4/MXFP8 Inkling 2 without model-specific plugins
6c0013a verified
Raw History Blame Contribute Delete
16.9 kB
Inkling2ForConditionalGeneration(
(model): Inkling2Model(
(language_model): Inkling2TextModel(
(embed_tokens): Inkling2NormedEmbedding(
201024, 128
(embed_norm): Inkling2RMSNorm((128,), eps=1e-06)
)
(layers): ModuleList(
(0): Inkling2SlidingDecoderLayer(
(mlp_input): Inkling2BlockAttentionResidual(
(norm): Inkling2UnweightedRMSNorm()
)
(self_attn): Inkling2SlidingAttention(
(q_proj): Linear(in_features=128, out_features=1024, bias=False)
(k_proj): Linear(in_features=128, out_features=512, bias=False)
(v_proj): Linear(in_features=128, out_features=512, bias=False)
(r_proj): Linear(in_features=128, out_features=128, bias=False)
(o_proj): Linear(in_features=1024, out_features=128, bias=False)
(k_sconv): Inkling2ShortConvolution(
(conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
)
(v_sconv): Inkling2ShortConvolution(
(conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
)
(rel_logits_proj): Inkling2RelativeLogits()
(gate_proj): Linear(in_features=128, out_features=1024, bias=False)
(sink_proj): Linear(in_features=128, out_features=8, bias=False)
(q_l2_clip): Inkling2ScaledL2Clip()
(k_l2_clip): Inkling2L2Clip()
(rel_l2_clip): Inkling2ScaledL2Clip()
(v_norm): Inkling2UnweightedRMSNorm()
)
(mlp): Inkling2SparseMoeBlock(
(experts): Inkling2Experts(
(act_fn): Inkling2SoftcappedSwiGLU(
(gate_softcap): Inkling2Softcap()
(up_softcap): Inkling2Softcap()
)
)
(gate): Inkling2TopKRouter(
(score_fn): Sigmoid()
)
)
(input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
(post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
(post_attention_convolution): Inkling2ShortConvolution(
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
)
(post_mlp_convolution): Inkling2ShortConvolution(
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
)
)
(1-4): 4 x Inkling2SlidingDecoderLayer(
(attention_input): Inkling2BlockAttentionResidual(
(norm): Inkling2UnweightedRMSNorm()
)
(mlp_input): Inkling2BlockAttentionResidual(
(norm): Inkling2UnweightedRMSNorm()
)
(self_attn): Inkling2SlidingAttention(
(q_proj): Linear(in_features=128, out_features=1024, bias=False)
(k_proj): Linear(in_features=128, out_features=512, bias=False)
(v_proj): Linear(in_features=128, out_features=512, bias=False)
(r_proj): Linear(in_features=128, out_features=128, bias=False)
(o_proj): Linear(in_features=1024, out_features=128, bias=False)
(k_sconv): Inkling2ShortConvolution(
(conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
)
(v_sconv): Inkling2ShortConvolution(
(conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
)
(rel_logits_proj): Inkling2RelativeLogits()
(gate_proj): Linear(in_features=128, out_features=1024, bias=False)
(sink_proj): Linear(in_features=128, out_features=8, bias=False)
(q_l2_clip): Inkling2ScaledL2Clip()
(k_l2_clip): Inkling2L2Clip()
(rel_l2_clip): Inkling2ScaledL2Clip()
(v_norm): Inkling2UnweightedRMSNorm()
)
(mlp): Inkling2SparseMoeBlock(
(experts): Inkling2Experts(
(act_fn): Inkling2SoftcappedSwiGLU(
(gate_softcap): Inkling2Softcap()
(up_softcap): Inkling2Softcap()
)
)
(gate): Inkling2TopKRouter(
(score_fn): Sigmoid()
)
)
(input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
(post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
(post_attention_convolution): Inkling2ShortConvolution(
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
)
(post_mlp_convolution): Inkling2ShortConvolution(
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
)
)
(5): Inkling2FullDecoderLayer(
(attention_input): Inkling2BlockAttentionResidual(
(norm): Inkling2UnweightedRMSNorm()
)
(mlp_input): Inkling2BlockAttentionResidual(
(norm): Inkling2UnweightedRMSNorm()
)
(self_attn): Inkling2FullAttention(
(q_proj): Linear(in_features=128, out_features=1024, bias=False)
(k_proj): Linear(in_features=128, out_features=512, bias=False)
(v_proj): Linear(in_features=128, out_features=512, bias=False)
(r_proj): Linear(in_features=128, out_features=128, bias=False)
(o_proj): Linear(in_features=1024, out_features=128, bias=False)
(k_sconv): Inkling2DynamicShortConvolution(
(weight_proj): Linear(in_features=128, out_features=16, bias=False)
)
(v_sconv): Inkling2DynamicShortConvolution(
(weight_proj): Linear(in_features=128, out_features=16, bias=False)
)
(rel_logits_proj): Inkling2RelativeLogits()
(gate_proj): Linear(in_features=128, out_features=1024, bias=False)
(sink_proj): Linear(in_features=128, out_features=8, bias=False)
(q_l2_clip): Inkling2ScaledL2Clip()
(k_l2_clip): Inkling2L2Clip()
(rel_l2_clip): Inkling2ScaledL2Clip()
(v_norm): Inkling2UnweightedRMSNorm()
)
(mlp): Inkling2SparseMoeBlock(
(experts): Inkling2Experts(
(act_fn): Inkling2SoftcappedSwiGLU(
(gate_softcap): Inkling2Softcap()
(up_softcap): Inkling2Softcap()
)
)
(gate): Inkling2TopKRouter(
(score_fn): Sigmoid()
)
)
(q_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
(k_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
(v_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
(post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
(post_attention_convolution): Inkling2ShortConvolution(
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
)
(post_mlp_convolution): Inkling2ShortConvolution(
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
)
)
(6-10): 5 x Inkling2SlidingDecoderLayer(
(attention_input): Inkling2BlockAttentionResidual(
(norm): Inkling2UnweightedRMSNorm()
)
(mlp_input): Inkling2BlockAttentionResidual(
(norm): Inkling2UnweightedRMSNorm()
)
(self_attn): Inkling2SlidingAttention(
(q_proj): Linear(in_features=128, out_features=1024, bias=False)
(k_proj): Linear(in_features=128, out_features=512, bias=False)
(v_proj): Linear(in_features=128, out_features=512, bias=False)
(r_proj): Linear(in_features=128, out_features=128, bias=False)
(o_proj): Linear(in_features=1024, out_features=128, bias=False)
(k_sconv): Inkling2ShortConvolution(
(conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
)
(v_sconv): Inkling2ShortConvolution(
(conv1d): Conv1d(512, 512, kernel_size=(4,), stride=(1,), padding=(3,), groups=512, bias=False)
)
(rel_logits_proj): Inkling2RelativeLogits()
(gate_proj): Linear(in_features=128, out_features=1024, bias=False)
(sink_proj): Linear(in_features=128, out_features=8, bias=False)
(q_l2_clip): Inkling2ScaledL2Clip()
(k_l2_clip): Inkling2L2Clip()
(rel_l2_clip): Inkling2ScaledL2Clip()
(v_norm): Inkling2UnweightedRMSNorm()
)
(mlp): Inkling2SparseMoeBlock(
(experts): Inkling2Experts(
(act_fn): Inkling2SoftcappedSwiGLU(
(gate_softcap): Inkling2Softcap()
(up_softcap): Inkling2Softcap()
)
)
(gate): Inkling2TopKRouter(
(score_fn): Sigmoid()
)
)
(input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
(post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
(post_attention_convolution): Inkling2ShortConvolution(
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
)
(post_mlp_convolution): Inkling2ShortConvolution(
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
)
)
(11): Inkling2FullDecoderLayer(
(attention_input): Inkling2BlockAttentionResidual(
(norm): Inkling2UnweightedRMSNorm()
)
(mlp_input): Inkling2BlockAttentionResidual(
(norm): Inkling2UnweightedRMSNorm()
)
(self_attn): Inkling2FullAttention(
(q_proj): Linear(in_features=128, out_features=1024, bias=False)
(k_proj): Linear(in_features=128, out_features=512, bias=False)
(v_proj): Linear(in_features=128, out_features=512, bias=False)
(r_proj): Linear(in_features=128, out_features=128, bias=False)
(o_proj): Linear(in_features=1024, out_features=128, bias=False)
(k_sconv): Inkling2DynamicShortConvolution(
(weight_proj): Linear(in_features=128, out_features=16, bias=False)
)
(v_sconv): Inkling2DynamicShortConvolution(
(weight_proj): Linear(in_features=128, out_features=16, bias=False)
)
(rel_logits_proj): Inkling2RelativeLogits()
(gate_proj): Linear(in_features=128, out_features=1024, bias=False)
(sink_proj): Linear(in_features=128, out_features=8, bias=False)
(q_l2_clip): Inkling2ScaledL2Clip()
(k_l2_clip): Inkling2L2Clip()
(rel_l2_clip): Inkling2ScaledL2Clip()
(v_norm): Inkling2UnweightedRMSNorm()
)
(mlp): Inkling2SparseMoeBlock(
(experts): Inkling2Experts(
(act_fn): Inkling2SoftcappedSwiGLU(
(gate_softcap): Inkling2Softcap()
(up_softcap): Inkling2Softcap()
)
)
(gate): Inkling2TopKRouter(
(score_fn): Sigmoid()
)
)
(q_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
(k_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
(v_input_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
(post_attention_layernorm): Inkling2RMSNorm((128,), eps=1e-06)
(post_attention_convolution): Inkling2ShortConvolution(
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
)
(post_mlp_convolution): Inkling2ShortConvolution(
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
)
)
)
(norm): Inkling2RMSNorm((128,), eps=1e-06)
(final_input): Inkling2BlockAttentionResidual(
(norm): Inkling2UnweightedRMSNorm()
)
)
(vision_tower): Inkling2VisionModel(
(embeddings): Inkling2VisionEmbeddings(
(patch_embedding): Linear(in_features=1200, out_features=32, bias=False)
(position_embedding): Embedding(1024, 32)
)
(pre_layernorm): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
(rotary_emb): Inkling2VisionRotaryEmbedding()
(layers): ModuleList(
(0-3): 4 x Inkling2VisionEncoderLayer(
(self_attn): Inkling2VisionAttention(
(k_proj): Linear(in_features=32, out_features=32, bias=True)
(v_proj): Linear(in_features=32, out_features=32, bias=True)
(q_proj): Linear(in_features=32, out_features=32, bias=True)
(out_proj): Linear(in_features=32, out_features=32, bias=True)
)
(mlp): Inkling2VisionMLP(
(activation_fn): GELUActivation()
(fc1): Linear(in_features=32, out_features=128, bias=True)
(fc2): Linear(in_features=128, out_features=32, bias=True)
)
(layernorm_before): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
(layernorm_after): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
)
)
(merger): Inkling2VisionMerger(
(linear_1): Linear(in_features=128, out_features=32, bias=False)
(act): GELUActivation()
(linear_2): Linear(in_features=32, out_features=128, bias=False)
)
)
)
(lm_head): Linear(in_features=128, out_features=201024, bias=False)
(logit_softcap): Inkling2Softcap()
(mtp): Module(
(layers): Module(
(0): Module(
(dflare_norm): Module()
(input_layernorm): Module()
(mlp): Module(
(down_proj): Module()
(gate_proj): Module()
(up_proj): Module()
)
(post_attention_layernorm): Module()
(self_attn): Module(
(k_norm): Module()
(k_norm_ctx): Module()
(k_proj): Module()
(k_proj_ctx): Module()
(o_proj): Module()
(q_norm): Module()
(q_proj): Module()
(v_proj): Module()
(v_proj_ctx): Module()
)
)
(1): Module(
(dflare_norm): Module()
(input_layernorm): Module()
(mlp): Module(
(down_proj): Module()
(gate_proj): Module()
(up_proj): Module()
)
(post_attention_layernorm): Module()
(self_attn): Module(
(k_norm): Module()
(k_norm_ctx): Module()
(k_proj): Module()
(k_proj_ctx): Module()
(o_proj): Module()
(q_norm): Module()
(q_proj): Module()
(v_proj): Module()
(v_proj_ctx): Module()
)
)
(2): Module(
(dflare_norm): Module()
(input_layernorm): Module()
(mlp): Module(
(down_proj): Module()
(gate_proj): Module()
(up_proj): Module()
)
(post_attention_layernorm): Module()
(self_attn): Module(
(k_norm): Module()
(k_norm_ctx): Module()
(k_proj): Module()
(k_proj_ctx): Module()
(o_proj): Module()
(q_norm): Module()
(q_proj): Module()
(v_proj): Module()
(v_proj_ctx): Module()
)
)
(3): Module(
(dflare_norm): Module()
(input_layernorm): Module()
(mlp): Module(
(down_proj): Module()
(gate_proj): Module()
(up_proj): Module()
)
(post_attention_layernorm): Module()
(self_attn): Module(
(k_norm): Module()
(k_norm_ctx): Module()
(k_proj): Module()
(k_proj_ctx): Module()
(o_proj): Module()
(q_norm): Module()
(q_proj): Module()
(v_proj): Module()
(v_proj_ctx): Module()
)
)
(4): Module(
(dflare_norm): Module()
(input_layernorm): Module()
(mlp): Module(
(down_proj): Module()
(gate_proj): Module()
(up_proj): Module()
)
(post_attention_layernorm): Module()
(self_attn): Module(
(k_norm): Module()
(k_norm_ctx): Module()
(k_proj): Module()
(k_proj_ctx): Module()
(o_proj): Module()
(q_norm): Module()
(q_proj): Module()
(v_proj): Module()
(v_proj_ctx): Module()
)
)
)
(markov_head): Module(
(dflare_norm): Module()
(hidden_out): Module()
(markov_w1): Module()
(rnn_proj): Module()
)
(norm): Module()
)
)