{
"schema": 6,
"components": [
{
"id": "target",
"role": "primary_text",
"source_artifact": "gemma-4-31B-it",
"num_layers": 60,
"hidden_size": 5376,
"attention": [
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "partial_rope",
"theta": 1000000.0,
"rotary_fraction": 0.25,
"basis": "head_width"
},
"geometry": {
"head_dim": 512,
"num_kv_heads": 4
},
"v_from_k": true,
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "partial_rope",
"theta": 1000000.0,
"rotary_fraction": 0.25,
"basis": "head_width"
},
"geometry": {
"head_dim": 512,
"num_kv_heads": 4
},
"v_from_k": true,
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "partial_rope",
"theta": 1000000.0,
"rotary_fraction": 0.25,
"basis": "head_width"
},
"geometry": {
"head_dim": 512,
"num_kv_heads": 4
},
"v_from_k": true,
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "partial_rope",
"theta": 1000000.0,
"rotary_fraction": 0.25,
"basis": "head_width"
},
"geometry": {
"head_dim": 512,
"num_kv_heads": 4
},
"v_from_k": true,
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "partial_rope",
"theta": 1000000.0,
"rotary_fraction": 0.25,
"basis": "head_width"
},
"geometry": {
"head_dim": 512,
"num_kv_heads": 4
},
"v_from_k": true,
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "partial_rope",
"theta": 1000000.0,
"rotary_fraction": 0.25,
"basis": "head_width"
},
"geometry": {
"head_dim": 512,
"num_kv_heads": 4
},
"v_from_k": true,
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "partial_rope",
"theta": 1000000.0,
"rotary_fraction": 0.25,
"basis": "head_width"
},
"geometry": {
"head_dim": 512,
"num_kv_heads": 4
},
"v_from_k": true,
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "partial_rope",
"theta": 1000000.0,
"rotary_fraction": 0.25,
"basis": "head_width"
},
"geometry": {
"head_dim": 512,
"num_kv_heads": 4
},
"v_from_k": true,
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "partial_rope",
"theta": 1000000.0,
"rotary_fraction": 0.25,
"basis": "head_width"
},
"geometry": {
"head_dim": 512,
"num_kv_heads": 4
},
"v_from_k": true,
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "partial_rope",
"theta": 1000000.0,
"rotary_fraction": 0.25,
"basis": "head_width"
},
"geometry": {
"head_dim": 512,
"num_kv_heads": 4
},
"v_from_k": true,
"declared_span": "full_attention"
}
],
"execution": {
"context_length": 262144,
"attention": {
"num_q_heads": 32,
"num_kv_heads": 16,
"head_dim": 256,
"score_scale": 1.0,
"qk_norm_scope": "per_head",
"qk_norm_weight_offset": 0.0,
"parameter_free_qk_norm": {
"q": false,
"k": false,
"v": true
},
"attention_bias": false
},
"ffn": {
"intermediate_size": 21504,
"activation": "gelu_tanh",
"ffn_type": "gated",
"gate_policy": {
"kind": "gated"
}
},
"norm": {
"pre": {
"kind": "rms_norm",
"eps": 9.999999974752427e-7,
"weight_offset": 0.0
},
"post": {
"kind": "rms_norm",
"eps": 9.999999974752427e-7,
"weight_offset": 0.0
},
"final_norm": {
"kind": "rms_norm",
"eps": 9.999999974752427e-7,
"weight_offset": 0.0
},
"placement": "pre_post"
},
"head": {
"vocab_size": 262144,
"embed_scale": 73.32121,
"final_logit_softcapping": 30.0,
"head_reuses_embedding": true
},
"residual_topology": {
"kind": "single_stream"
}
}
},
{
"id": "vision",
"role": "perception",
"source_artifact": "gemma-4-31B-it",
"num_layers": 27,
"hidden_size": 1152,
"attention": [
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "rope",
"theta": 100.0
},
"declared_span": "full_attention"
}
],
"execution": {
"attention": {
"num_q_heads": 16,
"num_kv_heads": 16,
"head_dim": 72,
"score_scale": 0.11785113019775792,
"qk_norm_scope": "per_head",
"qk_norm_weight_offset": 0.0,
"parameter_free_qk_norm": {
"q": false,
"k": false,
"v": false
},
"attention_bias": false
},
"ffn": {
"intermediate_size": 4304,
"activation": "gelu_tanh",
"ffn_type": "gated",
"gate_policy": {
"kind": "gated"
}
},
"norm": {
"pre": {
"kind": "rms_norm",
"eps": 1e-6,
"weight_offset": 0.0
},
"final_norm": {
"kind": "rms_norm",
"eps": 1e-6,
"weight_offset": 0.0
}
},
"residual_topology": {
"kind": "single_stream"
}
},
"perception": {
"modality": "image",
"transform": {
"encoder": {
"depth": 27,
"width": 1152,
"num_heads": 16
}
}
}
}
],
"objects": [
{
"id": "target.decoder_stack",
"component": "target",
"kind": "decoder_stack",
"source_bindings": [
{
"artifact": "gemma-4-31B-it",
"tensor_prefix": "model.language_model.layers",
"tensors": 830,
"bytes": 58576107640
}
],
"representations": [
{
"encoding": "BF16",
"fidelity": "canonical"
}
]
},
{
"id": "target.embedding",
"component": "target",
"kind": "embedding",
"source_bindings": [
{
"artifact": "gemma-4-31B-it",
"tensor_prefix": "model.language_model.embed_tokens",
"tensors": 1,
"bytes": 2818572288
}
],
"representations": [
{
"encoding": "BF16",
"fidelity": "canonical"
}
]
},
{
"id": "target.final_norm",
"component": "target",
"kind": "final_norm",
"source_bindings": [
{
"artifact": "gemma-4-31B-it",
"tensor_prefix": "model.language_model.norm",
"tensors": 1,
"bytes": 10752
}
],
"representations": [
{
"encoding": "BF16",
"fidelity": "canonical"
}
]
},
{
"id": "vision.perception_adapter",
"component": "vision",
"kind": "perception_adapter",
"source_bindings": [
{
"artifact": "gemma-4-31B-it",
"tensor_prefix": "model.embed_vision.embedding_projection",
"tensors": 1,
"bytes": 12386304
}
],
"representations": [
{
"encoding": "BF16",
"fidelity": "canonical"
}
]
},
{
"id": "vision.perception_tower",
"component": "vision",
"kind": "perception_tower",
"source_bindings": [
{
"artifact": "gemma-4-31B-it",
"tensor_prefix": "model.vision_tower",
"tensors": 2,
"bytes": 4608
},
{
"artifact": "gemma-4-31B-it",
"tensor_prefix": "model.vision_tower.encoder.layers",
"tensors": 351,
"bytes": 1090140768
},
{
"artifact": "gemma-4-31B-it",
"tensor_prefix": "model.vision_tower.patch_embedder",
"tensors": 1,
"bytes": 47185920
},
{
"artifact": "gemma-4-31B-it",
"tensor_prefix": "model.vision_tower.patch_embedder.input_proj",
"tensors": 1,
"bytes": 1769472
}
],
"representations": [
{
"encoding": "BF16",
"fidelity": "canonical"
}
]
}
],
"edges": []
}
{
"schema": 6,
"components": [
{
"id": "target",
"role": "primary_text",
"source_artifact": "gemma-4-31B-it",
"num_layers": 60,
"hidden_size": 5376,
"attention": [
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,
"position": {
"kind": "rope",
"theta": 10000.0
},
"geometry": {
"head_dim": 256,
"num_kv_heads": 16
},
"declared_span": "sliding_attention"
},
{
"operator": "softmax",
"span": "full",
"position": {
"kind": "partial_rope",
"theta": 1000000.0,
"rotary_fraction": 0.25,
"basis": "head_width"
},
"geometry": {
"head_dim": 512,
"num_kv_heads": 4
},
"v_from_k": true,
"declared_span": "full_attention"
},
{
"operator": "softmax",
"span": "sliding",
"window": 1024,