| { |
| "schema": 6, |
| "components": [ |
| { |
| "id": "audio", |
| "role": "perception", |
| "source_artifact": "gemma-4-12B-it", |
| "num_layers": 0, |
| "hidden_size": 640, |
| "perception": { |
| "modality": "audio", |
| "transform": { |
| "direct_projection": { |
| "output_width": 640 |
| } |
| } |
| } |
| }, |
| { |
| "id": "target", |
| "role": "primary_text", |
| "source_artifact": "gemma-4-12B-it", |
| "num_layers": 48, |
| "hidden_size": 3840, |
| "attention": [ |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "full", |
| "position": { |
| "kind": "partial_rope", |
| "theta": 1000000.0, |
| "rotary_fraction": 0.25, |
| "basis": "head_width" |
| }, |
| "geometry": { |
| "head_dim": 512, |
| "num_kv_heads": 1 |
| }, |
| "v_from_k": true, |
| "declared_span": "full_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "full", |
| "position": { |
| "kind": "partial_rope", |
| "theta": 1000000.0, |
| "rotary_fraction": 0.25, |
| "basis": "head_width" |
| }, |
| "geometry": { |
| "head_dim": 512, |
| "num_kv_heads": 1 |
| }, |
| "v_from_k": true, |
| "declared_span": "full_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "full", |
| "position": { |
| "kind": "partial_rope", |
| "theta": 1000000.0, |
| "rotary_fraction": 0.25, |
| "basis": "head_width" |
| }, |
| "geometry": { |
| "head_dim": 512, |
| "num_kv_heads": 1 |
| }, |
| "v_from_k": true, |
| "declared_span": "full_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "full", |
| "position": { |
| "kind": "partial_rope", |
| "theta": 1000000.0, |
| "rotary_fraction": 0.25, |
| "basis": "head_width" |
| }, |
| "geometry": { |
| "head_dim": 512, |
| "num_kv_heads": 1 |
| }, |
| "v_from_k": true, |
| "declared_span": "full_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "full", |
| "position": { |
| "kind": "partial_rope", |
| "theta": 1000000.0, |
| "rotary_fraction": 0.25, |
| "basis": "head_width" |
| }, |
| "geometry": { |
| "head_dim": 512, |
| "num_kv_heads": 1 |
| }, |
| "v_from_k": true, |
| "declared_span": "full_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "full", |
| "position": { |
| "kind": "partial_rope", |
| "theta": 1000000.0, |
| "rotary_fraction": 0.25, |
| "basis": "head_width" |
| }, |
| "geometry": { |
| "head_dim": 512, |
| "num_kv_heads": 1 |
| }, |
| "v_from_k": true, |
| "declared_span": "full_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "full", |
| "position": { |
| "kind": "partial_rope", |
| "theta": 1000000.0, |
| "rotary_fraction": 0.25, |
| "basis": "head_width" |
| }, |
| "geometry": { |
| "head_dim": 512, |
| "num_kv_heads": 1 |
| }, |
| "v_from_k": true, |
| "declared_span": "full_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "sliding", |
| "window": 1024, |
| "position": { |
| "kind": "rope", |
| "theta": 10000.0 |
| }, |
| "geometry": { |
| "head_dim": 256, |
| "num_kv_heads": 8 |
| }, |
| "declared_span": "sliding_attention" |
| }, |
| { |
| "operator": "softmax", |
| "span": "full", |
| "position": { |
| "kind": "partial_rope", |
| "theta": 1000000.0, |
| "rotary_fraction": 0.25, |
| "basis": "head_width" |
| }, |
| "geometry": { |
| "head_dim": 512, |
| "num_kv_heads": 1 |
| }, |
| "v_from_k": true, |
| "declared_span": "full_attention" |
| } |
| ], |
| "execution": { |
| "context_length": 262144, |
| "attention": { |
| "num_q_heads": 16, |
| "num_kv_heads": 8, |
| "head_dim": 256, |
| "score_scale": 1.0, |
| "qk_norm_scope": "per_head", |
| "qk_norm_weight_offset": 0.0, |
| "parameter_free_qk_norm": { |
| "q": false, |
| "k": false, |
| "v": true |
| }, |
| "attention_bias": false |
| }, |
| "ffn": { |
| "intermediate_size": 15360, |
| "activation": "gelu_tanh", |
| "ffn_type": "gated", |
| "gate_policy": { |
| "kind": "gated" |
| } |
| }, |
| "norm": { |
| "pre": { |
| "kind": "rms_norm", |
| "eps": 9.999999974752427e-7, |
| "weight_offset": 0.0 |
| }, |
| "post": { |
| "kind": "rms_norm", |
| "eps": 9.999999974752427e-7, |
| "weight_offset": 0.0 |
| }, |
| "final_norm": { |
| "kind": "rms_norm", |
| "eps": 9.999999974752427e-7, |
| "weight_offset": 0.0 |
| }, |
| "placement": "pre_post" |
| }, |
| "head": { |
| "vocab_size": 262144, |
| "embed_scale": 61.967735, |
| "final_logit_softcapping": 30.0, |
| "head_reuses_embedding": true |
| }, |
| "residual_topology": { |
| "kind": "single_stream" |
| } |
| } |
| }, |
| { |
| "id": "vision", |
| "role": "perception", |
| "source_artifact": "gemma-4-12B-it", |
| "num_layers": 0, |
| "hidden_size": 0, |
| "perception": { |
| "modality": "image", |
| "transform": { |
| "direct_projection": {} |
| } |
| } |
| } |
| ], |
| "objects": [ |
| { |
| "id": "audio.perception_adapter", |
| "component": "audio", |
| "kind": "perception_adapter", |
| "source_bindings": [ |
| { |
| "artifact": "gemma-4-12B-it", |
| "tensor_prefix": "model.embed_audio.embedding_projection", |
| "tensors": 1, |
| "bytes": 4915200 |
| } |
| ], |
| "representations": [ |
| { |
| "encoding": "BF16", |
| "fidelity": "canonical" |
| } |
| ] |
| }, |
| { |
| "id": "target.decoder_stack", |
| "component": "target", |
| "kind": "decoder_stack", |
| "source_bindings": [ |
| { |
| "artifact": "gemma-4-12B-it", |
| "tensor_prefix": "model.language_model.layers", |
| "tensors": 664, |
| "bytes": 21801427040 |
| } |
| ], |
| "representations": [ |
| { |
| "encoding": "BF16", |
| "fidelity": "canonical" |
| } |
| ] |
| }, |
| { |
| "id": "target.embedding", |
| "component": "target", |
| "kind": "embedding", |
| "source_bindings": [ |
| { |
| "artifact": "gemma-4-12B-it", |
| "tensor_prefix": "model.language_model.embed_tokens", |
| "tensors": 1, |
| "bytes": 2013265920 |
| } |
| ], |
| "representations": [ |
| { |
| "encoding": "BF16", |
| "fidelity": "canonical" |
| } |
| ] |
| }, |
| { |
| "id": "target.final_norm", |
| "component": "target", |
| "kind": "final_norm", |
| "source_bindings": [ |
| { |
| "artifact": "gemma-4-12B-it", |
| "tensor_prefix": "model.language_model.norm", |
| "tensors": 1, |
| "bytes": 7680 |
| } |
| ], |
| "representations": [ |
| { |
| "encoding": "BF16", |
| "fidelity": "canonical" |
| } |
| ] |
| }, |
| { |
| "id": "vision.perception_adapter", |
| "component": "vision", |
| "kind": "perception_adapter", |
| "source_bindings": [ |
| { |
| "artifact": "gemma-4-12B-it", |
| "tensor_prefix": "model.embed_vision.embedding_projection", |
| "tensors": 1, |
| "bytes": 29491200 |
| }, |
| { |
| "artifact": "gemma-4-12B-it", |
| "tensor_prefix": "model.vision_embedder", |
| "tensors": 1, |
| "bytes": 17203200 |
| }, |
| { |
| "artifact": "gemma-4-12B-it", |
| "tensor_prefix": "model.vision_embedder.patch_dense", |
| "tensors": 2, |
| "bytes": 53091840 |
| }, |
| { |
| "artifact": "gemma-4-12B-it", |
| "tensor_prefix": "model.vision_embedder.patch_ln1", |
| "tensors": 2, |
| "bytes": 27648 |
| }, |
| { |
| "artifact": "gemma-4-12B-it", |
| "tensor_prefix": "model.vision_embedder.patch_ln2", |
| "tensors": 2, |
| "bytes": 15360 |
| }, |
| { |
| "artifact": "gemma-4-12B-it", |
| "tensor_prefix": "model.vision_embedder.pos_norm", |
| "tensors": 2, |
| "bytes": 15360 |
| } |
| ], |
| "representations": [ |
| { |
| "encoding": "BF16", |
| "fidelity": "canonical" |
| } |
| ] |
| } |
| ], |
| "edges": [] |
| } |
