{ "batch_size": 1, "padding_supported": false, "minimum_tokens": 1, "native_activation_dtype": "BFLOAT16", "logits_dtype": "FLOAT", "constraints": [ "input_ids and position_ids have identical shape; tokens must be in vocabulary range", "forward/prefill positions are contiguous starting at0 and states initialize to zero", "decode consumes one token; position_ids equals the consumed prefix length", "all MLA past caches share the same prefix length; all layers belong to the same sequence", "feed every present cache into its mapped past input on the next decode step", "forward omits states; prefill returns states; decode consumes and returns states", "BF16 cache transport requires a runtime interface supporting BF16 tensors" ], "layer_types": [ "kda", "kda", "kda", "mla", "kda", "kda", "kda", "mla", "kda", "kda", "kda", "mla", "kda", "kda", "kda", "mla", "kda", "kda", "kda", "mla", "kda", "kda", "kda", "mla" ], "note": "These are ordinary ONNX graphs, not an ORT GenAI or Foundry catalog integration package", "graphs": [ { "file": "onnx/forward.onnx", "bytes": 17651227, "sha256": "6b280be94ca4ca808a2a3f9810b564c9b1fdb61edacfe337607e94a61951ea1e", "structure_check": "PASS", "opset": [ { "domain": "", "version": 18 } ], "ir_version": 10, "inputs": [ { "name": "input_ids", "dtype": "INT64", "shape": [ 1, "tokens" ] }, { "name": "position_ids", "dtype": "INT64", "shape": [ 1, "tokens" ] } ], "outputs": [ { "name": "logits", "dtype": "FLOAT", "shape": [ 1, "tokens", 157184 ] } ], "mode": "forward", "source_tensor_count": 9283, "state_mapping": [] }, { "file": "onnx/prefill.onnx", "bytes": 17662815, "sha256": "7aa66dccedcbeb4745a8e8801fbd34aead84c196894b9efcc2adbcbaf1453ebf", "structure_check": "PASS", "opset": [ { "domain": "", "version": 18 } ], "ir_version": 10, "inputs": [ { "name": "input_ids", "dtype": "INT64", "shape": [ 1, "tokens" ] }, { "name": "position_ids", "dtype": "INT64", "shape": [ 1, "tokens" ] } ], "outputs": [ { "name": "logits", "dtype": "FLOAT", "shape": [ 1, "tokens", 157184 ] }, { "name": "present.0.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.0.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.0.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.0.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.1.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.1.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.1.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.1.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.2.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.2.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.2.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.2.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.3.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "name": "present.3.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "name": "present.4.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.4.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.4.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.4.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.5.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.5.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.5.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.5.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.6.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.6.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.6.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.6.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.7.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "name": "present.7.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "name": "present.8.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.8.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.8.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.8.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.9.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.9.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.9.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.9.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.10.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.10.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.10.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.10.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.11.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "name": "present.11.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "name": "present.12.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.12.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.12.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.12.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.13.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.13.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.13.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.13.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.14.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.14.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.14.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.14.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.15.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "name": "present.15.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "name": "present.16.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.16.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.16.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.16.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.17.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.17.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.17.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.17.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.18.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.18.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.18.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.18.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.19.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "name": "present.19.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "name": "present.20.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.20.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.20.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.20.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.21.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.21.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.21.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.21.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.22.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.22.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.22.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.22.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.23.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "name": "present.23.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] } ], "mode": "prefill", "source_tensor_count": 9283, "state_mapping": [ { "layer": 0, "kind": "kda", "key": "conv_q", "next_decode_input": "past.0.conv_q", "output": "present.0.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 0, "kind": "kda", "key": "conv_k", "next_decode_input": "past.0.conv_k", "output": "present.0.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 0, "kind": "kda", "key": "conv_v", "next_decode_input": "past.0.conv_v", "output": "present.0.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 0, "kind": "kda", "key": "state", "next_decode_input": "past.0.state", "output": "present.0.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 1, "kind": "kda", "key": "conv_q", "next_decode_input": "past.1.conv_q", "output": "present.1.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 1, "kind": "kda", "key": "conv_k", "next_decode_input": "past.1.conv_k", "output": "present.1.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 1, "kind": "kda", "key": "conv_v", "next_decode_input": "past.1.conv_v", "output": "present.1.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 1, "kind": "kda", "key": "state", "next_decode_input": "past.1.state", "output": "present.1.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 2, "kind": "kda", "key": "conv_q", "next_decode_input": "past.2.conv_q", "output": "present.2.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 2, "kind": "kda", "key": "conv_k", "next_decode_input": "past.2.conv_k", "output": "present.2.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 2, "kind": "kda", "key": "conv_v", "next_decode_input": "past.2.conv_v", "output": "present.2.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 2, "kind": "kda", "key": "state", "next_decode_input": "past.2.state", "output": "present.2.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 3, "kind": "mla", "key": "key", "next_decode_input": "past.3.key", "output": "present.3.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "layer": 3, "kind": "mla", "key": "value", "next_decode_input": "past.3.value", "output": "present.3.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "layer": 4, "kind": "kda", "key": "conv_q", "next_decode_input": "past.4.conv_q", "output": "present.4.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 4, "kind": "kda", "key": "conv_k", "next_decode_input": "past.4.conv_k", "output": "present.4.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 4, "kind": "kda", "key": "conv_v", "next_decode_input": "past.4.conv_v", "output": "present.4.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 4, "kind": "kda", "key": "state", "next_decode_input": "past.4.state", "output": "present.4.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 5, "kind": "kda", "key": "conv_q", "next_decode_input": "past.5.conv_q", "output": "present.5.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 5, "kind": "kda", "key": "conv_k", "next_decode_input": "past.5.conv_k", "output": "present.5.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 5, "kind": "kda", "key": "conv_v", "next_decode_input": "past.5.conv_v", "output": "present.5.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 5, "kind": "kda", "key": "state", "next_decode_input": "past.5.state", "output": "present.5.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 6, "kind": "kda", "key": "conv_q", "next_decode_input": "past.6.conv_q", "output": "present.6.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 6, "kind": "kda", "key": "conv_k", "next_decode_input": "past.6.conv_k", "output": "present.6.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 6, "kind": "kda", "key": "conv_v", "next_decode_input": "past.6.conv_v", "output": "present.6.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 6, "kind": "kda", "key": "state", "next_decode_input": "past.6.state", "output": "present.6.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 7, "kind": "mla", "key": "key", "next_decode_input": "past.7.key", "output": "present.7.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "layer": 7, "kind": "mla", "key": "value", "next_decode_input": "past.7.value", "output": "present.7.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "layer": 8, "kind": "kda", "key": "conv_q", "next_decode_input": "past.8.conv_q", "output": "present.8.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 8, "kind": "kda", "key": "conv_k", "next_decode_input": "past.8.conv_k", "output": "present.8.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 8, "kind": "kda", "key": "conv_v", "next_decode_input": "past.8.conv_v", "output": "present.8.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 8, "kind": "kda", "key": "state", "next_decode_input": "past.8.state", "output": "present.8.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 9, "kind": "kda", "key": "conv_q", "next_decode_input": "past.9.conv_q", "output": "present.9.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 9, "kind": "kda", "key": "conv_k", "next_decode_input": "past.9.conv_k", "output": "present.9.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 9, "kind": "kda", "key": "conv_v", "next_decode_input": "past.9.conv_v", "output": "present.9.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 9, "kind": "kda", "key": "state", "next_decode_input": "past.9.state", "output": "present.9.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 10, "kind": "kda", "key": "conv_q", "next_decode_input": "past.10.conv_q", "output": "present.10.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 10, "kind": "kda", "key": "conv_k", "next_decode_input": "past.10.conv_k", "output": "present.10.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 10, "kind": "kda", "key": "conv_v", "next_decode_input": "past.10.conv_v", "output": "present.10.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 10, "kind": "kda", "key": "state", "next_decode_input": "past.10.state", "output": "present.10.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 11, "kind": "mla", "key": "key", "next_decode_input": "past.11.key", "output": "present.11.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "layer": 11, "kind": "mla", "key": "value", "next_decode_input": "past.11.value", "output": "present.11.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "layer": 12, "kind": "kda", "key": "conv_q", "next_decode_input": "past.12.conv_q", "output": "present.12.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 12, "kind": "kda", "key": "conv_k", "next_decode_input": "past.12.conv_k", "output": "present.12.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 12, "kind": "kda", "key": "conv_v", "next_decode_input": "past.12.conv_v", "output": "present.12.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 12, "kind": "kda", "key": "state", "next_decode_input": "past.12.state", "output": "present.12.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 13, "kind": "kda", "key": "conv_q", "next_decode_input": "past.13.conv_q", "output": "present.13.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 13, "kind": "kda", "key": "conv_k", "next_decode_input": "past.13.conv_k", "output": "present.13.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 13, "kind": "kda", "key": "conv_v", "next_decode_input": "past.13.conv_v", "output": "present.13.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 13, "kind": "kda", "key": "state", "next_decode_input": "past.13.state", "output": "present.13.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 14, "kind": "kda", "key": "conv_q", "next_decode_input": "past.14.conv_q", "output": "present.14.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 14, "kind": "kda", "key": "conv_k", "next_decode_input": "past.14.conv_k", "output": "present.14.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 14, "kind": "kda", "key": "conv_v", "next_decode_input": "past.14.conv_v", "output": "present.14.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 14, "kind": "kda", "key": "state", "next_decode_input": "past.14.state", "output": "present.14.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 15, "kind": "mla", "key": "key", "next_decode_input": "past.15.key", "output": "present.15.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "layer": 15, "kind": "mla", "key": "value", "next_decode_input": "past.15.value", "output": "present.15.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "layer": 16, "kind": "kda", "key": "conv_q", "next_decode_input": "past.16.conv_q", "output": "present.16.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 16, "kind": "kda", "key": "conv_k", "next_decode_input": "past.16.conv_k", "output": "present.16.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 16, "kind": "kda", "key": "conv_v", "next_decode_input": "past.16.conv_v", "output": "present.16.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 16, "kind": "kda", "key": "state", "next_decode_input": "past.16.state", "output": "present.16.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 17, "kind": "kda", "key": "conv_q", "next_decode_input": "past.17.conv_q", "output": "present.17.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 17, "kind": "kda", "key": "conv_k", "next_decode_input": "past.17.conv_k", "output": "present.17.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 17, "kind": "kda", "key": "conv_v", "next_decode_input": "past.17.conv_v", "output": "present.17.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 17, "kind": "kda", "key": "state", "next_decode_input": "past.17.state", "output": "present.17.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 18, "kind": "kda", "key": "conv_q", "next_decode_input": "past.18.conv_q", "output": "present.18.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 18, "kind": "kda", "key": "conv_k", "next_decode_input": "past.18.conv_k", "output": "present.18.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 18, "kind": "kda", "key": "conv_v", "next_decode_input": "past.18.conv_v", "output": "present.18.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 18, "kind": "kda", "key": "state", "next_decode_input": "past.18.state", "output": "present.18.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 19, "kind": "mla", "key": "key", "next_decode_input": "past.19.key", "output": "present.19.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "layer": 19, "kind": "mla", "key": "value", "next_decode_input": "past.19.value", "output": "present.19.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "layer": 20, "kind": "kda", "key": "conv_q", "next_decode_input": "past.20.conv_q", "output": "present.20.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 20, "kind": "kda", "key": "conv_k", "next_decode_input": "past.20.conv_k", "output": "present.20.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 20, "kind": "kda", "key": "conv_v", "next_decode_input": "past.20.conv_v", "output": "present.20.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 20, "kind": "kda", "key": "state", "next_decode_input": "past.20.state", "output": "present.20.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 21, "kind": "kda", "key": "conv_q", "next_decode_input": "past.21.conv_q", "output": "present.21.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 21, "kind": "kda", "key": "conv_k", "next_decode_input": "past.21.conv_k", "output": "present.21.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 21, "kind": "kda", "key": "conv_v", "next_decode_input": "past.21.conv_v", "output": "present.21.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 21, "kind": "kda", "key": "state", "next_decode_input": "past.21.state", "output": "present.21.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 22, "kind": "kda", "key": "conv_q", "next_decode_input": "past.22.conv_q", "output": "present.22.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 22, "kind": "kda", "key": "conv_k", "next_decode_input": "past.22.conv_k", "output": "present.22.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 22, "kind": "kda", "key": "conv_v", "next_decode_input": "past.22.conv_v", "output": "present.22.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 22, "kind": "kda", "key": "state", "next_decode_input": "past.22.state", "output": "present.22.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 23, "kind": "mla", "key": "key", "next_decode_input": "past.23.key", "output": "present.23.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "layer": 23, "kind": "mla", "key": "value", "next_decode_input": "past.23.value", "output": "present.23.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] } ] }, { "file": "onnx/decode.onnx", "bytes": 17643627, "sha256": "f011364059e327222b3647910bbc20bf155157362342fa7ccb0eacb6784d54b5", "structure_check": "PASS", "opset": [ { "domain": "", "version": 18 } ], "ir_version": 10, "inputs": [ { "name": "input_ids", "dtype": "INT64", "shape": [ 1, 1 ] }, { "name": "position_ids", "dtype": "INT64", "shape": [ 1, 1 ] }, { "name": "past.0.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.0.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.0.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.0.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.1.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.1.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.1.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.1.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.2.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.2.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.2.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.2.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.3.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "past_tokens", 192 ] }, { "name": "past.3.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "past_tokens", 128 ] }, { "name": "past.4.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.4.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.4.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.4.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.5.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.5.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.5.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.5.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.6.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.6.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.6.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.6.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.7.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "past_tokens", 192 ] }, { "name": "past.7.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "past_tokens", 128 ] }, { "name": "past.8.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.8.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.8.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.8.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.9.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.9.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.9.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.9.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.10.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.10.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.10.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.10.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.11.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "past_tokens", 192 ] }, { "name": "past.11.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "past_tokens", 128 ] }, { "name": "past.12.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.12.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.12.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.12.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.13.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.13.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.13.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.13.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.14.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.14.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.14.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.14.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.15.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "past_tokens", 192 ] }, { "name": "past.15.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "past_tokens", 128 ] }, { "name": "past.16.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.16.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.16.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.16.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.17.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.17.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.17.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.17.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.18.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.18.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.18.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.18.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.19.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "past_tokens", 192 ] }, { "name": "past.19.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "past_tokens", 128 ] }, { "name": "past.20.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.20.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.20.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.20.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.21.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.21.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.21.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.21.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.22.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "past.22.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.22.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.22.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "past.23.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "past_tokens", 192 ] }, { "name": "past.23.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "past_tokens", 128 ] } ], "outputs": [ { "name": "logits", "dtype": "FLOAT", "shape": [ 1, 1, 157184 ] }, { "name": "present.0.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.0.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.0.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.0.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.1.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.1.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.1.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.1.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.2.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.2.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.2.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.2.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.3.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "name": "present.3.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "name": "present.4.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.4.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.4.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.4.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.5.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.5.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.5.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.5.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.6.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.6.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.6.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.6.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.7.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "name": "present.7.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "name": "present.8.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.8.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.8.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.8.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.9.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.9.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.9.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.9.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.10.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.10.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.10.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.10.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.11.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "name": "present.11.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "name": "present.12.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.12.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.12.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.12.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.13.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.13.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.13.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.13.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.14.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.14.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.14.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.14.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.15.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "name": "present.15.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "name": "present.16.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.16.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.16.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.16.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.17.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.17.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.17.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.17.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.18.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.18.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.18.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.18.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.19.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "name": "present.19.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "name": "present.20.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.20.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.20.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.20.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.21.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.21.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.21.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.21.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.22.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.22.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.22.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "name": "present.22.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "name": "present.23.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "name": "present.23.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] } ], "mode": "decode", "source_tensor_count": 9283, "state_mapping": [ { "layer": 0, "kind": "kda", "key": "conv_q", "next_decode_input": "past.0.conv_q", "output": "present.0.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 0, "kind": "kda", "key": "conv_k", "next_decode_input": "past.0.conv_k", "output": "present.0.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 0, "kind": "kda", "key": "conv_v", "next_decode_input": "past.0.conv_v", "output": "present.0.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 0, "kind": "kda", "key": "state", "next_decode_input": "past.0.state", "output": "present.0.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 1, "kind": "kda", "key": "conv_q", "next_decode_input": "past.1.conv_q", "output": "present.1.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 1, "kind": "kda", "key": "conv_k", "next_decode_input": "past.1.conv_k", "output": "present.1.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 1, "kind": "kda", "key": "conv_v", "next_decode_input": "past.1.conv_v", "output": "present.1.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 1, "kind": "kda", "key": "state", "next_decode_input": "past.1.state", "output": "present.1.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 2, "kind": "kda", "key": "conv_q", "next_decode_input": "past.2.conv_q", "output": "present.2.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 2, "kind": "kda", "key": "conv_k", "next_decode_input": "past.2.conv_k", "output": "present.2.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 2, "kind": "kda", "key": "conv_v", "next_decode_input": "past.2.conv_v", "output": "present.2.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 2, "kind": "kda", "key": "state", "next_decode_input": "past.2.state", "output": "present.2.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 3, "kind": "mla", "key": "key", "next_decode_input": "past.3.key", "output": "present.3.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "layer": 3, "kind": "mla", "key": "value", "next_decode_input": "past.3.value", "output": "present.3.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "layer": 4, "kind": "kda", "key": "conv_q", "next_decode_input": "past.4.conv_q", "output": "present.4.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 4, "kind": "kda", "key": "conv_k", "next_decode_input": "past.4.conv_k", "output": "present.4.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 4, "kind": "kda", "key": "conv_v", "next_decode_input": "past.4.conv_v", "output": "present.4.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 4, "kind": "kda", "key": "state", "next_decode_input": "past.4.state", "output": "present.4.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 5, "kind": "kda", "key": "conv_q", "next_decode_input": "past.5.conv_q", "output": "present.5.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 5, "kind": "kda", "key": "conv_k", "next_decode_input": "past.5.conv_k", "output": "present.5.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 5, "kind": "kda", "key": "conv_v", "next_decode_input": "past.5.conv_v", "output": "present.5.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 5, "kind": "kda", "key": "state", "next_decode_input": "past.5.state", "output": "present.5.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 6, "kind": "kda", "key": "conv_q", "next_decode_input": "past.6.conv_q", "output": "present.6.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 6, "kind": "kda", "key": "conv_k", "next_decode_input": "past.6.conv_k", "output": "present.6.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 6, "kind": "kda", "key": "conv_v", "next_decode_input": "past.6.conv_v", "output": "present.6.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 6, "kind": "kda", "key": "state", "next_decode_input": "past.6.state", "output": "present.6.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 7, "kind": "mla", "key": "key", "next_decode_input": "past.7.key", "output": "present.7.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "layer": 7, "kind": "mla", "key": "value", "next_decode_input": "past.7.value", "output": "present.7.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "layer": 8, "kind": "kda", "key": "conv_q", "next_decode_input": "past.8.conv_q", "output": "present.8.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 8, "kind": "kda", "key": "conv_k", "next_decode_input": "past.8.conv_k", "output": "present.8.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 8, "kind": "kda", "key": "conv_v", "next_decode_input": "past.8.conv_v", "output": "present.8.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 8, "kind": "kda", "key": "state", "next_decode_input": "past.8.state", "output": "present.8.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 9, "kind": "kda", "key": "conv_q", "next_decode_input": "past.9.conv_q", "output": "present.9.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 9, "kind": "kda", "key": "conv_k", "next_decode_input": "past.9.conv_k", "output": "present.9.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 9, "kind": "kda", "key": "conv_v", "next_decode_input": "past.9.conv_v", "output": "present.9.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 9, "kind": "kda", "key": "state", "next_decode_input": "past.9.state", "output": "present.9.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 10, "kind": "kda", "key": "conv_q", "next_decode_input": "past.10.conv_q", "output": "present.10.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 10, "kind": "kda", "key": "conv_k", "next_decode_input": "past.10.conv_k", "output": "present.10.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 10, "kind": "kda", "key": "conv_v", "next_decode_input": "past.10.conv_v", "output": "present.10.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 10, "kind": "kda", "key": "state", "next_decode_input": "past.10.state", "output": "present.10.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 11, "kind": "mla", "key": "key", "next_decode_input": "past.11.key", "output": "present.11.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "layer": 11, "kind": "mla", "key": "value", "next_decode_input": "past.11.value", "output": "present.11.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "layer": 12, "kind": "kda", "key": "conv_q", "next_decode_input": "past.12.conv_q", "output": "present.12.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 12, "kind": "kda", "key": "conv_k", "next_decode_input": "past.12.conv_k", "output": "present.12.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 12, "kind": "kda", "key": "conv_v", "next_decode_input": "past.12.conv_v", "output": "present.12.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 12, "kind": "kda", "key": "state", "next_decode_input": "past.12.state", "output": "present.12.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 13, "kind": "kda", "key": "conv_q", "next_decode_input": "past.13.conv_q", "output": "present.13.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 13, "kind": "kda", "key": "conv_k", "next_decode_input": "past.13.conv_k", "output": "present.13.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 13, "kind": "kda", "key": "conv_v", "next_decode_input": "past.13.conv_v", "output": "present.13.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 13, "kind": "kda", "key": "state", "next_decode_input": "past.13.state", "output": "present.13.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 14, "kind": "kda", "key": "conv_q", "next_decode_input": "past.14.conv_q", "output": "present.14.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 14, "kind": "kda", "key": "conv_k", "next_decode_input": "past.14.conv_k", "output": "present.14.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 14, "kind": "kda", "key": "conv_v", "next_decode_input": "past.14.conv_v", "output": "present.14.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 14, "kind": "kda", "key": "state", "next_decode_input": "past.14.state", "output": "present.14.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 15, "kind": "mla", "key": "key", "next_decode_input": "past.15.key", "output": "present.15.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "layer": 15, "kind": "mla", "key": "value", "next_decode_input": "past.15.value", "output": "present.15.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "layer": 16, "kind": "kda", "key": "conv_q", "next_decode_input": "past.16.conv_q", "output": "present.16.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 16, "kind": "kda", "key": "conv_k", "next_decode_input": "past.16.conv_k", "output": "present.16.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 16, "kind": "kda", "key": "conv_v", "next_decode_input": "past.16.conv_v", "output": "present.16.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 16, "kind": "kda", "key": "state", "next_decode_input": "past.16.state", "output": "present.16.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 17, "kind": "kda", "key": "conv_q", "next_decode_input": "past.17.conv_q", "output": "present.17.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 17, "kind": "kda", "key": "conv_k", "next_decode_input": "past.17.conv_k", "output": "present.17.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 17, "kind": "kda", "key": "conv_v", "next_decode_input": "past.17.conv_v", "output": "present.17.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 17, "kind": "kda", "key": "state", "next_decode_input": "past.17.state", "output": "present.17.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 18, "kind": "kda", "key": "conv_q", "next_decode_input": "past.18.conv_q", "output": "present.18.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 18, "kind": "kda", "key": "conv_k", "next_decode_input": "past.18.conv_k", "output": "present.18.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 18, "kind": "kda", "key": "conv_v", "next_decode_input": "past.18.conv_v", "output": "present.18.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 18, "kind": "kda", "key": "state", "next_decode_input": "past.18.state", "output": "present.18.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 19, "kind": "mla", "key": "key", "next_decode_input": "past.19.key", "output": "present.19.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "layer": 19, "kind": "mla", "key": "value", "next_decode_input": "past.19.value", "output": "present.19.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] }, { "layer": 20, "kind": "kda", "key": "conv_q", "next_decode_input": "past.20.conv_q", "output": "present.20.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 20, "kind": "kda", "key": "conv_k", "next_decode_input": "past.20.conv_k", "output": "present.20.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 20, "kind": "kda", "key": "conv_v", "next_decode_input": "past.20.conv_v", "output": "present.20.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 20, "kind": "kda", "key": "state", "next_decode_input": "past.20.state", "output": "present.20.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 21, "kind": "kda", "key": "conv_q", "next_decode_input": "past.21.conv_q", "output": "present.21.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 21, "kind": "kda", "key": "conv_k", "next_decode_input": "past.21.conv_k", "output": "present.21.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 21, "kind": "kda", "key": "conv_v", "next_decode_input": "past.21.conv_v", "output": "present.21.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 21, "kind": "kda", "key": "state", "next_decode_input": "past.21.state", "output": "present.21.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 22, "kind": "kda", "key": "conv_q", "next_decode_input": "past.22.conv_q", "output": "present.22.conv_q", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 22, "kind": "kda", "key": "conv_k", "next_decode_input": "past.22.conv_k", "output": "present.22.conv_k", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 22, "kind": "kda", "key": "conv_v", "next_decode_input": "past.22.conv_v", "output": "present.22.conv_v", "dtype": "BFLOAT16", "shape": [ 1, 2048, 4 ] }, { "layer": 22, "kind": "kda", "key": "state", "next_decode_input": "past.22.state", "output": "present.22.state", "dtype": "FLOAT", "shape": [ 1, 16, 128, 128 ] }, { "layer": 23, "kind": "mla", "key": "key", "next_decode_input": "past.23.key", "output": "present.23.key", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 192 ] }, { "layer": 23, "kind": "mla", "key": "value", "next_decode_input": "past.23.value", "output": "present.23.value", "dtype": "BFLOAT16", "shape": [ 1, 16, "total_tokens", 128 ] } ] } ], "assets": [ { "file": "chat_template.jinja", "bytes": 6041, "sha256": "eb6226c94ae38058f875d159f86a206b3a165828c0e7d6bda664ae14667f798a" }, { "file": "config.json", "bytes": 2306, "sha256": "cf935efa3c3536bfefca7a947a6249c649d1b090db00264596552eae3868673f" }, { "file": "generation_config.json", "bytes": 57, "sha256": "7bf8894f5838e096a6cb6147707a98733e730985618c8d72e3e0d49c38a273e0" }, { "file": "special_tokens_map.json", "bytes": 580, "sha256": "69b63b9f81044ead642d16a5fdc01bcc737dc1183746485c8397ab14d3126614" }, { "file": "tokenizer.json", "bytes": 12205732, "sha256": "40fb9d7d7795b8bd305aeff39ce9963f3f450915b9553f2938e009be9a1fed60" }, { "file": "tokenizer_config.json", "bytes": 49672, "sha256": "489ec35d61d99a85d730ba48f7b38df2eeff966b1765e0de2f21a1b645759ed4" } ], "external_weights": { "file": "onnx/weights/model.bin", "bytes": 15786859904, "sha256": "bc3030ea87f8b8ed93fa403db30d06afc1184cca2b84347549128bfe3a0156dc" }, "runtime_validation": "NOT_RUN", "numerical_alignment": "NOT_RUN", "path_base": "package_root" }