Download model_interface.json from onnx-community/Ling-3.0-tiny-ONNX: direct link, hf CLI and curl.
- Browser
- Download file 100 kB
-
https://huggingface.co/onnx-community/Ling-3.0-tiny-ONNX/resolve/main/model_interface.json
- Command line
-
hf download hf://onnx-community/Ling-3.0-tiny-ONNX/model_interface.json
-
curl -L -o model_interface.json https://huggingface.co/onnx-community/Ling-3.0-tiny-ONNX/resolve/main/model_interface.json
100 kB
{
"batch_size": 1,
"padding_supported": false,
"minimum_tokens": 1,
"native_activation_dtype": "BFLOAT16",
"logits_dtype": "FLOAT",
"constraints": [
"input_ids and position_ids have identical shape; tokens must be in vocabulary range",
"forward/prefill positions are contiguous starting at0 and states initialize to zero",
"decode consumes one token; position_ids equals the consumed prefix length",
"all MLA past caches share the same prefix length; all layers belong to the same sequence",
"feed every present cache into its mapped past input on the next decode step",
"forward omits states; prefill returns states; decode consumes and returns states",
"BF16 cache transport requires a runtime interface supporting BF16 tensors"
],
"layer_types": [
"kda",
"kda",
"kda",
"mla",
"kda",
"kda",
"kda",
"mla",
"kda",
"kda",
"kda",
"mla",
"kda",
"kda",
"kda",
"mla",
"kda",
"kda",
"kda",
"mla",
"kda",
"kda",
"kda",
"mla"
],
"note": "These are ordinary ONNX graphs, not an ORT GenAI or Foundry catalog integration package",
"graphs": [
{
"file": "onnx/forward.onnx",
"bytes": 17651227,
"sha256": "6b280be94ca4ca808a2a3f9810b564c9b1fdb61edacfe337607e94a61951ea1e",
"structure_check": "PASS",
"opset": [
{
"domain": "",
"version": 18
}
],
"ir_version": 10,
"inputs": [
{
"name": "input_ids",
"dtype": "INT64",
"shape": [
1,
"tokens"
]
},
{
"name": "position_ids",
"dtype": "INT64",
"shape": [
1,
"tokens"
]
}
],
"outputs": [
{
"name": "logits",
"dtype": "FLOAT",
"shape": [
1,
"tokens",
157184
]
}
],
"mode": "forward",
"source_tensor_count": 9283,
"state_mapping": []
},
{
"file": "onnx/prefill.onnx",
"bytes": 17662815,
"sha256": "7aa66dccedcbeb4745a8e8801fbd34aead84c196894b9efcc2adbcbaf1453ebf",
"structure_check": "PASS",
"opset": [
{
"domain": "",
"version": 18
}
],
"ir_version": 10,
"inputs": [
{
"name": "input_ids",
"dtype": "INT64",
"shape": [
1,
"tokens"
]
},
{
"name": "position_ids",
"dtype": "INT64",
"shape": [
1,
"tokens"
]
}
],
"outputs": [
{
"name": "logits",
"dtype": "FLOAT",
"shape": [
1,
"tokens",
157184
]
},
{
"name": "present.0.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.0.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.0.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.0.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.1.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.1.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.1.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.1.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.2.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.2.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.2.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.2.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.3.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"name": "present.3.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"name": "present.4.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.4.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.4.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.4.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.5.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.5.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.5.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.5.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.6.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.6.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.6.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.6.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.7.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"name": "present.7.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"name": "present.8.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.8.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.8.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.8.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.9.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.9.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.9.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.9.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.10.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.10.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.10.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.10.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.11.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"name": "present.11.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"name": "present.12.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.12.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.12.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.12.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.13.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.13.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.13.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.13.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.14.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.14.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.14.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.14.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.15.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"name": "present.15.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"name": "present.16.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.16.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.16.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.16.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.17.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.17.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.17.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.17.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.18.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.18.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.18.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.18.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.19.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"name": "present.19.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"name": "present.20.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.20.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.20.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.20.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.21.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.21.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.21.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.21.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.22.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.22.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.22.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.22.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.23.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"name": "present.23.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
}
],
"mode": "prefill",
"source_tensor_count": 9283,
"state_mapping": [
{
"layer": 0,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.0.conv_q",
"output": "present.0.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 0,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.0.conv_k",
"output": "present.0.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 0,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.0.conv_v",
"output": "present.0.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 0,
"kind": "kda",
"key": "state",
"next_decode_input": "past.0.state",
"output": "present.0.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 1,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.1.conv_q",
"output": "present.1.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 1,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.1.conv_k",
"output": "present.1.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 1,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.1.conv_v",
"output": "present.1.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 1,
"kind": "kda",
"key": "state",
"next_decode_input": "past.1.state",
"output": "present.1.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 2,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.2.conv_q",
"output": "present.2.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 2,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.2.conv_k",
"output": "present.2.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 2,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.2.conv_v",
"output": "present.2.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 2,
"kind": "kda",
"key": "state",
"next_decode_input": "past.2.state",
"output": "present.2.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 3,
"kind": "mla",
"key": "key",
"next_decode_input": "past.3.key",
"output": "present.3.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"layer": 3,
"kind": "mla",
"key": "value",
"next_decode_input": "past.3.value",
"output": "present.3.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"layer": 4,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.4.conv_q",
"output": "present.4.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 4,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.4.conv_k",
"output": "present.4.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 4,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.4.conv_v",
"output": "present.4.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 4,
"kind": "kda",
"key": "state",
"next_decode_input": "past.4.state",
"output": "present.4.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 5,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.5.conv_q",
"output": "present.5.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 5,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.5.conv_k",
"output": "present.5.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 5,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.5.conv_v",
"output": "present.5.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 5,
"kind": "kda",
"key": "state",
"next_decode_input": "past.5.state",
"output": "present.5.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 6,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.6.conv_q",
"output": "present.6.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 6,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.6.conv_k",
"output": "present.6.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 6,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.6.conv_v",
"output": "present.6.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 6,
"kind": "kda",
"key": "state",
"next_decode_input": "past.6.state",
"output": "present.6.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 7,
"kind": "mla",
"key": "key",
"next_decode_input": "past.7.key",
"output": "present.7.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"layer": 7,
"kind": "mla",
"key": "value",
"next_decode_input": "past.7.value",
"output": "present.7.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"layer": 8,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.8.conv_q",
"output": "present.8.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 8,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.8.conv_k",
"output": "present.8.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 8,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.8.conv_v",
"output": "present.8.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 8,
"kind": "kda",
"key": "state",
"next_decode_input": "past.8.state",
"output": "present.8.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 9,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.9.conv_q",
"output": "present.9.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 9,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.9.conv_k",
"output": "present.9.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 9,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.9.conv_v",
"output": "present.9.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 9,
"kind": "kda",
"key": "state",
"next_decode_input": "past.9.state",
"output": "present.9.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 10,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.10.conv_q",
"output": "present.10.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 10,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.10.conv_k",
"output": "present.10.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 10,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.10.conv_v",
"output": "present.10.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 10,
"kind": "kda",
"key": "state",
"next_decode_input": "past.10.state",
"output": "present.10.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 11,
"kind": "mla",
"key": "key",
"next_decode_input": "past.11.key",
"output": "present.11.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"layer": 11,
"kind": "mla",
"key": "value",
"next_decode_input": "past.11.value",
"output": "present.11.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"layer": 12,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.12.conv_q",
"output": "present.12.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 12,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.12.conv_k",
"output": "present.12.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 12,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.12.conv_v",
"output": "present.12.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 12,
"kind": "kda",
"key": "state",
"next_decode_input": "past.12.state",
"output": "present.12.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 13,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.13.conv_q",
"output": "present.13.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 13,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.13.conv_k",
"output": "present.13.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 13,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.13.conv_v",
"output": "present.13.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 13,
"kind": "kda",
"key": "state",
"next_decode_input": "past.13.state",
"output": "present.13.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 14,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.14.conv_q",
"output": "present.14.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 14,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.14.conv_k",
"output": "present.14.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 14,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.14.conv_v",
"output": "present.14.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 14,
"kind": "kda",
"key": "state",
"next_decode_input": "past.14.state",
"output": "present.14.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 15,
"kind": "mla",
"key": "key",
"next_decode_input": "past.15.key",
"output": "present.15.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"layer": 15,
"kind": "mla",
"key": "value",
"next_decode_input": "past.15.value",
"output": "present.15.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"layer": 16,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.16.conv_q",
"output": "present.16.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 16,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.16.conv_k",
"output": "present.16.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 16,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.16.conv_v",
"output": "present.16.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 16,
"kind": "kda",
"key": "state",
"next_decode_input": "past.16.state",
"output": "present.16.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 17,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.17.conv_q",
"output": "present.17.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 17,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.17.conv_k",
"output": "present.17.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 17,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.17.conv_v",
"output": "present.17.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 17,
"kind": "kda",
"key": "state",
"next_decode_input": "past.17.state",
"output": "present.17.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 18,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.18.conv_q",
"output": "present.18.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 18,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.18.conv_k",
"output": "present.18.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 18,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.18.conv_v",
"output": "present.18.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 18,
"kind": "kda",
"key": "state",
"next_decode_input": "past.18.state",
"output": "present.18.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 19,
"kind": "mla",
"key": "key",
"next_decode_input": "past.19.key",
"output": "present.19.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"layer": 19,
"kind": "mla",
"key": "value",
"next_decode_input": "past.19.value",
"output": "present.19.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"layer": 20,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.20.conv_q",
"output": "present.20.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 20,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.20.conv_k",
"output": "present.20.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 20,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.20.conv_v",
"output": "present.20.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 20,
"kind": "kda",
"key": "state",
"next_decode_input": "past.20.state",
"output": "present.20.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 21,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.21.conv_q",
"output": "present.21.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 21,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.21.conv_k",
"output": "present.21.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 21,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.21.conv_v",
"output": "present.21.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 21,
"kind": "kda",
"key": "state",
"next_decode_input": "past.21.state",
"output": "present.21.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 22,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.22.conv_q",
"output": "present.22.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 22,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.22.conv_k",
"output": "present.22.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 22,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.22.conv_v",
"output": "present.22.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 22,
"kind": "kda",
"key": "state",
"next_decode_input": "past.22.state",
"output": "present.22.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 23,
"kind": "mla",
"key": "key",
"next_decode_input": "past.23.key",
"output": "present.23.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"layer": 23,
"kind": "mla",
"key": "value",
"next_decode_input": "past.23.value",
"output": "present.23.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
}
]
},
{
"file": "onnx/decode.onnx",
"bytes": 17643627,
"sha256": "f011364059e327222b3647910bbc20bf155157362342fa7ccb0eacb6784d54b5",
"structure_check": "PASS",
"opset": [
{
"domain": "",
"version": 18
}
],
"ir_version": 10,
"inputs": [
{
"name": "input_ids",
"dtype": "INT64",
"shape": [
1,
1
]
},
{
"name": "position_ids",
"dtype": "INT64",
"shape": [
1,
1
]
},
{
"name": "past.0.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.0.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.0.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.0.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.1.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.1.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.1.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.1.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.2.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.2.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.2.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.2.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.3.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"past_tokens",
192
]
},
{
"name": "past.3.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"past_tokens",
128
]
},
{
"name": "past.4.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.4.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.4.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.4.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.5.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.5.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.5.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.5.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.6.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.6.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.6.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.6.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.7.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"past_tokens",
192
]
},
{
"name": "past.7.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"past_tokens",
128
]
},
{
"name": "past.8.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.8.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.8.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.8.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.9.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.9.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.9.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.9.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.10.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.10.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.10.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.10.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.11.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"past_tokens",
192
]
},
{
"name": "past.11.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"past_tokens",
128
]
},
{
"name": "past.12.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.12.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.12.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.12.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.13.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.13.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.13.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.13.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.14.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.14.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.14.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.14.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.15.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"past_tokens",
192
]
},
{
"name": "past.15.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"past_tokens",
128
]
},
{
"name": "past.16.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.16.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.16.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.16.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.17.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.17.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.17.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.17.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.18.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.18.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.18.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.18.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.19.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"past_tokens",
192
]
},
{
"name": "past.19.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"past_tokens",
128
]
},
{
"name": "past.20.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.20.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.20.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.20.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.21.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.21.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.21.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.21.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.22.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "past.22.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.22.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.22.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "past.23.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"past_tokens",
192
]
},
{
"name": "past.23.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"past_tokens",
128
]
}
],
"outputs": [
{
"name": "logits",
"dtype": "FLOAT",
"shape": [
1,
1,
157184
]
},
{
"name": "present.0.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.0.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.0.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.0.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.1.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.1.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.1.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.1.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.2.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.2.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.2.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.2.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.3.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"name": "present.3.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"name": "present.4.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.4.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.4.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.4.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.5.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.5.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.5.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.5.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.6.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.6.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.6.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.6.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.7.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"name": "present.7.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"name": "present.8.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.8.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.8.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.8.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.9.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.9.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.9.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.9.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.10.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.10.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.10.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.10.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.11.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"name": "present.11.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"name": "present.12.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.12.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.12.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.12.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.13.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.13.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.13.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.13.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.14.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.14.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.14.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.14.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.15.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"name": "present.15.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"name": "present.16.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.16.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.16.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.16.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.17.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.17.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.17.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.17.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.18.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.18.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.18.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.18.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.19.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"name": "present.19.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"name": "present.20.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.20.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.20.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.20.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.21.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.21.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.21.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.21.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.22.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.22.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.22.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"name": "present.22.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"name": "present.23.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"name": "present.23.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
}
],
"mode": "decode",
"source_tensor_count": 9283,
"state_mapping": [
{
"layer": 0,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.0.conv_q",
"output": "present.0.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 0,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.0.conv_k",
"output": "present.0.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 0,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.0.conv_v",
"output": "present.0.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 0,
"kind": "kda",
"key": "state",
"next_decode_input": "past.0.state",
"output": "present.0.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 1,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.1.conv_q",
"output": "present.1.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 1,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.1.conv_k",
"output": "present.1.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 1,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.1.conv_v",
"output": "present.1.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 1,
"kind": "kda",
"key": "state",
"next_decode_input": "past.1.state",
"output": "present.1.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 2,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.2.conv_q",
"output": "present.2.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 2,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.2.conv_k",
"output": "present.2.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 2,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.2.conv_v",
"output": "present.2.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 2,
"kind": "kda",
"key": "state",
"next_decode_input": "past.2.state",
"output": "present.2.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 3,
"kind": "mla",
"key": "key",
"next_decode_input": "past.3.key",
"output": "present.3.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"layer": 3,
"kind": "mla",
"key": "value",
"next_decode_input": "past.3.value",
"output": "present.3.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"layer": 4,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.4.conv_q",
"output": "present.4.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 4,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.4.conv_k",
"output": "present.4.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 4,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.4.conv_v",
"output": "present.4.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 4,
"kind": "kda",
"key": "state",
"next_decode_input": "past.4.state",
"output": "present.4.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 5,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.5.conv_q",
"output": "present.5.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 5,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.5.conv_k",
"output": "present.5.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 5,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.5.conv_v",
"output": "present.5.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 5,
"kind": "kda",
"key": "state",
"next_decode_input": "past.5.state",
"output": "present.5.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 6,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.6.conv_q",
"output": "present.6.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 6,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.6.conv_k",
"output": "present.6.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 6,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.6.conv_v",
"output": "present.6.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 6,
"kind": "kda",
"key": "state",
"next_decode_input": "past.6.state",
"output": "present.6.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 7,
"kind": "mla",
"key": "key",
"next_decode_input": "past.7.key",
"output": "present.7.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"layer": 7,
"kind": "mla",
"key": "value",
"next_decode_input": "past.7.value",
"output": "present.7.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"layer": 8,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.8.conv_q",
"output": "present.8.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 8,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.8.conv_k",
"output": "present.8.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 8,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.8.conv_v",
"output": "present.8.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 8,
"kind": "kda",
"key": "state",
"next_decode_input": "past.8.state",
"output": "present.8.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 9,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.9.conv_q",
"output": "present.9.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 9,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.9.conv_k",
"output": "present.9.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 9,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.9.conv_v",
"output": "present.9.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 9,
"kind": "kda",
"key": "state",
"next_decode_input": "past.9.state",
"output": "present.9.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 10,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.10.conv_q",
"output": "present.10.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 10,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.10.conv_k",
"output": "present.10.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 10,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.10.conv_v",
"output": "present.10.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 10,
"kind": "kda",
"key": "state",
"next_decode_input": "past.10.state",
"output": "present.10.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 11,
"kind": "mla",
"key": "key",
"next_decode_input": "past.11.key",
"output": "present.11.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"layer": 11,
"kind": "mla",
"key": "value",
"next_decode_input": "past.11.value",
"output": "present.11.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"layer": 12,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.12.conv_q",
"output": "present.12.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 12,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.12.conv_k",
"output": "present.12.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 12,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.12.conv_v",
"output": "present.12.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 12,
"kind": "kda",
"key": "state",
"next_decode_input": "past.12.state",
"output": "present.12.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 13,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.13.conv_q",
"output": "present.13.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 13,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.13.conv_k",
"output": "present.13.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 13,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.13.conv_v",
"output": "present.13.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 13,
"kind": "kda",
"key": "state",
"next_decode_input": "past.13.state",
"output": "present.13.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 14,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.14.conv_q",
"output": "present.14.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 14,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.14.conv_k",
"output": "present.14.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 14,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.14.conv_v",
"output": "present.14.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 14,
"kind": "kda",
"key": "state",
"next_decode_input": "past.14.state",
"output": "present.14.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 15,
"kind": "mla",
"key": "key",
"next_decode_input": "past.15.key",
"output": "present.15.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"layer": 15,
"kind": "mla",
"key": "value",
"next_decode_input": "past.15.value",
"output": "present.15.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"layer": 16,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.16.conv_q",
"output": "present.16.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 16,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.16.conv_k",
"output": "present.16.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 16,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.16.conv_v",
"output": "present.16.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 16,
"kind": "kda",
"key": "state",
"next_decode_input": "past.16.state",
"output": "present.16.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 17,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.17.conv_q",
"output": "present.17.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 17,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.17.conv_k",
"output": "present.17.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 17,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.17.conv_v",
"output": "present.17.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 17,
"kind": "kda",
"key": "state",
"next_decode_input": "past.17.state",
"output": "present.17.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 18,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.18.conv_q",
"output": "present.18.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 18,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.18.conv_k",
"output": "present.18.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 18,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.18.conv_v",
"output": "present.18.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 18,
"kind": "kda",
"key": "state",
"next_decode_input": "past.18.state",
"output": "present.18.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 19,
"kind": "mla",
"key": "key",
"next_decode_input": "past.19.key",
"output": "present.19.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"layer": 19,
"kind": "mla",
"key": "value",
"next_decode_input": "past.19.value",
"output": "present.19.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
},
{
"layer": 20,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.20.conv_q",
"output": "present.20.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 20,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.20.conv_k",
"output": "present.20.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 20,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.20.conv_v",
"output": "present.20.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 20,
"kind": "kda",
"key": "state",
"next_decode_input": "past.20.state",
"output": "present.20.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 21,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.21.conv_q",
"output": "present.21.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 21,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.21.conv_k",
"output": "present.21.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 21,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.21.conv_v",
"output": "present.21.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 21,
"kind": "kda",
"key": "state",
"next_decode_input": "past.21.state",
"output": "present.21.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 22,
"kind": "kda",
"key": "conv_q",
"next_decode_input": "past.22.conv_q",
"output": "present.22.conv_q",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 22,
"kind": "kda",
"key": "conv_k",
"next_decode_input": "past.22.conv_k",
"output": "present.22.conv_k",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 22,
"kind": "kda",
"key": "conv_v",
"next_decode_input": "past.22.conv_v",
"output": "present.22.conv_v",
"dtype": "BFLOAT16",
"shape": [
1,
2048,
4
]
},
{
"layer": 22,
"kind": "kda",
"key": "state",
"next_decode_input": "past.22.state",
"output": "present.22.state",
"dtype": "FLOAT",
"shape": [
1,
16,
128,
128
]
},
{
"layer": 23,
"kind": "mla",
"key": "key",
"next_decode_input": "past.23.key",
"output": "present.23.key",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
192
]
},
{
"layer": 23,
"kind": "mla",
"key": "value",
"next_decode_input": "past.23.value",
"output": "present.23.value",
"dtype": "BFLOAT16",
"shape": [
1,
16,
"total_tokens",
128
]
}
]
}
],
"assets": [
{
"file": "chat_template.jinja",
"bytes": 6041,
"sha256": "eb6226c94ae38058f875d159f86a206b3a165828c0e7d6bda664ae14667f798a"
},
{
"file": "config.json",
"bytes": 2306,
"sha256": "cf935efa3c3536bfefca7a947a6249c649d1b090db00264596552eae3868673f"
},
{
"file": "generation_config.json",
"bytes": 57,
"sha256": "7bf8894f5838e096a6cb6147707a98733e730985618c8d72e3e0d49c38a273e0"
},
{
"file": "special_tokens_map.json",
"bytes": 580,
"sha256": "69b63b9f81044ead642d16a5fdc01bcc737dc1183746485c8397ab14d3126614"
},
{
"file": "tokenizer.json",
"bytes": 12205732,
"sha256": "40fb9d7d7795b8bd305aeff39ce9963f3f450915b9553f2938e009be9a1fed60"
},
{
"file": "tokenizer_config.json",
"bytes": 49672,
"sha256": "489ec35d61d99a85d730ba48f7b38df2eeff966b1765e0de2f21a1b645759ed4"
}
],
"external_weights": {
"file": "onnx/weights/model.bin",
"bytes": 15786859904,
"sha256": "bc3030ea87f8b8ed93fa403db30d06afc1184cca2b84347549128bfe3a0156dc"
},
"runtime_validation": "NOT_RUN",
"numerical_alignment": "NOT_RUN",
"path_base": "package_root"
}
{
"batch_size": 1,
"padding_supported": false,
"minimum_tokens": 1,
"native_activation_dtype": "BFLOAT16",
"logits_dtype": "FLOAT",
"constraints": [
"input_ids and position_ids have identical shape; tokens must be in vocabulary range",
"forward/prefill positions are contiguous starting at0 and states initialize to zero",
"decode consumes one token; position_ids equals the consumed prefix length",
"all MLA past caches share the same prefix length; all layers belong to the same sequence",
"feed every present cache into its mapped past input on the next decode step",
"forward omits states; prefill returns states; decode consumes and returns states",
"BF16 cache transport requires a runtime interface supporting BF16 tensors"
],
"layer_types": [
"kda",
"kda",
"kda",
"mla",
"kda",
"kda",
"kda",
"mla",
"kda",
"kda",
"kda",
"mla",
"kda",
"kda",
"kda",
"mla",
"kda",
"kda",
"kda",
"mla",
"kda",
"kda",
"kda",
"mla"
],
"note": "These are ordinary ONNX graphs, not an ORT GenAI or Foundry catalog integration package",
"graphs": [
{
"file": "onnx/forward.onnx",
"bytes": 17651227,
"sha256": "6b280be94ca4ca808a2a3f9810b564c9b1fdb61edacfe337607e94a61951ea1e",
"structure_check": "PASS",
"opset": [
{
"domain": "",
"version": 18
}
],
"ir_version": 10,
"inputs": [
{
"name": "input_ids",
"dtype": "INT64",
"shape": [
1,
"tokens"
]
},
{
"name": "position_ids",
"dtype": "INT64",
"shape": [
1,
"tokens"
]
}
],
"outputs": [
{
"name": "logits",
"dtype": "FLOAT",
"shape": [
1,
"tokens",
157184
]
}
],
"mode": "forward",
"source_tensor_count": 9283,
"state_mapping": []
},
{
"file": "onnx/prefill.onnx",
"bytes": 17662815,
"sha256": "7aa66dccedcbeb4745a8e8801fbd34aead84c196894b9efcc2adbcbaf1453ebf",
"structure_check": "PASS",
"opset": [
{
"domain": "",
"version": 18
}
],
"ir_version": 10,