lingeasy's picture
Release vulcan-decision-shell-v1
ade659b verified
Raw History Blame Contribute Delete
7.18 kB
{
"schema": "vulcan-decision-shell-runtime-catalog-v1",
"interface": {
"inputs": [
"input_ids",
"attention_mask"
],
"input_shape": [
"batch",
"sequence"
],
"input_type": "int64",
"output": "logits",
"output_shape": [
"batch",
3
],
"output_type": "float32",
"labels": [
"allow",
"review",
"block"
],
"padding": "right"
},
"measured_on": "RTX 5080 16 GB, ONNX Runtime 1.30.0, CUDA 13; CPU timings with 16 threads; WebGPU timings through the ONNX Runtime WebGPU plugin on the same GPU (VulcanCode judge host benchmark)",
"families": {
"mini": {
"parameters": 199088899,
"base_model": "answerdotai/ModernBERT-base",
"recommended": {
"cuda": "fp16",
"cpu": "fp32",
"webgpu": "fp16"
},
"runtime": {
"max_tokens": 8192,
"max_request_items": 256,
"micro_batch": 8,
"padded_batch_tokens": 8192,
"allow_margin": 1.0,
"gpu_memory_limit_mib": 12288,
"arena_trim_above_tokens": 4096,
"encoding": {
"kind": "marker-head",
"head_tokens": 288,
"option_tokens": 48
}
},
"limits": "max_tokens is 8192, the base model context: the last training stage used 4,096-token rows and longer calls are unproven (above about 12,000 tokens a 16 GB GPU also runs out of memory).",
"not_included": "bf16 formats change 34-37 of 300 sample decisions; int8-mlp changes 6 and is no faster. They stay in the vulcan-decision-shell-v1-Mini-ONNX repository with their notes.",
"formats": {
"fp16": {
"path": "mini/fp16",
"devices": [
"cuda",
"webgpu"
],
"precision": "fp16 projections, fp32 attention",
"manifest_sha256": "dc57ba263b4b32c1443b78b42ce81371aed85f93fb3f3afd5e5f77afd059dbf9",
"total_bytes": 486866780,
"note": "Matrix multiplications in FP16, attention in FP32. Same decisions as fp32 on the sample. CUDA and WebGPU only: ONNX Runtime 1.30 crashes running FP16 Mini graphs on CPU.",
"measured": {
"cuda_one_call_ms": 13.8,
"cuda_ten_calls_ms": 39.6,
"webgpu_ten_calls_ms": 395,
"sample_decision_changes_vs_fp32": 0
}
},
"fp32": {
"path": "mini/fp32",
"devices": [
"cuda",
"cpu",
"webgpu"
],
"precision": "fp32",
"manifest_sha256": "eb3d48b93d030b9f67bbcf7fcea1179b854287b44e5c16b72a8e0be9694c194e",
"total_bytes": 800073215,
"note": "Full FP32. The CPU format; on CUDA it is slower than fp16 for the same decisions.",
"measured": {
"cuda_one_call_ms": 14.9,
"cuda_ten_calls_ms": 68.3,
"cpu_one_call_ms": 96.9,
"cpu_ten_calls_ms": 1179,
"webgpu_ten_calls_ms": 521
}
}
}
},
"flash": {
"parameters": 361821120,
"base_model": "HuggingFaceTB/SmolLM2-360M",
"recommended": {
"cuda": "fp16",
"cpu": "fp16",
"webgpu": "fp16"
},
"runtime": {
"max_tokens": 32768,
"max_request_items": 256,
"micro_batch": 16,
"padded_batch_tokens": 8192,
"allow_margin": 0.0,
"gpu_memory_limit_mib": 12288,
"arena_trim_above_tokens": 4096,
"encoding": {
"kind": "decision-prompt",
"pad_token_id": 0
}
},
"limits": "CPU is usable for short calls only: long calls take seconds to minutes.",
"not_included": "Full-FP16 graphs change decisions on long calls (logit drift up to 11); the original FP32 attention runs out of memory above about 8,000 tokens.",
"formats": {
"fp16": {
"path": "flash/fp16",
"devices": [
"cuda",
"cpu",
"webgpu"
],
"precision": "fp16 projections, fp32 norms, fp16 flash attention",
"manifest_sha256": "075e61a05063272b13d966c1f9b6e55ce99acbf5055de16b675ae1b442e6ffed",
"total_bytes": 822382021,
"note": "Matrix multiplications in FP16, norms in FP32, flash attention. No decision change against FP32 on the sample or on long calls; a 32,757-token call runs in 1.5 s on a 16 GB GPU.",
"measured": {
"cuda_one_call_ms": 16.2,
"cuda_ten_calls_ms": 48.3,
"cpu_one_call_ms": 182.8,
"cpu_ten_calls_ms": 1997,
"webgpu_ten_calls_ms": 174,
"sample_decision_changes_vs_fp32": 0,
"long_decision_changes_vs_fp32": 0
}
},
"fp32": {
"path": "flash/fp32",
"devices": [
"cuda",
"cpu",
"webgpu"
],
"precision": "fp32, fp16 flash attention",
"manifest_sha256": "6b652427b61922ea573fd3509ed41b47cbc6037ffa47db7d6edf325cd12c4374",
"total_bytes": 1451479031,
"note": "FP32 weights and arithmetic with FP16 flash attention. Closest to the trained model, twice the size of fp16 and slower.",
"measured": {
"cuda_one_call_ms": 19.7,
"cuda_ten_calls_ms": 95.0,
"webgpu_ten_calls_ms": 195,
"sample_decision_changes_vs_fp32": 0,
"long_decision_changes_vs_fp32": 0
}
}
}
},
"standard": {
"parameters": 752393024,
"base_model": "Qwen/Qwen3.5-0.8B-Base",
"recommended": {
"cuda": "fp16",
"cpu": "fp16",
"webgpu": "fp16"
},
"runtime": {
"max_tokens": 32768,
"max_request_items": 256,
"micro_batch": 32,
"padded_batch_tokens": 8192,
"allow_margin": 0.0,
"gpu_memory_limit_mib": 12288,
"arena_trim_above_tokens": 4096,
"encoding": {
"kind": "decision-prompt",
"pad_token_id": 248044
}
},
"limits": "CPU runs about 0.35 s per short call.",
"not_included": null,
"formats": {
"fp16": {
"path": "standard/fp16",
"devices": [
"cuda",
"cpu",
"webgpu"
],
"precision": "fp16",
"manifest_sha256": "de1d7e101998b4ac978038c2ae3251a0fdb86567ec22b263fe7cec5b76b26ade",
"total_bytes": 1533602510,
"note": "The published FP16 graph with unused state inputs and outputs removed; identical scores. A 32,745-token call runs in 2.8 s on a 16 GB GPU.",
"measured": {
"cuda_one_call_ms": 18.1,
"cuda_ten_calls_ms": 125.7,
"cpu_one_call_ms": 347.9,
"cpu_ten_calls_ms": 4118,
"webgpu_ten_calls_ms": 305,
"sample_decision_changes_vs_published": 0
}
}
}
}
}
}