| {"problems":[{"board":"L1","id":"001_attention_softmax_dropout_value_matmul_backward","name":"Attention softmax dropout value matmul backward","group":"Backward pass","op":"attention softmax + dropout + PV backward (GQA 80/8/128)","direction":"backward","model":"ByteDance/Seed-Seed-OSS-36B-Instruct","workloads":16},{"board":"L1","id":"002_vae_conv3x3_groupnorm_silu_residual_fused","name":"VAE conv3x3 GroupNorm SiLU residual fused","group":"Image and video","op":"VAE conv3x3 + GroupNorm + SiLU residual block","direction":"forward","model":"Efficient-Large-Model/Sana-Sprint-1.6B-1024px","workloads":20},{"board":"L1","id":"003_lm_head_projection_with_logit_slicing","name":"LM head projection with logit slicing","group":"Projections and GEMM","op":"LM head GEMM on the last N positions (logit slicing)","direction":"forward","model":"LGAI-EXAONE/EXAONE-4.0-1.2B","workloads":16},{"board":"L1","id":"004_attention_output_projection_with_reshape_backward","name":"Attention output projection with reshape backward","group":"Backward pass","op":"attention o_proj + reshape backward","direction":"backward","model":"LiquidAI/LFM2-1.2B","workloads":16},{"board":"L1","id":"005_conv_gated_projection_with_causal_conv","name":"Conv gated projection with causal conv","group":"State space and conv","op":"LFM2 short-conv block: in-proj, gate, causal depthwise conv1d, out-proj","direction":"forward","model":"LiquidAI/LFM2-1.2B","workloads":16},{"board":"L1","id":"006_hyena_depthwise_conv1d_split_gate","name":"Hyena depthwise Conv1D split gate","group":"State space and conv","op":"Hyena depthwise conv1d + split + gate","direction":"forward","model":"LongSafari/hyenadna-small-32k-seqlen-hf","workloads":16},{"board":"L1","id":"007_hyena_fft_size_padding_rfft","name":"Hyena FFT size padding RFFT","group":"State space and conv","op":"Hyena FFT padding + rfft","direction":"forward","model":"LongSafari/hyenadna-small-32k-seqlen-hf","workloads":16},{"board":"L1","id":"008_expert_output_weighted_index_add_accumulation","name":"Expert output weighted index add accumulation","group":"MLP and MoE","op":"MoE combine: weighted index_add of expert outputs","direction":"forward","model":"MiniMaxAI/MiniMax-M2","workloads":16},{"board":"L1","id":"009_expert_token_scatter_with_weighted_forward_backward","name":"Expert token scatter with weighted forward backward","group":"Backward pass","op":"MoE token scatter, weighted, forward + backward","direction":"backward","model":"MiniMaxAI/MiniMax-M2","workloads":16},{"board":"L1","id":"010_attention_value_projection_with_transpose","name":"Attention value projection with transpose","group":"Projections and GEMM","op":"V projection (5120 -> 1024) + reshape / transpose","direction":"forward","model":"NousResearch/DeepHermes-3-Mistral-24B-Preview","workloads":16},{"board":"L1","id":"011_rotary_position_embedding","name":"Rotary position embedding","group":"Attention","op":"RoPE cos/sin from positions with llama3 scaling","direction":"forward","model":"NousResearch/Hermes-4-70B","workloads":16},{"board":"L1","id":"012_fused_cos_sin_embedding_generation","name":"Fused cos sin embedding generation","group":"Attention","op":"RoPE cos/sin from frequencies with attention scaling","direction":"forward","model":"NousResearch/Hermes-4-405B","workloads":16},{"board":"L1","id":"013_fused_residual_rms_norm_backward","name":"Fused residual RMS norm backward","group":"Backward pass","op":"fused residual + RMSNorm backward","direction":"backward","model":"Qwen/Qwen3-4B-Instruct-2507","workloads":16},{"board":"L1","id":"014_rotary_embedding_with_attention_scaling_backward","name":"Rotary embedding with attention scaling backward","group":"Backward pass","op":"RoPE with attention scaling backward","direction":"backward","model":"Qwen/Qwen3-4B-Instruct-2507","workloads":16},{"board":"L1","id":"015_grouped_query_attention_with_rope_and_qk_norm","name":"Grouped query attention with RoPE and QK norm","group":"Attention","op":"GQA attention block: QKV proj, QK-RMSNorm, RoPE, causal SDPA, o_proj","direction":"forward","model":"Qwen/Qwen3-8B","workloads":16},{"board":"L1","id":"016_rope_inverse_frequency_computation","name":"RoPE inverse frequency computation","group":"Attention","op":"RoPE inverse-frequency table","direction":"forward","model":"Qwen/Qwen3-8B","workloads":16},{"board":"L1","id":"017_moe_expert_swiglu_with_down_projection_backward","name":"MoE expert SwiGLU with down projection backward","group":"Backward pass","op":"MoE expert SwiGLU + down projection backward","direction":"backward","model":"Qwen/Qwen3-30B-A3B-Instruct-2507","workloads":16},{"board":"L1","id":"018_fused_rope_with_qk_norm_and_kv_cache_update","name":"Fused RoPE with QK norm and KV cache update","group":"Attention","op":"fused QK-RMSNorm + RoPE + KV-cache write","direction":"forward","model":"Qwen/Qwen3-Coder-480B-A35B-Instruct","workloads":13},{"board":"L1","id":"019_vision_3d_rotary_embedding_with_spatial_merge_indexing_backward","name":"Vision 3D rotary embedding with spatial merge indexing backward","group":"Backward pass","op":"vision 3D RoPE with spatial-merge indexing backward","direction":"backward","model":"Qwen/Qwen3-Omni-30B-A3B-Instruct","workloads":16},{"board":"L1","id":"020_vision_patch_merger_spatial_shuffle_mlp","name":"Vision patch merger spatial shuffle MLP","group":"Image and video","op":"vision patch merger: 2x2 shuffle + LayerNorm + GELU MLP","direction":"forward","model":"Qwen/Qwen3-Omni-30B-A3B-Instruct","workloads":15},{"board":"L1","id":"021_vision_cu_seqlens_variable_length_attention","name":"Vision cu seqlens variable length attention","group":"Image and video","op":"vision varlen attention (cu_seqlens) with QKV proj + RoPE","direction":"forward","model":"Qwen/Qwen3-VL-8B-Instruct","workloads":16},{"board":"L1","id":"022_vision_mlp_gelu_backward","name":"Vision MLP GELU backward","group":"Backward pass","op":"vision MLP GELU backward","direction":"backward","model":"Qwen/Qwen3-VL-8B-Instruct","workloads":16},{"board":"L1","id":"023_multimodal_rope_position_computation_with_grid_based_indexing","name":"Multimodal RoPE position computation with grid based indexing","group":"Image and video","op":"multimodal RoPE positions + learned position-embedding interpolation","direction":"forward","model":"Qwen/Qwen3-VL-235B-A22B-Instruct","workloads":16},{"board":"L1","id":"024_vision_rotary_position_embedding_generation_backward","name":"Vision rotary position embedding generation backward","group":"Backward pass","op":"vision RoPE generation backward","direction":"backward","model":"Qwen/Qwen-Image-Edit-2509","workloads":16},{"board":"L1","id":"025_video_latent_gelu_activation","name":"Video latent GELU activation","group":"Image and video","op":"GELU-tanh elementwise (video latents)","direction":"forward","model":"Wan-AI/Wan2.2-T2V-A14B","workloads":16},{"board":"L1","id":"026_video_patch_embedding_projection","name":"Video patch embedding projection","group":"Image and video","op":"Conv3D video patch embed + position embed + LayerNorm","direction":"forward","model":"Wan-AI/Wan2.2-T2V-A14B","workloads":4},{"board":"L1","id":"027_video_spatial_attention_with_rope_3d","name":"Video spatial attention with RoPE 3D","group":"Image and video","op":"video spatial attention with 3D RoPE","direction":"forward","model":"Wan-AI/Wan2.2-T2V-A14B","workloads":16},{"board":"L1","id":"028_hybrid_attention_mask_preparation","name":"Hybrid attention mask preparation","group":"Attention","op":"causal + sliding-window mask construction","direction":"forward","model":"XiaomiMiMo/MiMo-V2-Flash","workloads":12},{"board":"L1","id":"029_mamba_conv1d_with_gating","name":"Mamba Conv1D with gating","group":"State space and conv","op":"Mamba in-proj + causal depthwise conv1d + SiLU gate","direction":"forward","model":"ai21labs/AI21-Jamba-Large-1.7","workloads":16},{"board":"L1","id":"030_attention_output_projection_with_residual","name":"Attention output projection with residual","group":"Projections and GEMM","op":"o_proj GEMM (2560x2560) + residual add","direction":"forward","model":"ai21labs/AI21-Jamba-Reasoning-3B","workloads":16},{"board":"L1","id":"031_repeat_kv_attention_matmul","name":"Repeat KV attention matmul","group":"Attention","op":"GQA repeat_kv + Q@K^T scores","direction":"forward","model":"allenai/Olmo-3-7B-Think","workloads":16},{"board":"L1","id":"032_attention_weights_matmul_with_value_projection","name":"Attention weights matmul with value projection","group":"Attention","op":"attention weights @ V + transpose / reshape","direction":"forward","model":"allenai/Olmo-3-32B-Think","workloads":16},{"board":"L1","id":"033_post_norm_residual","name":"Post norm residual","group":"Normalization","op":"post-norm residual: residual + RMSNorm(x)","direction":"forward","model":"allenai/Olmo-3-1025-7B","workloads":16},{"board":"L1","id":"034_flux_multi_axis_rope_frequency_computation","name":"Flux multi axis RoPE frequency computation","group":"Image and video","op":"FLUX multi-axis RoPE frequencies","direction":"forward","model":"black-forest-labs/FLUX.1-Fill-dev","workloads":16},{"board":"L1","id":"035_flux_ada_layer_norm_zero_modulation_extraction","name":"Flux ada layer norm zero modulation extraction","group":"Image and video","op":"FLUX AdaLN-Zero modulation linear + chunk","direction":"forward","model":"black-forest-labs/FLUX.1-Kontext-dev","workloads":16},{"board":"L1","id":"036_flux_output_norm_projection_chain","name":"Flux output norm projection chain","group":"Image and video","op":"FLUX final AdaLN + output projection","direction":"forward","model":"black-forest-labs/FLUX.1-Kontext-dev","workloads":16},{"board":"L1","id":"037_flux_feedforward_gelu_approximate","name":"Flux feedforward GELU approximate","group":"Image and video","op":"FLUX feed-forward with GELU-tanh","direction":"forward","model":"black-forest-labs/FLUX.1-dev","workloads":16},{"board":"L1","id":"038_flux_multi_head_rmsnorm_qk","name":"Flux multi head RMSNorm QK","group":"Image and video","op":"FLUX.2 per-head QK RMSNorm","direction":"forward","model":"black-forest-labs/FLUX.2-dev","workloads":16},{"board":"L1","id":"039_adaptive_instance_normalization_2d_backward","name":"Adaptive instance normalization 2D backward","group":"Backward pass","op":"adaptive instance norm 2D backward","direction":"backward","model":"chuxiaojie/NAFNet","workloads":16},{"board":"L1","id":"040_conv2d_residual_block","name":"Conv2D residual block","group":"Image and video","op":"NAFNet conv residual block","direction":"forward","model":"chuxiaojie/NAFNet","workloads":16},{"board":"L1","id":"041_strided_conv2d_downsampling","name":"Strided Conv2D downsampling","group":"Image and video","op":"NAFNet strided conv2d downsampling","direction":"forward","model":"chuxiaojie/NAFNet","workloads":16},{"board":"L1","id":"042_moe_expert_load_balancing_and_token_capacity_backward","name":"MoE expert load balancing and token capacity backward","group":"Backward pass","op":"MoE load-balancing loss / capacity backward","direction":"backward","model":"deepseek-ai/DeepSeek-R1-0528","workloads":16},{"board":"L1","id":"043_mla_fused_qkv_rope_split","name":"MLA fused QKV RoPE split","group":"Projections and GEMM","op":"MLA latent projections: q_a -> RMSNorm -> q_b, kv_a + split","direction":"forward","model":"deepseek-ai/DeepSeek-V3","workloads":16},{"board":"L1","id":"044_moe_expert_computation","name":"MoE expert computation","group":"MLP and MoE","op":"MoE experts: SwiGLU per expert from given ids / weights","direction":"forward","model":"deepseek-ai/DeepSeek-V3","workloads":16},{"board":"L1","id":"045_fused_linear_gelu_grn_linear","name":"Fused linear GELU GRN linear","group":"Image and video","op":"ConvNeXtV2 MLP + GRN","direction":"forward","model":"facebook/convnextv2-base-22k-224","workloads":14},{"board":"L1","id":"046_attention_softmax_with_softcapping_and_dropout","name":"Attention softmax with softcapping and dropout","group":"Attention","op":"attention softmax with logit soft-cap + dropout","direction":"forward","model":"google/gemma-3-1b-it","workloads":16},{"board":"L1","id":"047_attention_with_qk_norm_and_rope","name":"Attention with QK norm and RoPE","group":"Attention","op":"Gemma3 attention block: QKV/O proj, QK-RMSNorm, RoPE, soft-cap, causal","direction":"forward","model":"google/gemma-3-12b-it","workloads":16},{"board":"L1","id":"048_fused_gate_up_projection_with_swiglu","name":"Fused gate up projection with SwiGLU","group":"MLP and MoE","op":"fused gate+up GEMM with GELU-tanh gating (no down proj)","direction":"forward","model":"google/gemma-3-12b-it","workloads":16},{"board":"L1","id":"049_attention_qk_matmul_with_gqa_repeat_and_scaling","name":"Attention QK matmul with GQA repeat and scaling","group":"Attention","op":"GQA repeat + Q@K^T + scale","direction":"forward","model":"google/gemma-3-270m","workloads":16},{"board":"L1","id":"050_attention_qkv_projection_with_bias_and_reshape","name":"Attention QKV projection with bias and reshape","group":"Projections and GEMM","op":"QKV projection with bias + reshape","direction":"forward","model":"google/gemma-3-270m","workloads":16},{"board":"L1","id":"051_attention_qkv_with_qk_norm_single_kernel_backward","name":"Attention QKV with QK norm single kernel backward","group":"Backward pass","op":"QKV + QK-norm backward","direction":"backward","model":"google/gemma-3-270m","workloads":16},{"board":"L1","id":"052_altup_hidden_state_collapse_with_magnitude_normalization","name":"AltUp hidden state collapse with magnitude normalization","group":"Other","op":"Gemma3n AltUp stream collapse","direction":"forward","model":"google/gemma-3n-E2B","workloads":16},{"board":"L1","id":"053_gaussian_topk_sparse_activation","name":"Gaussian topk sparse activation","group":"Other","op":"Gemma3n Gaussian top-k sparse activation","direction":"forward","model":"google/gemma-3n-E2B","workloads":12},{"board":"L1","id":"054_audio_attention_qkv_projection_with_normalization","name":"Audio attention QKV projection with normalization","group":"Audio","op":"audio attention QKV proj + per-head RMSNorm","direction":"forward","model":"google/gemma-3n-E4B","workloads":16},{"board":"L1","id":"055_attention_qkv_projection_with_gqa_reshape","name":"Attention QKV projection with GQA reshape","group":"Projections and GEMM","op":"QKV projection + GQA reshape","direction":"forward","model":"ibm-granite/granite-4.0-1b","workloads":16},{"board":"L1","id":"056_mamba_ssm_dt_projection_softplus_clamp_backward","name":"Mamba SSM dt projection softplus clamp backward","group":"Backward pass","op":"Mamba dt projection + softplus + clamp backward","direction":"backward","model":"ibm-granite/granite-4.0-micro","workloads":16},{"board":"L1","id":"057_mtp_shifted_embedding_with_dual_rms_norm_fusion","name":"MTP shifted embedding with dual RMS norm fusion","group":"Full decoder layers","op":"MTP input: shifted embedding + dual RMSNorm + concat + proj","direction":"forward","model":"inclusionAI/Ling-flash-2.0","workloads":16},{"board":"L1","id":"058_moe_expert_token_radix_sort_with_prefix_sum","name":"MoE expert token radix sort with prefix sum","group":"MLP and MoE","op":"MoE token sort by expert (counting sort + prefix sum)","direction":"forward","model":"inclusionAI/Ling-mini-2.0","workloads":16},{"board":"L1","id":"059_moe_group_score_aggregation_and_masking","name":"MoE group score aggregation and masking","group":"MLP and MoE","op":"MoE group-limited routing step: top-2 group sums, top-4 groups, mask","direction":"forward","model":"inclusionAI/Ling-mini-2.0","workloads":16},{"board":"L1","id":"060_fused_attention_qk_matmul_scale_mask_softmax_backward","name":"Fused attention QK matmul scale mask softmax backward","group":"Backward pass","op":"attention QK scale mask softmax backward","direction":"backward","model":"meta-llama/Llama-3.1-405B","workloads":16},{"board":"L1","id":"061_tanh_gated_residual_add_backward","name":"Tanh gated residual add backward","group":"Backward pass","op":"tanh-gated residual add backward","direction":"backward","model":"meta-llama/Llama-3.2-11B-Vision","workloads":16},{"board":"L1","id":"062_kv_cache_update_with_rope_backward","name":"KV cache update with RoPE backward","group":"Backward pass","op":"KV-cache update with RoPE backward","direction":"backward","model":"mistralai/Devstral-Small-2507","workloads":16},{"board":"L1","id":"063_attention_output_reshape_and_projection","name":"Attention output reshape and projection","group":"Projections and GEMM","op":"attention output reshape + o_proj (heads*v_head_dim -> hidden)","direction":"forward","model":"moonshotai/Kimi-K2-Instruct-0905","workloads":16},{"board":"L1","id":"064_latent_kv_expansion_with_split","name":"Latent KV expansion with split","group":"Projections and GEMM","op":"MLA latent KV expansion: RMSNorm + kv_b_proj + split K_nope / V","direction":"forward","model":"moonshotai/Kimi-K2-Instruct-0905","workloads":16},{"board":"L1","id":"065_fused_statistics_projection_and_split","name":"Fused statistics projection and split","group":"Audio","op":"Conv1d stats projection + split (TTS)","direction":"forward","model":"myshell-ai/OpenVoice","workloads":16},{"board":"L1","id":"066_masked_softmax_with_attention_dropout_backward","name":"Masked softmax with attention dropout backward","group":"Backward pass","op":"masked softmax + dropout backward","direction":"backward","model":"myshell-ai/OpenVoice","workloads":16},{"board":"L1","id":"067_flash_attention_gqa_ultralong","name":"Flash attention GQA ultralong","group":"Attention","op":"GQA attention block: QKV proj, RoPE, causal SDPA, o_proj (no QK-norm)","direction":"forward","model":"nvidia/Llama-3.1-Nemotron-8B-UltraLong-1M-Instruct","workloads":18},{"board":"L1","id":"068_fused_rope_application_backward","name":"Fused RoPE application backward","group":"Backward pass","op":"RoPE application backward","direction":"backward","model":"nvidia/Llama-3.1-Nemotron-8B-UltraLong-1M-Instruct","workloads":16},{"board":"L1","id":"069_rms_norm","name":"RMS norm","group":"Normalization","op":"residual add + RMSNorm (returns the normed output only)","direction":"forward","model":"nvidia/Llama-3.3-Nemotron-70B-Select","workloads":16},{"board":"L1","id":"070_mamba2_fused_intra_chunk_diagonal_computation","name":"Mamba2 fused intra chunk diagonal computation","group":"State space and conv","op":"Mamba2 SSD intra-chunk diagonal block","direction":"forward","model":"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8","workloads":14},{"board":"L1","id":"071_kv_cache_update_with_rope","name":"KV cache update with RoPE","group":"Attention","op":"KV-cache update with RoPE on K","direction":"forward","model":"nvidia/Phi-4-reasoning-plus-NVFP4","workloads":16},{"board":"L1","id":"072_cross_attention_qkv_projection_with_gqa_repeat_backward","name":"Cross attention QKV projection with GQA repeat backward","group":"Backward pass","op":"cross-attention QKV + GQA repeat backward","direction":"backward","model":"nvidia/parakeet-tdt-0.6b-v3","workloads":16},{"board":"L1","id":"073_fused_encoder_final_norm_to_decoder_cross_attention_kv_projection","name":"Fused encoder final norm to decoder cross attention KV projection","group":"Audio","op":"encoder final RMSNorm + cross-attention K/V projection","direction":"forward","model":"nvidia/parakeet-tdt-0.6b-v3","workloads":16},{"board":"L1","id":"074_fused_gated_mlp_silu","name":"Fused gated MLP SiLU","group":"MLP and MoE","op":"gated MLP: fused gate_up GEMM + SiLU*up + down","direction":"forward","model":"nvidia/parakeet-tdt-0.6b-v3","workloads":16},{"board":"L1","id":"075_grouped_query_self_attention_with_rope","name":"Grouped query self attention with RoPE","group":"Audio","op":"GQA self-attention block with RoPE (encoder / decoder)","direction":"forward","model":"nvidia/parakeet-tdt-0.6b-v3","workloads":16},{"board":"L1","id":"076_batched_expert_forward","name":"Batched expert forward","group":"MLP and MoE","op":"gpt-oss experts: gate_up + clamped SwiGLU (alpha, limit) + down, dense over all experts","direction":"forward","model":"openai/gpt-oss-120b","workloads":14},{"board":"L1","id":"077_whisper_decoder_output_projection","name":"Whisper decoder output projection","group":"Projections and GEMM","op":"Whisper LM head GEMM","direction":"forward","model":"openai/whisper-large-v3","workloads":16},{"board":"L1","id":"078_group_norm_fusion","name":"Group norm fusion","group":"Image and video","op":"GroupNorm (SD UNet)","direction":"forward","model":"runwayml/stable-diffusion-v1-5","workloads":16},{"board":"L1","id":"079_ImageNet_83.6_ssm_output_projection_gate_multiply_backward","name":"ImageNet 83.6 SSM output projection gate multiply backward","group":"Backward pass","op":"SSM output projection gate multiply backward","direction":"backward","model":"saurabhati/VMamba","workloads":16},{"board":"L1","id":"080_adaptive_layernorm_continuous_with_modulation","name":"Adaptive LayerNorm continuous with modulation","group":"Image and video","op":"SD3.5 AdaLayerNormContinuous","direction":"forward","model":"stabilityai/stable-diffusion-3.5-large","workloads":16},{"board":"L1","id":"081_joint_attention_context_projection","name":"Joint attention context projection","group":"Image and video","op":"SD3.5 joint-attention context projection","direction":"forward","model":"stabilityai/stable-diffusion-3.5-large","workloads":16},{"board":"L1","id":"082_qk_norm_scaled_dot_product_attention","name":"QK norm scaled dot product attention","group":"Image and video","op":"SD3.5 QK-LayerNorm SDPA","direction":"forward","model":"stabilityai/stable-diffusion-3.5-medium","workloads":16},{"board":"L1","id":"083_attention_score_value_matmul","name":"Attention score value matmul","group":"Image and video","op":"SDXL attention weights @ V","direction":"forward","model":"stabilityai/stable-diffusion-xl-base-1.0","workloads":16},{"board":"L1","id":"084_silu_activation_backward","name":"SiLU activation backward","group":"Backward pass","op":"SiLU backward","direction":"backward","model":"stabilityai/stable-diffusion-xl-base-1.0","workloads":16},{"board":"L1","id":"085_geglu_activation","name":"GEGLU activation","group":"Image and video","op":"GEGLU activation","direction":"forward","model":"stabilityai/stable-diffusion-xl-refiner-1.0","workloads":16},{"board":"L1","id":"086_sam_hq_mask_decoder_iou_hypernetwork_fusion","name":"SAM hq mask decoder IoU hypernetwork fusion","group":"Image and video","op":"SAM-HQ IoU head + hypernetwork MLPs","direction":"forward","model":"syscv-community/sam-hq-vit-base","workloads":16},{"board":"L1","id":"087_embedding_with_initial_layernorm_backward","name":"Embedding with initial LayerNorm backward","group":"Backward pass","op":"embedding + LayerNorm backward","direction":"backward","model":"taozi555/jamba-mini-1.7-rp","workloads":16},{"board":"L1","id":"088_rotary_position_embedding_application","name":"Rotary position embedding application","group":"Image and video","op":"RoPE application to Q/K (HunyuanImage)","direction":"forward","model":"tencent/HunyuanImage-2.1","workloads":16},{"board":"L1","id":"089_vae_attention_block_with_groupnorm","name":"VAE attention block with GroupNorm","group":"Image and video","op":"VAE attention block with GroupNorm","direction":"forward","model":"tencent/HunyuanImage-2.1","workloads":16},{"board":"L1","id":"090_batched_2d_rope_position_encoding_backward","name":"Batched 2D RoPE position encoding backward","group":"Backward pass","op":"batched 2D RoPE backward","direction":"backward","model":"tencent/HunyuanImage-3.0","workloads":16},{"board":"L1","id":"091_fused_gqa_qkv_rope_qk_norm_backward","name":"Fused GQA QKV RoPE QK norm backward","group":"Backward pass","op":"fused GQA QKV + RoPE + QK-norm backward","direction":"backward","model":"tencent/HunyuanImage-3.0","workloads":16},{"board":"L1","id":"092_gqa_attention_with_qk_norm","name":"GQA attention with QK norm","group":"Attention","op":"GQA attention block: QKV proj (bias), QK-RMSNorm, RoPE, causal, o_proj","direction":"forward","model":"zai-org/GLM-4.5-Air","workloads":16},{"board":"L1","id":"093_grouped_topk_moe_routing_backward","name":"Grouped topk MoE routing backward","group":"Backward pass","op":"grouped top-k MoE routing backward","direction":"backward","model":"zai-org/GLM-4.7","workloads":16},{"board":"L1","id":"094_time_decay_exponential_stabilization","name":"Time decay exponential stabilization","group":"Other","op":"RWKV time-decay exponent with stabilisation","direction":"forward","model":"zhiyuan8/RWKV-v7-1.5B-G1-GGUF","workloads":16},{"board":"L2","id":"001_fused_vision_multihead_attention_with_norms_backward","name":"Fused vision multihead attention with norms backward","group":"Backward pass","op":"vision MHA with norms backward","direction":"backward","model":"1024m/Molmo-7B-D-Base","workloads":16},{"board":"L2","id":"002_decoder_layer_full_block","name":"Decoder layer full block","group":"Full decoder layers","op":"Llama decoder layer: RMSNorm, GQA + RoPE, residual, RMSNorm, SwiGLU MLP, residual","direction":"forward","model":"NousResearch/DeepHermes-3-Llama-3-8B-Preview","workloads":18},{"board":"L2","id":"003_grouped_query_attention_with_rope_backward","name":"Grouped query attention with RoPE backward","group":"Backward pass","op":"GQA attention with RoPE backward","direction":"backward","model":"NousResearch/DeepHermes-3-Mistral-24B-Preview","workloads":16},{"board":"L2","id":"004_fused_residual_rms_mlp","name":"Fused residual RMS MLP","group":"MLP and MoE","op":"residual + RMSNorm + SwiGLU MLP","direction":"forward","model":"NousResearch/Hermes-4-405B","workloads":16},{"board":"L2","id":"005_swiglu_mlp_backward","name":"SwiGLU MLP backward","group":"Backward pass","op":"SwiGLU MLP backward","direction":"backward","model":"NousResearch/Hermes-4-405B","workloads":18},{"board":"L2","id":"006_multimodal_rope_position_calculation","name":"Multimodal RoPE position calculation","group":"Attention","op":"multimodal 3D RoPE position-id calculation","direction":"forward","model":"Qwen/Qwen2.5-VL-7B-Instruct","workloads":16},{"board":"L2","id":"007_multimodal_rotary_embedding_attention","name":"Multimodal rotary embedding attention","group":"Attention","op":"GQA attention with multimodal 3D RoPE","direction":"forward","model":"Qwen/Qwen2.5-VL-7B-Instruct","workloads":16},{"board":"L2","id":"008_moe_sparse_routing_and_dispatch","name":"MoE sparse routing and dispatch","group":"MLP and MoE","op":"MoE block: softmax router + top-8 + SwiGLU experts + weighted sum","direction":"forward","model":"Qwen/Qwen3-30B-A3B-Instruct-2507","workloads":16},{"board":"L2","id":"009_decoder_layer_with_residual_connections","name":"Decoder layer with residual connections","group":"Full decoder layers","op":"MoE decoder layer: RMSNorm, GQA + QK-norm, MoE, residuals","direction":"forward","model":"Qwen/Qwen3-Coder-30B-A3B-Instruct","workloads":16},{"board":"L2","id":"010_moe_expert_computation_with_weighted_accumulation","name":"MoE expert computation with weighted accumulation","group":"MLP and MoE","op":"MoE experts SwiGLU + weighted accumulation (given top-k)","direction":"forward","model":"Qwen/Qwen3-Coder-30B-A3B-Instruct","workloads":16},{"board":"L2","id":"011_moe_sparse_routing_and_dispatch_backward","name":"MoE sparse routing and dispatch backward","group":"Backward pass","op":"MoE routing + dispatch backward","direction":"backward","model":"Qwen/Qwen3-Coder-30B-A3B-Instruct","workloads":16},{"board":"L2","id":"012_moe_expert_batched_execution_with_capacity_factor","name":"MoE expert batched execution with capacity factor","group":"MLP and MoE","op":"MoE experts with capacity-factor batching","direction":"forward","model":"Qwen/Qwen3-Coder-480B-A35B-Instruct","workloads":16},{"board":"L2","id":"013_expert_weighted_aggregation_with_shared_expert","name":"Expert weighted aggregation with shared expert","group":"MLP and MoE","op":"MoE experts + weighted sum + sigmoid-gated shared expert","direction":"forward","model":"Qwen/Qwen3-Next-80B-A3B-Instruct","workloads":16},{"board":"L2","id":"014_audio_encoder_varlen_attention_with_chunking_backward","name":"Audio encoder varlen attention with chunking backward","group":"Backward pass","op":"audio varlen attention with chunking backward","direction":"backward","model":"Qwen/Qwen3-Omni-30B-A3B-Instruct","workloads":16},{"board":"L2","id":"015_audio_sinusoidal_position_embedding_with_conv_projection","name":"Audio sinusoidal position embedding with conv projection","group":"Audio","op":"audio sinusoidal position embed + Conv2d downsampling + proj","direction":"forward","model":"Qwen/Qwen3-Omni-30B-A3B-Instruct","workloads":16},{"board":"L2","id":"016_moe_expert_mlp_with_load_balancing","name":"MoE expert MLP with load balancing","group":"MLP and MoE","op":"MoE block: softmax router + top-8 + SwiGLU experts (returns router logits)","direction":"forward","model":"Qwen/Qwen3-Omni-30B-A3B-Instruct","workloads":16},{"board":"L2","id":"017_fused_vision_cu_seqlens_attention_with_2d_rope_backward","name":"Fused vision cu seqlens attention with 2D RoPE backward","group":"Backward pass","op":"vision varlen attention with 2D RoPE backward","direction":"backward","model":"Qwen/Qwen3-VL-4B-Instruct","workloads":16},{"board":"L2","id":"018_cu_seqlens_variable_length_vision_attention","name":"Cu seqlens variable length vision attention","group":"Image and video","op":"vision varlen attention (cu_seqlens) with RoPE","direction":"forward","model":"Qwen/Qwen-Image-Edit-2509","workloads":16},{"board":"L2","id":"019_decoder_layer_fused_attention_mlp","name":"Decoder layer fused attention MLP","group":"Full decoder layers","op":"Qwen2-VL decoder layer (M-RoPE GQA + SwiGLU)","direction":"forward","model":"Qwen/Qwen-Image-Edit-2509","workloads":16},{"board":"L2","id":"020_decoder_layer_pre_post_norm_residual","name":"Decoder layer pre post norm residual","group":"Full decoder layers","op":"decoder layer with pre / post norm residuals","direction":"forward","model":"RekaAI/reka-flash-3","workloads":16},{"board":"L2","id":"021_cross_attention_text_video_conditioning_backward","name":"Cross attention text video conditioning backward","group":"Backward pass","op":"text-video cross-attention backward","direction":"backward","model":"Wan-AI/Wan2.2-T2V-A14B","workloads":16},{"board":"L2","id":"022_video_latent_denoising_unet_block","name":"Video latent denoising UNet block","group":"Image and video","op":"video latent denoising block","direction":"forward","model":"Wan-AI/Wan2.2-T2V-A14B","workloads":16},{"board":"L2","id":"023_video_latent_vae_encoder_downsampling","name":"Video latent VAE encoder downsampling","group":"Image and video","op":"video VAE encoder downsampling","direction":"forward","model":"Wan-AI/Wan2.2-T2V-A14B","workloads":16},{"board":"L2","id":"024_moe_expert_parallel_execution","name":"MoE expert parallel execution","group":"MLP and MoE","op":"MoE experts (256, given top-k) SwiGLU + scatter-add","direction":"forward","model":"XiaomiMiMo/MiMo-V2-Flash","workloads":16},{"board":"L2","id":"025_moe_expert_parallel_execution_backward","name":"MoE expert parallel execution backward","group":"Backward pass","op":"MoE experts backward","direction":"backward","model":"XiaomiMiMo/MiMo-V2-Flash","workloads":16},{"board":"L2","id":"026_moe_expert_parallel_execution_with_weighted_aggregation","name":"MoE expert parallel execution with weighted aggregation","group":"MLP and MoE","op":"MoE experts (16, top-2) with weighted aggregation","direction":"forward","model":"ai21labs/AI21-Jamba-Large-1.7","workloads":16},{"board":"L2","id":"027_grouped_query_attention_with_yarn_rope_and_qk_norm","name":"Grouped query attention with YaRN RoPE and QK norm","group":"Attention","op":"GQA attention with YaRN RoPE, QK-norm, sliding window","direction":"forward","model":"allenai/Olmo-3-32B-Think","workloads":16},{"board":"L2","id":"028_gqa_rotary_attention_core_backward","name":"GQA rotary attention core backward","group":"Backward pass","op":"GQA rotary attention core backward","direction":"backward","model":"baidu/ERNIE-4.5-300B-A47B-PT","workloads":16},{"board":"L2","id":"029_moe_sparse_routing_and_dispatch","name":"MoE sparse routing and dispatch","group":"MLP and MoE","op":"MoE block: biased router, top-k, experts + shared expert","direction":"forward","model":"baidu/ERNIE-4.5-300B-A47B-PT","workloads":9},{"board":"L2","id":"030_flux_concatenated_sequence_processing_with_split","name":"Flux concatenated sequence processing with split","group":"Image and video","op":"FLUX concat text+image, linear, split","direction":"forward","model":"black-forest-labs/FLUX.1-Kontext-dev","workloads":16},{"board":"L2","id":"031_flux_timestep_guidance_projection_embedding","name":"Flux timestep guidance projection embedding","group":"Image and video","op":"FLUX timestep + guidance embedding","direction":"forward","model":"black-forest-labs/FLUX.1-dev","workloads":16},{"board":"L2","id":"032_dual_stream_attention_with_conditional_cross_attention","name":"Dual stream attention with conditional cross attention","group":"Image and video","op":"FIBO dual-stream attention block","direction":"forward","model":"briaai/FIBO","workloads":16},{"board":"L2","id":"033_multi_scale_feature_pyramid","name":"Multi scale feature pyramid","group":"Image and video","op":"multi-scale feature pyramid (U-Net)","direction":"forward","model":"chuxiaojie/NAFNet","workloads":16},{"board":"L2","id":"034_vision_language_cross_attention_fusion","name":"Vision language cross attention fusion","group":"Attention","op":"language-to-vision cross-attention with 1D / 3D RoPE","direction":"forward","model":"cybermotaz/Qwen3-VL-32B-Instruct-NVFP4","workloads":16},{"board":"L2","id":"035_convnextv2_block_with_grn","name":"ConvNeXtV2 block with GRN","group":"Image and video","op":"ConvNeXtV2 block with GRN","direction":"forward","model":"facebook/convnextv2-base-22k-224","workloads":16},{"board":"L2","id":"036_convnextv2_layer_with_nhwc_persistence_backward","name":"ConvNeXtV2 layer with NHWC persistence backward","group":"Backward pass","op":"ConvNeXtV2 layer backward","direction":"backward","model":"facebook/convnextv2-base-22k-224","workloads":14},{"board":"L2","id":"037_qk_norm_rope_attention_core_backward","name":"QK norm RoPE attention core backward","group":"Backward pass","op":"QK-norm RoPE attention core backward","direction":"backward","model":"google/gemma-3-4b-it","workloads":16},{"board":"L2","id":"038_audio_relative_position_attention","name":"Audio relative position attention","group":"Audio","op":"Gemma3n audio relative-position attention","direction":"forward","model":"google/gemma-3n-E2B","workloads":14},{"board":"L2","id":"039_kv_shared_attention_with_rope","name":"KV shared attention with RoPE","group":"Attention","op":"Gemma3n KV-shared attention with RoPE","direction":"forward","model":"google/gemma-3n-E2B","workloads":16},{"board":"L2","id":"040_altup_predict_correction_cycle_backward","name":"AltUp predict correction cycle backward","group":"Backward pass","op":"AltUp predict / correct backward","direction":"backward","model":"google/gemma-3n-E4B","workloads":16},{"board":"L2","id":"041_kv_shared_attention_with_dual_rope","name":"KV shared attention with dual RoPE","group":"Attention","op":"Gemma3n KV-shared attention with dual RoPE","direction":"forward","model":"google/gemma-3n-E4B","workloads":16},{"board":"L2","id":"042_ffn_gelu_projection_fused_backward","name":"FFN GELU projection fused backward","group":"Backward pass","op":"FFN GELU projection backward","direction":"backward","model":"hexgrad/Kokoro-82M","workloads":16},{"board":"L2","id":"043_mamba_chunk_scan_with_segsum","name":"Mamba chunk scan with segsum","group":"State space and conv","op":"Mamba2 chunk scan with segsum","direction":"forward","model":"ibm-granite/granite-4.0-1b","workloads":16},{"board":"L2","id":"044_mamba_discretization_and_segsum","name":"Mamba discretization and segsum","group":"State space and conv","op":"Mamba SSM discretisation + segsum","direction":"forward","model":"ibm-granite/granite-4.0-micro","workloads":16},{"board":"L2","id":"045_audio_encoder_to_language_model_multimodal_fusion","name":"Audio encoder to language model multimodal fusion","group":"Audio","op":"speech encoder + Q-Former + scatter into LM embeddings","direction":"forward","model":"ibm-granite/granite-speech-3.3-8b","workloads":16},{"board":"L2","id":"046_conformer_relative_position_attention","name":"Conformer relative position attention","group":"Audio","op":"Conformer relative-position attention","direction":"forward","model":"ibm-granite/granite-speech-3.3-8b","workloads":16},{"board":"L2","id":"047_moe_training_token_repeat_and_expert_computation","name":"MoE training token repeat and expert computation","group":"MLP and MoE","op":"MoE training forward: token repeat + masked expert SwiGLU","direction":"forward","model":"inclusionAI/Ling-1T","workloads":16},{"board":"L2","id":"048_moe_expert_inference_batched_dispatch","name":"MoE expert inference batched dispatch","group":"MLP and MoE","op":"MoE experts inference dispatch (sorted by expert)","direction":"forward","model":"inclusionAI/Ling-flash-2.0","workloads":16},{"board":"L2","id":"049_group_limited_topk_routing","name":"Group limited topk routing","group":"MLP and MoE","op":"group-limited top-k routing: router GEMM, sigmoid + bias, 8 groups / top-4 / top-8","direction":"forward","model":"inclusionAI/Ring-flash-2.0","workloads":16},{"board":"L2","id":"050_vae_decoder_mid_block_attention_resnet","name":"VAE decoder mid block attention ResNet","group":"Image and video","op":"VAE decoder mid-block (ResNet + attention)","direction":"forward","model":"kaiyuyue/FLUX.2-dev-vae","workloads":11},{"board":"L2","id":"051_seqlen-finetuned-reconstructed_hyena_complete_forward_block","name":"Seqlen-finetuned-reconstructed Hyena complete forward block","group":"State space and conv","op":"Hyena block forward","direction":"forward","model":"magicslabnu/Hyena-small-32k","workloads":16},{"board":"L2","id":"052_seqlen-finetuned-reconstructed_hyena_gating_and_output_projection","name":"Seqlen-finetuned-reconstructed Hyena gating and output projection","group":"State space and conv","op":"Hyena gating + FFT conv + output projection","direction":"forward","model":"magicslabnu/Hyena-small-32k","workloads":16},{"board":"L2","id":"053_text_decoder_layer_with_self_attention_and_mlp","name":"Text decoder layer with self attention and MLP","group":"Full decoder layers","op":"Mllama text decoder layer: RMSNorm, GQA + RoPE, SwiGLU MLP, residuals","direction":"forward","model":"meta-llama/Llama-3.2-11B-Vision","workloads":16},{"board":"L2","id":"054_vision_encoder_layer_with_gated_residuals","name":"Vision encoder layer with gated residuals","group":"Image and video","op":"vision encoder layer with tanh-gated residuals","direction":"forward","model":"meta-llama/Llama-3.2-11B-Vision","workloads":16},{"board":"L2","id":"055_audio_encoder_conv_positional_layer_stack","name":"Audio encoder conv positional layer stack","group":"Audio","op":"audio conv stem + first encoder layer","direction":"forward","model":"mistralai/Voxtral-Small-24B-2507","workloads":15},{"board":"L2","id":"056_language_model_decoder_prenorm_attention_ffn_residual_backward","name":"Language model decoder prenorm attention FFN residual backward","group":"Backward pass","op":"decoder layer backward","direction":"backward","model":"mistralai/Voxtral-Small-24B-2507","workloads":16},{"board":"L2","id":"057_residual_coupling_flow_block","name":"Residual coupling flow block","group":"Audio","op":"normalising-flow residual coupling block (TTS)","direction":"forward","model":"myshell-ai/OpenVoice","workloads":16},{"board":"L2","id":"058_mamba2_selective_scan","name":"Mamba2 selective scan","group":"State space and conv","op":"Mamba2 selective scan with conv + gated norm","direction":"forward","model":"nvidia/Nemotron-H-47B-Base-8K","workloads":16},{"board":"L2","id":"059_decoder_layer_full_block","name":"Decoder layer full block","group":"Full decoder layers","op":"Qwen3 decoder layer: RMSNorm, QKV + QK-norm, RoPE, GQA, o_proj, SwiGLU MLP","direction":"forward","model":"nvidia/Qwen3-14B-NVFP4","workloads":16},{"board":"L2","id":"060_chunk_gated_delta_rule_linear_attention","name":"Chunk gated delta rule linear attention","group":"Attention","op":"chunked gated delta-rule linear attention","direction":"forward","model":"nvidia/Qwen3-Next-80B-A3B-Thinking-NVFP4","workloads":16},{"board":"L2","id":"061_sparse_moe_routing_and_experts","name":"Sparse MoE routing and experts","group":"MLP and MoE","op":"MoE block: softmax router top-10 + experts + sigmoid-gated shared expert","direction":"forward","model":"nvidia/Qwen3-Next-80B-A3B-Thinking-NVFP4","workloads":16},{"board":"L2","id":"062_decoder_complete_layer","name":"Decoder complete layer","group":"Audio","op":"speech decoder layer: self-attn + cross-attn + MLP","direction":"forward","model":"nvidia/canary-qwen-2.5b","workloads":16},{"board":"L2","id":"063_encoder_layer_dual_residual_norm_chain","name":"Encoder layer dual residual norm chain","group":"Audio","op":"speech encoder layer (dual residual-norm chain)","direction":"forward","model":"nvidia/parakeet-tdt-0.6b-v3","workloads":16},{"board":"L2","id":"064_multi_head_qkv_projection_with_rope_backward","name":"Multi head QKV projection with RoPE backward","group":"Backward pass","op":"QKV projection with RoPE backward","direction":"backward","model":"nvidia/parakeet-tdt-0.6b-v3","workloads":16},{"board":"L2","id":"065_sparse_expert_dispatch_and_combine","name":"Sparse expert dispatch and combine","group":"MLP and MoE","op":"gpt-oss sparse experts: gate_up + clamped SwiGLU + down + index_add","direction":"forward","model":"openai/gpt-oss-120b","workloads":16},{"board":"L2","id":"066_resnet_block_with_time_embedding","name":"ResNet block with time embedding","group":"Image and video","op":"SD ResNet block with time embedding","direction":"forward","model":"runwayml/stable-diffusion-v1-5","workloads":16},{"board":"L2","id":"067_patch_embed_to_joint_attention_input","name":"Patch embed to joint attention input","group":"Image and video","op":"SD3.5 patch embed + time / text conditioning","direction":"forward","model":"stabilityai/stable-diffusion-3.5-large","workloads":16},{"board":"L2","id":"068_gelu_approximate_feedforward_backward","name":"GELU approximate feedforward backward","group":"Backward pass","op":"GELU feed-forward backward","direction":"backward","model":"stabilityai/stable-diffusion-3.5-medium","workloads":16},{"board":"L2","id":"069_joint_transformer_block_residual_path","name":"Joint transformer block residual path","group":"Image and video","op":"SD3.5 joint transformer block residual path","direction":"forward","model":"stabilityai/stable-diffusion-3.5-medium","workloads":16},{"board":"L2","id":"070_basic_transformer_block","name":"Basic transformer block","group":"Image and video","op":"SDXL basic transformer block","direction":"forward","model":"stabilityai/stable-diffusion-xl-refiner-1.0","workloads":16},{"board":"L2","id":"071_edit_consistency_loss_with_perceptual_weighting","name":"Edit consistency loss with perceptual weighting","group":"Image and video","op":"edit-consistency perceptual loss","direction":"forward","model":"stepfun-ai/Step1X-Edit-v1p2","workloads":16},{"board":"L2","id":"072_region_aware_self_attention_with_edit_bias_backward","name":"Region aware self attention with edit bias backward","group":"Backward pass","op":"region-aware self-attention backward","direction":"backward","model":"stepfun-ai/Step1X-Edit-v1p2","workloads":16},{"board":"L2","id":"073_feedforward_mlp_backward","name":"Feedforward MLP backward","group":"Backward pass","op":"feed-forward MLP backward","direction":"backward","model":"stepfun-ai/Step1X-Edit","workloads":16},{"board":"L2","id":"074_sam_hq_mask_decoder_iou_hypernetwork_fusion_backward","name":"SAM hq mask decoder IoU hypernetwork fusion backward","group":"Backward pass","op":"SAM-HQ hypernetwork backward","direction":"backward","model":"syscv-community/sam-hq-vit-base","workloads":16},{"board":"L2","id":"075_sam_hq_mask_decoder_two_way_transformer","name":"SAM hq mask decoder two way transformer","group":"Image and video","op":"SAM-HQ two-way transformer block","direction":"forward","model":"syscv-community/sam-hq-vit-base","workloads":16},{"board":"L2","id":"076_sam_hq_vision_attention_with_relative_position_backward","name":"SAM hq vision attention with relative position backward","group":"Backward pass","op":"SAM-HQ relative-position attention backward","direction":"backward","model":"syscv-community/sam-hq-vit-base","workloads":16},{"board":"L2","id":"077_sam_hq_vision_encoder_window_partition_attention","name":"SAM hq vision encoder window partition attention","group":"Image and video","op":"SAM-HQ windowed attention block","direction":"forward","model":"syscv-community/sam-hq-vit-base","workloads":16},{"board":"L2","id":"078_fused_final_layer_upsample_with_adaptive_norm","name":"Fused final layer upsample with adaptive norm","group":"Image and video","op":"HunyuanImage final layer upsample + adaptive norm","direction":"forward","model":"tencent/HunyuanImage-3.0","workloads":16},{"board":"L2","id":"079_git_vision_encoder_layer","name":"Git vision encoder layer","group":"Image and video","op":"ViT encoder layer (MaskGIT)","direction":"forward","model":"valhalla/maskgit-imagenet-soft-code-34k","workloads":16},{"board":"L2","id":"080_moe_complete_layer_with_shared_expert_backward","name":"MoE complete layer with shared expert backward","group":"Backward pass","op":"MoE layer with shared expert backward","direction":"backward","model":"zai-org/GLM-4.5-Air","workloads":16},{"board":"L2","id":"081_moe_sparse_expert_dispatch","name":"MoE sparse expert dispatch","group":"MLP and MoE","op":"MoE block: sigmoid router + top-8 + experts + shared expert","direction":"forward","model":"zai-org/GLM-4.5-Air","workloads":16},{"board":"L2","id":"082_moe_layer_complete_forward_with_residual","name":"MoE layer complete forward with residual","group":"MLP and MoE","op":"full MoE layer: sigmoid + bias router, group top-8, 160 experts, shared expert, residual","direction":"forward","model":"zai-org/GLM-4.7","workloads":16},{"board":"Quant","id":"001_fp8_attention_output_projection","name":"FP8 attention output projection","group":"Projections and GEMM","op":"FP8 o_proj GEMM + bias (60x128 -> 7680)","direction":"forward","model":"Qwen/Qwen3-VL-30B-A3B-Instruct-FP8","workloads":16},{"board":"Quant","id":"002_fp8_attention_qkv_projection","name":"FP8 attention QKV projection","group":"Projections and GEMM","op":"FP8 QKV projection (bias) + QK-RMSNorm","direction":"forward","model":"Qwen/Qwen3-VL-30B-A3B-Instruct-FP8","workloads":16},{"board":"Quant","id":"003_fp8_mlp_gate_up_projection","name":"FP8 MLP gate up projection","group":"MLP and MoE","op":"FP8 gate+up GEMMs + SiLU gating (pre-quantized inputs)","direction":"forward","model":"Qwen/Qwen3-VL-30B-A3B-Instruct-FP8","workloads":16},{"board":"Quant","id":"004_fp8_moe_expert_linear","name":"FP8 MoE expert linear","group":"MLP and MoE","op":"FP8 single-expert MLP: gate_up + SiLU + down, x routing weight","direction":"forward","model":"Qwen/Qwen3-VL-30B-A3B-Instruct-FP8","workloads":16},{"board":"Quant","id":"005_fp8_moe_router_projection","name":"FP8 MoE router projection","group":"MLP and MoE","op":"FP8 MoE router GEMM (experts padded to 128)","direction":"forward","model":"Qwen/Qwen3-VL-30B-A3B-Instruct-FP8","workloads":16},{"board":"Quant","id":"006_fp8_vision_attention_output_projection","name":"FP8 vision attention output projection","group":"Image and video","op":"FP8 vision o_proj GEMM (scales given)","direction":"forward","model":"Qwen/Qwen3-VL-30B-A3B-Instruct-FP8","workloads":16},{"board":"Quant","id":"007_fp8_vision_attention_qkv_projection","name":"FP8 vision attention QKV projection","group":"Image and video","op":"FP8 vision QKV projection + bias","direction":"forward","model":"Qwen/Qwen3-VL-30B-A3B-Instruct-FP8","workloads":16},{"board":"Quant","id":"008_nvfp4_multimodal_embedding_projection","name":"NVFP4 multimodal embedding projection","group":"Image and video","op":"NVFP4 vision projection + merge into text embeddings","direction":"forward","model":"cybermotaz/Qwen3-VL-32B-Instruct-NVFP4","workloads":16},{"board":"Quant","id":"009_nvfp4_vision_temporal_patch_merge_with_projection","name":"NVFP4 vision temporal patch merge with projection","group":"Image and video","op":"NVFP4 temporal patch merge + projection","direction":"forward","model":"cybermotaz/Qwen3-VL-32B-Instruct-NVFP4","workloads":16},{"board":"Quant","id":"010_fp8_attention_output_projection","name":"FP8 attention output projection","group":"Projections and GEMM","op":"FP8 o_proj GEMM (16384 -> 7168)","direction":"forward","model":"deepseek-ai/DeepSeek-R1-0528","workloads":16},{"board":"Quant","id":"011_fp8_moe_gate_routing","name":"FP8 MoE gate routing","group":"MLP and MoE","op":"FP8 router GEMM + sigmoid + bias + group top-k (8 groups / top-4 / top-8)","direction":"forward","model":"deepseek-ai/DeepSeek-R1-0528","workloads":16},{"board":"Quant","id":"012_fp8_shared_expert_mlp","name":"FP8 shared expert MLP","group":"MLP and MoE","op":"FP8 shared-expert SwiGLU MLP","direction":"forward","model":"deepseek-ai/DeepSeek-R1-0528","workloads":16},{"board":"Quant","id":"013_fp8_mla_kv_compression_projection","name":"FP8 MLA KV compression projection","group":"Projections and GEMM","op":"FP8 MLA KV path: kv_a GEMM -> RMSNorm -> kv_b GEMM","direction":"forward","model":"deepseek-ai/DeepSeek-V3.1-Terminus","workloads":16},{"board":"Quant","id":"014_fp8_yarn_rope_embedding","name":"FP8 YaRN RoPE embedding","group":"Attention","op":"YaRN RoPE cos/sin table (DeepSeek)","direction":"forward","model":"deepseek-ai/DeepSeek-V3.1-Terminus","workloads":16},{"board":"Quant","id":"015_fp8_mla_attention_output_projection","name":"FP8 MLA attention output projection","group":"Projections and GEMM","op":"FP8 MLA o_proj GEMM (pre-quantized act + weight)","direction":"forward","model":"deepseek-ai/DeepSeek-V3","workloads":16},{"board":"Quant","id":"016_fp8_multi_latent_attention_qkv_projection","name":"FP8 multi latent attention QKV projection","group":"Projections and GEMM","op":"FP8 MLA Q and KV projections with both latent RMSNorms","direction":"forward","model":"deepseek-ai/DeepSeek-V3","workloads":16},{"board":"Quant","id":"017_fp8_shared_expert_mlp","name":"FP8 shared expert MLP","group":"MLP and MoE","op":"FP8 shared-expert SwiGLU MLP (weight scales given)","direction":"forward","model":"deepseek-ai/DeepSeek-V3","workloads":16},{"board":"Quant","id":"018_nvfp4_attention_output_projection_with_residual","name":"NVFP4 attention output projection with residual","group":"Projections and GEMM","op":"NVFP4 o_proj GEMM + residual","direction":"forward","model":"mratsim/Nova-70B-NVFP4","workloads":16},{"board":"Quant","id":"019_nvfp4_grouped_query_attention","name":"NVFP4 grouped query attention","group":"Attention","op":"NVFP4 GQA attention block: QKV, RoPE, attention, o_proj","direction":"forward","model":"mratsim/Nova-70B-NVFP4","workloads":16},{"board":"Quant","id":"020_nvfp4_linear_layer","name":"NVFP4 linear layer","group":"Projections and GEMM","op":"NVFP4 linear (8192 x 8192)","direction":"forward","model":"mratsim/Nova-70B-NVFP4","workloads":16},{"board":"Quant","id":"021_fp8_mamba2_in_projection","name":"FP8 Mamba2 in projection","group":"State space and conv","op":"FP8 Mamba2 in-projection GEMM","direction":"forward","model":"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8","workloads":16},{"board":"Quant","id":"022_fp8_mamba2_out_projection","name":"FP8 Mamba2 out projection","group":"State space and conv","op":"FP8 Mamba2 out-projection GEMM (pre-quantized)","direction":"forward","model":"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8","workloads":16},{"board":"Quant","id":"023_fp8_mamba2_ssm_discretization","name":"FP8 Mamba2 SSM discretization","group":"State space and conv","op":"FP8 Mamba2 SSM discretisation","direction":"forward","model":"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8","workloads":6},{"board":"Quant","id":"024_nvfp4_attention_output_projection","name":"NVFP4 attention output projection","group":"Projections and GEMM","op":"NVFP4 o_proj GEMM (FP32 input)","direction":"forward","model":"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4","workloads":16},{"board":"Quant","id":"025_nvfp4_attention_qkv_projection","name":"NVFP4 attention QKV projection","group":"Projections and GEMM","op":"NVFP4 Q, K, V projections (FP32 input)","direction":"forward","model":"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4","workloads":16},{"board":"Quant","id":"026_nvfp4_mamba2_out_projection","name":"NVFP4 Mamba2 out projection","group":"State space and conv","op":"NVFP4 Mamba2 out-projection (FP4 act + weight)","direction":"forward","model":"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4","workloads":16},{"board":"Quant","id":"027_nvfp4_moe_router_projection","name":"NVFP4 MoE router projection","group":"MLP and MoE","op":"NVFP4 MoE router GEMM + sigmoid + bias + top-6","direction":"forward","model":"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4","workloads":16},{"board":"Quant","id":"028_nvfp4_attention_qkv_projection_with_rope","name":"NVFP4 attention QKV projection with RoPE","group":"Projections and GEMM","op":"NVFP4 QKV projection + QK-RMSNorm + RoPE","direction":"forward","model":"nvidia/Qwen3-8B-NVFP4","workloads":16},{"board":"Quant","id":"029_nvfp4_fused_mlp_silu_gating","name":"NVFP4 fused MLP SiLU gating","group":"MLP and MoE","op":"NVFP4 SwiGLU MLP (gate, up, down)","direction":"forward","model":"nvidia/Qwen3-8B-NVFP4","workloads":16},{"board":"Quant","id":"030_nvfp4_fused_decoder_layer_pre_norm_attention","name":"NVFP4 fused decoder layer pre norm attention","group":"Attention","op":"NVFP4 decoder pre-norm + full attention block","direction":"forward","model":"nvidia/Qwen3-14B-NVFP4","workloads":16},{"board":"Quant","id":"031_nvfp4_grouped_query_attention_with_kv_repeat","name":"NVFP4 grouped query attention with KV repeat","group":"Attention","op":"NVFP4-quantized attention core (FP4 QK^T and PV)","direction":"forward","model":"nvidia/Qwen3-14B-NVFP4","workloads":16},{"board":"Quant","id":"032_nvfp4_moe_expert_linear_with_gating","name":"NVFP4 MoE expert linear with gating","group":"MLP and MoE","op":"NVFP4 SwiGLU MLP (one expert-sized MLP, 2048 -> 5120)","direction":"forward","model":"nvidia/Qwen3-Next-80B-A3B-Thinking-NVFP4","workloads":16},{"board":"Quant","id":"033_nvfp4_moe_routing_with_topk_selection","name":"NVFP4 MoE routing with topk selection","group":"MLP and MoE","op":"NVFP4 router GEMM + softmax + top-10","direction":"forward","model":"nvidia/Qwen3-Next-80B-A3B-Thinking-NVFP4","workloads":16},{"board":"FlashInfer","id":"001_fused_add_rmsnorm_h2048","name":"Fused add RMSNorm h2048","group":"Normalization","op":"fused add + RMSNorm","direction":"forward","model":"flashinfer-trace tag: qwen3-30b-a3b","workloads":7},{"board":"FlashInfer","id":"002_fused_add_rmsnorm_h4096","name":"Fused add RMSNorm h4096","group":"Normalization","op":"fused add + RMSNorm","direction":"forward","model":"flashinfer-trace tag: llama-3.1-8b","workloads":14},{"board":"FlashInfer","id":"003_fused_add_rmsnorm_h7168","name":"Fused add RMSNorm h7168","group":"Normalization","op":"fused add + RMSNorm","direction":"forward","model":"flashinfer-trace tag: deepseek-v3, deepseek-r1","workloads":8},{"board":"FlashInfer","id":"004_gemm_n128_k2048","name":"GEMM n128 k2048","group":"Projections and GEMM","op":"dense GEMM C = A @ B^T (fp16)","direction":"forward","model":"flashinfer-trace tag: qwen3-30b-a3b (Qwen 3 30B A3B moe.gate)","workloads":25},{"board":"FlashInfer","id":"005_gemm_n256_k7168","name":"GEMM n256 k7168","group":"Projections and GEMM","op":"dense GEMM C = A @ B^T (fp16)","direction":"forward","model":"flashinfer-trace tag: deepseek-v3, deepseek-r1 (DeepSeek-V3 moe.gate)","workloads":17},{"board":"FlashInfer","id":"006_gemm_n2048_k4096","name":"GEMM n2048 k4096","group":"Projections and GEMM","op":"dense GEMM C = A @ B^T (fp16)","direction":"forward","model":"flashinfer-trace tag: qwen3-30b-a3b (Qwen 3 30B A3B attn.o_proj)","workloads":29},{"board":"FlashInfer","id":"007_gemm_n4096_k4096","name":"GEMM n4096 k4096","group":"Projections and GEMM","op":"dense GEMM C = A @ B^T (fp16)","direction":"forward","model":"flashinfer-trace tag: llama-3.1-8b (Llama 3.1 8B attn.o_proj)","workloads":43},{"board":"FlashInfer","id":"008_gemm_n4096_k14336","name":"GEMM n4096 k14336","group":"Projections and GEMM","op":"dense GEMM C = A @ B^T (fp16)","direction":"forward","model":"flashinfer-trace tag: llama-3.1-8b (Llama 3.1 8B mlp.down_proj)","workloads":43},{"board":"FlashInfer","id":"009_gemm_n5120_k2048","name":"GEMM n5120 k2048","group":"Projections and GEMM","op":"dense GEMM C = A @ B^T (fp16)","direction":"forward","model":"flashinfer-trace tag: qwen3-30b-a3b (Qwen 3 30B A3B attn.qkv_proj)","workloads":25},{"board":"FlashInfer","id":"010_gemm_n6144_k4096","name":"GEMM n6144 k4096","group":"Projections and GEMM","op":"dense GEMM C = A @ B^T (fp16)","direction":"forward","model":"flashinfer-trace tag: llama-3.1-8b (Llama 3.1 8B attn.qkv_proj)","workloads":43},{"board":"FlashInfer","id":"011_gemm_n28672_k4096","name":"GEMM n28672 k4096","group":"Projections and GEMM","op":"dense GEMM C = A @ B^T (fp16)","direction":"forward","model":"flashinfer-trace tag: llama-3.1-8b (Llama 3.1 8B mlp.gate_up_proj)","workloads":43},{"board":"FlashInfer","id":"012_gqa_paged_decode_h32_kv4_d128_ps1","name":"GQA paged decode h32 kv4 d128 ps1","group":"Attention","op":"GQA paged decode attention (page size 1)","direction":"forward","model":"flashinfer-trace tag: qwen3-30b-a3b (Qwen3-30B-A3B)","workloads":48},{"board":"FlashInfer","id":"013_gqa_paged_decode_h32_kv8_d128_ps1","name":"GQA paged decode h32 kv8 d128 ps1","group":"Attention","op":"GQA paged decode attention (page size 1)","direction":"forward","model":"flashinfer-trace tag: llama-3.1-8b (Llama-3.1-8B)","workloads":48},{"board":"FlashInfer","id":"014_gqa_paged_prefill_causal_h32_kv4_d128_ps1","name":"GQA paged prefill causal h32 kv4 d128 ps1","group":"Attention","op":"GQA paged prefill attention, causal (page size 1)","direction":"forward","model":"flashinfer-trace tag: qwen3-30b-a3b (Qwen3-30B-A3B during incremental prefill)","workloads":30},{"board":"FlashInfer","id":"015_gqa_paged_prefill_causal_h32_kv8_d128_ps1","name":"GQA paged prefill causal h32 kv8 d128 ps1","group":"Attention","op":"GQA paged prefill attention, causal (page size 1)","direction":"forward","model":"flashinfer-trace tag: llama-3.1-8b (Llama-3.1-8B during incremental prefill)","workloads":38},{"board":"FlashInfer","id":"016_gqa_ragged_prefill_causal_h32_kv4_d128","name":"GQA ragged prefill causal h32 kv4 d128","group":"Attention","op":"GQA ragged prefill attention, causal","direction":"forward","model":"flashinfer-trace tag: qwen3-30b-a3b (Qwen3-30B-A3B during total prefill)","workloads":15},{"board":"FlashInfer","id":"017_gqa_ragged_prefill_causal_h32_kv8_d128","name":"GQA ragged prefill causal h32 kv8 d128","group":"Attention","op":"GQA ragged prefill attention, causal","direction":"forward","model":"flashinfer-trace tag: llama-3.1-8b (Llama-3.1-8B during total prefill)","workloads":21},{"board":"FlashInfer","id":"018_mla_paged_decode_h16_ckv512_kpe64_ps1","name":"MLA paged decode h16 ckv512 kpe64 ps1","group":"Attention","op":"MLA paged decode attention (absorbed, ckv 512 + kpe 64, page size 1)","direction":"forward","model":"flashinfer-trace tag: deepseek-v3, deepseek-r1 (DeepSeek-V3 with tensor parallel size 8)","workloads":47},{"board":"FlashInfer","id":"019_mla_paged_prefill_causal_h16_ckv512_kpe64_ps1","name":"MLA paged prefill causal h16 ckv512 kpe64 ps1","group":"Attention","op":"MLA paged prefill attention, causal (absorbed)","direction":"forward","model":"flashinfer-trace tag: deepseek-v3, deepseek-r1 (DeepSeek-V3 during incremental prefill with tensor parallel size 8)","workloads":38},{"board":"FlashInfer","id":"020_moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048","name":"MoE FP8 block scale ds routing topk8 ng8 kg4 e32 h7168 i2048","group":"MLP and MoE","op":"FP8 block-scale MoE: DeepSeek routing + 2 grouped GEMMs (32 local experts)","direction":"forward","model":"flashinfer-trace tag: deepseek-v3, deepseek-r1","workloads":19},{"board":"FlashInfer","id":"021_rmsnorm_h128","name":"RMSNorm h128","group":"Normalization","op":"RMSNorm","direction":"forward","model":"flashinfer-trace tag: qwen3-30b-a3b (Qwen3-30B-A3B. Epsilon is fixed at 1e-6)","workloads":14},{"board":"FlashInfer","id":"022_rmsnorm_h512","name":"RMSNorm h512","group":"Normalization","op":"RMSNorm","direction":"forward","model":"flashinfer-trace tag: deepseek-v3, deepseek-r1 (DeepSeek-V3/R1. Epsilon is fixed at 1e-6)","workloads":8},{"board":"FlashInfer","id":"023_rmsnorm_h1536","name":"RMSNorm h1536","group":"Normalization","op":"RMSNorm","direction":"forward","model":"flashinfer-trace tag: deepseek-v3, deepseek-r1 (DeepSeek-V3/R1. Epsilon is fixed at 1e-6)","workloads":8},{"board":"FlashInfer","id":"024_rmsnorm_h2048","name":"RMSNorm h2048","group":"Normalization","op":"RMSNorm","direction":"forward","model":"flashinfer-trace tag: qwen3-30b-a3b (Qwen3-30B-A3B. Epsilon is fixed at 1e-6)","workloads":7},{"board":"FlashInfer","id":"025_rmsnorm_h4096","name":"RMSNorm h4096","group":"Normalization","op":"RMSNorm","direction":"forward","model":"flashinfer-trace tag: llama-3.1-8b (Llama-3.1-8B. Epsilon is fixed at 1e-5)","workloads":14},{"board":"FlashInfer","id":"026_rmsnorm_h7168","name":"RMSNorm h7168","group":"Normalization","op":"RMSNorm","direction":"forward","model":"flashinfer-trace tag: deepseek-v3, deepseek-r1 (DeepSeek-V3/R1. Epsilon is fixed at 1e-6)","workloads":8}]} |
