| { | |
| "SOAFusedAttention": { | |
| "id": "SOAFusedAttention", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "adaptive_span_attention": { | |
| "id": "adaptive_span_attention", | |
| "category": "C", | |
| "primary_formula": "S_{ij} = \\frac{Q_i K_j^\\top}{\\sqrt{d}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "adaptive_span_inject": { | |
| "id": "adaptive_span_inject", | |
| "category": "C", | |
| "primary_formula": "S_{ij} = \\frac{Q_i K_j^\\top}{\\sqrt{d}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "adaptive_sparse_attention": { | |
| "id": "adaptive_sparse_attention", | |
| "category": "E", | |
| "primary_formula": "\\gamma_h = \\sigma(\\text{logit}_h) \\in (0, 1)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "adaptive_sparse_inject": { | |
| "id": "adaptive_sparse_inject", | |
| "category": "B", | |
| "primary_formula": "\\gamma_h = \\sigma(\\text{logit}_h) \\in (0, 1)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "adaptive_svd_attention": { | |
| "id": "adaptive_svd_attention", | |
| "category": "E", | |
| "primary_formula": "W = U \\Sigma V^\\top = \\sum_{i=1}^{\\min(d_{\\text{out}}, d_{\\text{in}})} \\sigma_i u_i v_i^\\top", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "adasplash_attention": { | |
| "id": "adasplash_attention", | |
| "category": "C", | |
| "primary_formula": "S_{ij} = \\frac{Q_i K_j^\\top}{\\sqrt{d_h}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "adasplash_inject": { | |
| "id": "adasplash_inject", | |
| "category": "B", | |
| "primary_formula": "S_{ij} = \\frac{Q_i K_j^\\top}{\\sqrt{d_h}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "aegis_hybrid": { | |
| "id": "aegis_hybrid", | |
| "category": "E", | |
| "primary_formula": "O_t = \\alpha_t \\odot O_t^{\\text{local}} + (1 - \\alpha_t) \\odot O_t^{\\text{global}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "aegis_hybrid_attention": { | |
| "id": "aegis_hybrid_attention", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "aegis_spectron_fp8": { | |
| "id": "aegis_spectron_fp8", | |
| "category": "E", | |
| "primary_formula": "p_{t, d} = \\frac{x_{t, d}^2}{\\sum_{k=1}^D x_{t, k}^2 + \\epsilon}, \\quad d \\in \\{1, \\dots, D\\}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "aegis_v1": { | |
| "id": "aegis_v1", | |
| "category": "E", | |
| "primary_formula": "W = U \\Sigma V^\\top = \\sum_{k=1}^{\\min(d_{\\text{in}}, d_{\\text{out}})} \\sigma_k u_k v_k^\\top", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "aegis_v2": { | |
| "id": "aegis_v2", | |
| "category": "E", | |
| "primary_formula": "Q = X W_{Q1} W_{Q2}, \\quad K = X W_{K1} W_{K2}, \\quad V = X W_{V1} W_{V2}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "aegis_v3": { | |
| "id": "aegis_v3", | |
| "category": "E", | |
| "primary_formula": "e_j = \\exp\\left(\\frac{x_j - m}{\\tau}\\right), \\quad m_{\\text{scaled}} = \\operatorname{clamp}\\left(-\\frac{m}{\\tau}, -\\infty, 80.0\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "aegis_v4": { | |
| "id": "aegis_v4", | |
| "category": "E", | |
| "primary_formula": "Q = \\text{FactorisedLinear}_Q(X), \\quad K = \\text{FactorisedLinear}_K(X), \\quad V = \\text{FactorisedLinear}_V(X)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "aegis_v5": { | |
| "id": "aegis_v5", | |
| "category": "E", | |
| "primary_formula": "Q_{\\text{feat}} = \\operatorname{ELU}(Q_g \\Omega) + 1.0 \\in \\mathbb{R}^{B \\times H_g \\times L \\times F}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "aegis_v6": { | |
| "id": "aegis_v6", | |
| "category": "E", | |
| "primary_formula": "y = x (V \\Sigma U^\\top) + x (B^\\top A^\\top) + b", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "asvd_attention": { | |
| "id": "asvd_attention", | |
| "category": "E", | |
| "primary_formula": "y = ((x V^\\top) \\odot S) U^\\top + (x A^\\top) B^\\top + b", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "bam_umoe": { | |
| "id": "bam_umoe", | |
| "category": "F", | |
| "primary_formula": "\\mathbf{Q} = \\mathbf{X} \\mathbf{W}_Q, \\quad \\mathbf{K} = \\mathbf{X} \\mathbf{W}_K, \\quad \\mathbf{V}_{\\text{base}} = \\mathbf{X} \\mathbf{W}_V", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Router Gating", | |
| "formula": "G(x) = \text{Softmax}(\text{TopK}(x W_g, k))", | |
| "flops": 12288 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Expert Execution", | |
| "formula": "Y = \\sum_{e \\in \text{TopK}} G(x)_e E_e(x)", | |
| "flops": 2359296 | |
| } | |
| ], | |
| "total_flops_per_token": 2371584 | |
| }, | |
| "block_transformer_hiba": { | |
| "id": "block_transformer_hiba", | |
| "category": "A", | |
| "primary_formula": "\\mathbf{Q}_b = \\mathbf{Q}[:, :, \\operatorname{start}_b:\\operatorname{end}_b, :], \\quad \\mathbf{K}_b = \\mathbf{K}[:, :, \\operatorname{start}_b:\\operatorname{end}_b, :], \\quad \\mathbf{V}_b = \\mathbf{V}[:, :, \\operatorname{start}_b:\\operatorname{end}_b, :]", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "chela_slay": { | |
| "id": "chela_slay", | |
| "category": "B", | |
| "primary_formula": "\\hat{Q} = \\frac{Q}{\\|Q\\|_2}, \\quad \\hat{K} = \\frac{K}{\\|K\\|_2}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "cluster_attention": { | |
| "id": "cluster_attention", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "cluster_inject": { | |
| "id": "cluster_inject", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "compressed_attention": { | |
| "id": "compressed_attention", | |
| "category": "E", | |
| "primary_formula": "\\mathbf{W} \\approx \\mathbf{U}_r \\mathbf{\\Sigma}_r \\mathbf{V}_r^T", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "compressed_mha": { | |
| "id": "compressed_mha", | |
| "category": "B", | |
| "primary_formula": "\\mathbf{W}_{QKV} = [\\mathbf{W}_Q; \\mathbf{W}_K; \\mathbf{W}_V] \\in \\mathbb{R}^{3D \\times D}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "compressed_mha_attention": { | |
| "id": "compressed_mha_attention", | |
| "category": "E", | |
| "primary_formula": "\\mathbf{W}_{QKV} = [\\mathbf{W}_Q; \\mathbf{W}_K; \\mathbf{W}_V] \\in \\mathbb{R}^{3D \\times D}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "cosformer": { | |
| "id": "cosformer", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "cosformer_inject": { | |
| "id": "cosformer_inject", | |
| "category": "A", | |
| "primary_formula": "h_t = \\bar{A} h_{t-1} + \\bar{B} x_t, \\quad y_t = C h_t + D x_t, \\quad \\bar{A} = \\exp(\\Delta A)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "decay_linear_attention": { | |
| "id": "decay_linear_attention", | |
| "category": "D", | |
| "primary_formula": "Q = X W_q, \\quad K = X W_k, \\quad V = X W_v", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "eat": { | |
| "id": "eat", | |
| "category": "G", | |
| "primary_formula": "O = \\operatorname{Softmax}\\left(\\frac{Q K^T}{\\sqrt{d_h}} + M\\right) V W_O", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "expert_choice_moe": { | |
| "id": "expert_choice_moe", | |
| "category": "E", | |
| "primary_formula": "\\mathbf{Q} = \\mathbf{X} \\mathbf{W}_Q, \\quad \\mathbf{K} = \\mathbf{X} \\mathbf{W}_K, \\quad \\mathbf{V} = \\mathbf{X} \\mathbf{W}_V", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "factorised_matrix_state": { | |
| "id": "factorised_matrix_state", | |
| "category": "E", | |
| "primary_formula": "Q = X W_q, \\quad K = X W_k, \\quad V = X W_v \\quad (W_q, W_k, W_v \\in \\mathbb{R}^{D \\times D})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "flash_attention3": { | |
| "id": "flash_attention3", | |
| "category": "E", | |
| "primary_formula": "S = \\frac{Q K^\\top}{\\sqrt{d_h}} + M", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "flash_attention3_fp8": { | |
| "id": "flash_attention3_fp8", | |
| "category": "E", | |
| "primary_formula": "Q_{\\text{fp8}} = \\text{cast}_{\\text{fp8\\_e4m3}}(Q) \\in \\mathbb{F}_{p8}^{B \\times L \\times H \\times d_h}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "full_attention": { | |
| "id": "full_attention", | |
| "category": "E", | |
| "primary_formula": "\\text{Attention}(Q, K, V) = \\text{softmax}\\left(\\frac{Q K^\\top}{\\sqrt{d_k}} + M\\right) V", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "fused_svd_kernel": { | |
| "id": "fused_svd_kernel", | |
| "category": "E", | |
| "primary_formula": "\\min_{\\text{rank}(\\widehat{W}) \\le r} \\|W - \\widehat{W}\\|_F^2 = \\sum_{i=r+1}^{D} \\sigma_i^2", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "fusion_attn": { | |
| "id": "fusion_attn", | |
| "category": "E", | |
| "primary_formula": "\\mathbf{S}_{\\text{blk}} = \\frac{\\mathbf{G}_Q(\\mathbf{Q}_{\\text{blk}}) \\mathbf{G}_K(\\mathbf{K}_{\\text{blk}})^T}{\\tau}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "galerkin": { | |
| "id": "galerkin", | |
| "category": "E", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "galerkin_attention": { | |
| "id": "galerkin_attention", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "galerkin_inject": { | |
| "id": "galerkin_inject", | |
| "category": "E", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "gated_delta_net": { | |
| "id": "gated_delta_net", | |
| "category": "D", | |
| "primary_formula": "Q = X W_q \\in \\mathbb{R}^{B \\times L \\times D}, \\quad K = X W_k \\in \\mathbb{R}^{B \\times L \\times D}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "gated_deltanet_block": { | |
| "id": "gated_deltanet_block", | |
| "category": "D", | |
| "primary_formula": "\\text{Output} = \\begin{cases}\n \\text{LayerNorm}(X + \\text{MHA}(X, X, X)), & \\text{if } \\text{use\\_full\\_attention} \\\\\n \\text{LayerNorm}(X + \\text{GatedDeltaNet}(X)), & \\text{otherwise}\n\\end{cases}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "gated_linear_attention": { | |
| "id": "gated_linear_attention", | |
| "category": "D", | |
| "primary_formula": "Q = X W_q, \\quad K = X W_k, \\quad V = X W_v, \\quad Z = X W_z", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "goldfinch_ahn": { | |
| "id": "goldfinch_ahn", | |
| "category": "D", | |
| "primary_formula": "h_t = \\bar{A} h_{t-1} + \\bar{B} x_t, \\quad y_t = C h_t + D x_t, \\quad \\bar{A} = \\exp(\\Delta A)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "gqa": { | |
| "id": "gqa", | |
| "category": "B", | |
| "primary_formula": "K_{\\text{shared}} = \\text{repeat\\_interleave}(K, H_Q / H_{KV}, \\text{dim}=\\text{heads})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "gqa_attention": { | |
| "id": "gqa_attention", | |
| "category": "A", | |
| "primary_formula": "K_{\\text{shared}} = \\text{repeat\\_interleave}(K, H_Q / H_{KV}, \\text{dim}=\\text{heads})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "granite_hybrid": { | |
| "id": "granite_hybrid", | |
| "category": "D", | |
| "primary_formula": "[X_{\\text{ssm}}, Z] = X W_{\\text{in}} \\in \\mathbb{R}^{B \\times L \\times 2 D_{\\text{inner}}}, \\quad D_{\\text{inner}} = D \\cdot \\text{expand}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "griffin_ssd": { | |
| "id": "griffin_ssd", | |
| "category": "E", | |
| "primary_formula": "S^{\\text{local}}_{b, h, i, j} = \\frac{\\langle q_{b, h, i}, k_{b, h, j} \\rangle}{\\sqrt{d}} + M^{\\text{window}}_{i, j} + M^{\\text{pad}}_{b, j}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "halo_regla": { | |
| "id": "halo_regla", | |
| "category": "B", | |
| "primary_formula": "Q_{\\text{rope}} = \\operatorname{RoPE}(Q), \\quad K_{\\text{rope}} = \\operatorname{RoPE}(K)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "hamming_attention": { | |
| "id": "hamming_attention", | |
| "category": "E", | |
| "primary_formula": "Q_b = \\tanh(5.0 \\cdot \\gamma_Q \\odot Q) \\in [-1, 1]^{B \\times H \\times L \\times d_k}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "hgrn2_gated_deltanet": { | |
| "id": "hgrn2_gated_deltanet", | |
| "category": "D", | |
| "primary_formula": "\\begin{aligned}\nY_{\\text{hgrn}} &= \\text{HGRN2}(X) \\\\\nY_{\\text{deltanet}} &= \\text{GatedDeltaNet}(X) \\\\\n\\gamma &= \\sigma(W_g X) \\\\\nY &= \\gamma \\odot Y_{\\text{hgrn}} + (1 - \\gamma) \\odot Y_{\\text{deltanet}} \\\\\n\\text{Output} &= Y W_o\n\\end{aligned}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "hgrn2_layer": { | |
| "id": "hgrn2_layer", | |
| "category": "D", | |
| "primary_formula": "f_t = \\exp(-\\operatorname{softplus}(W_f x_t + b_f)) \\in (0, 1)^D, \\quad b_f = 1.5", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "hgrn2_outer": { | |
| "id": "hgrn2_outer", | |
| "category": "D", | |
| "primary_formula": "Q = X W_q, \\quad K = X W_k, \\quad V = X W_v", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "hla_attention": { | |
| "id": "hla_attention", | |
| "category": "D", | |
| "primary_formula": "\\phi(Q) = \\text{ELU}(X W_q) + 1, \\quad \\phi(K) = \\text{ELU}(X W_k) + 1, \\quad V = X W_v", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "hla_inject": { | |
| "id": "hla_inject", | |
| "category": "B", | |
| "primary_formula": "\\phi(Q) = \\text{ELU}(X W_q) + 1, \\quad \\phi(K) = \\text{ELU}(X W_k) + 1, \\quad V = X W_v", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "hymba_transx": { | |
| "id": "hymba_transx", | |
| "category": "E", | |
| "primary_formula": "\\mathbf{R}_{\\Theta, m}^{d_k} \\mathbf{x}_m = \\begin{pmatrix} x_{m, 0} \\cos(m \\theta_0) - x_{m, 1} \\sin(m \\theta_0) \\\\ x_{m, 0} \\sin(m \\theta_0) + x_{m, 1} \\cos(m \\theta_0) \\\\ \\vdots \\end{pmatrix}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "infini_attention": { | |
| "id": "infini_attention", | |
| "category": "E", | |
| "primary_formula": "\\mathbf{Y} = \\mathbf{Y}_{\\text{local}} + \\sigma(\\beta) \\mathbf{Y}_{\\text{mem}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "jamba": { | |
| "id": "jamba", | |
| "category": "D", | |
| "primary_formula": "\\mathbf{X}_{\\text{proj}} = \\mathbf{X} \\mathbf{W}_{\\text{in}}, \\quad \\mathbf{W}_{\\text{in}} \\in \\mathbb{R}^{D \\times D}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "jamba_info": { | |
| "id": "jamba_info", | |
| "category": "E", | |
| "primary_formula": "\\mathbf{Y} = \\mathbf{W}_O \\left(\\mathbf{A}_{\\text{diff}} \\mathbf{V} + 0.1 \\cdot \\text{SimpleSSM}(\\text{MLP}_{\\text{down-up}}(\\mathbf{X}))\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "kda_attention": { | |
| "id": "kda_attention", | |
| "category": "D", | |
| "primary_formula": "Q = \\text{normalize}(X W_q, \\text{dim}=-1), \\quad K = \\frac{\\text{normalize}(X W_k, \\text{dim}=-1)}{\\sqrt{d_k}}, \\quad V = X W_v", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "kda_inject": { | |
| "id": "kda_inject", | |
| "category": "B", | |
| "primary_formula": "Q = \\text{normalize}(X W_q, \\text{dim}=-1), \\quad K = \\frac{\\text{normalize}(X W_k, \\text{dim}=-1)}{\\sqrt{d_k}}, \\quad V = X W_v", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "leka_v1": { | |
| "id": "leka_v1", | |
| "category": "E", | |
| "primary_formula": "\\mathbf{O}_i = \\sum_{j=1}^N \\frac{\\exp\\left(\\frac{\\mathbf{q}_i \\mathbf{k}_j^\\top}{\\sqrt{d_k}}\\right)}{\\sum_{m=1}^N \\exp\\left(\\frac{\\mathbf{q}_i \\mathbf{k}_m^\\top}{\\sqrt{d_k}}\\right)} \\mathbf{v}_j", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "leka_v1_inject": { | |
| "id": "leka_v1_inject", | |
| "category": "B", | |
| "primary_formula": "\\mathbf{O}_i = \\sum_{j=1}^N \\frac{\\exp\\left(\\frac{\\mathbf{q}_i \\mathbf{k}_j^\\top}{\\sqrt{d_k}}\\right)}{\\sum_{m=1}^N \\exp\\left(\\frac{\\mathbf{q}_i \\mathbf{k}_m^\\top}{\\sqrt{d_k}}\\right)} \\mathbf{v}_j", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "leka_v2": { | |
| "id": "leka_v2", | |
| "category": "E", | |
| "primary_formula": "\\phi_1(\\mathbf{x}) = \\frac{\\left[\\cos(\\mathbf{x} \\mathbf{W}_1), \\; \\sin(\\mathbf{x} \\mathbf{W}_1)\\right]}{\\sqrt{F_1}}, \\quad \\phi_2(\\mathbf{x}) = \\frac{\\left[\\cos(\\mathbf{x} \\mathbf{W}_2), \\; \\sin(\\mathbf{x} \\mathbf{W}_2)\\right]}{\\sqrt{F_2}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "leka_v2_inject": { | |
| "id": "leka_v2_inject", | |
| "category": "B", | |
| "primary_formula": "\\phi_1(\\mathbf{x}) = \\frac{\\left[\\cos(\\mathbf{x} \\mathbf{W}_1), \\; \\sin(\\mathbf{x} \\mathbf{W}_1)\\right]}{\\sqrt{F_1}}, \\quad \\phi_2(\\mathbf{x}) = \\frac{\\left[\\cos(\\mathbf{x} \\mathbf{W}_2), \\; \\sin(\\mathbf{x} \\mathbf{W}_2)\\right]}{\\sqrt{F_2}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "leka_v3": { | |
| "id": "leka_v3", | |
| "category": "E", | |
| "primary_formula": "\\mathbf{s} = \\operatorname{MLP}_{\\text{scorer}}(\\mathbf{V}) = \\mathbf{W}_2 \\operatorname{GELU}(\\mathbf{V} \\mathbf{W}_1 + \\mathbf{b}_1) + \\mathbf{b}_2 \\in \\mathbb{R}^{B \\times H \\times N}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "leka_v3_inject": { | |
| "id": "leka_v3_inject", | |
| "category": "B", | |
| "primary_formula": "\\mathbf{s} = \\operatorname{MLP}_{\\text{scorer}}(\\mathbf{V}) = \\mathbf{W}_2 \\operatorname{GELU}(\\mathbf{V} \\mathbf{W}_1 + \\mathbf{b}_1) + \\mathbf{b}_2 \\in \\mathbb{R}^{B \\times H \\times N}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "linformer_attention": { | |
| "id": "linformer_attention", | |
| "category": "E", | |
| "primary_formula": "c^{KV}_t = W^{DKV} x_t, \\quad k^C_t = W^{UK} c^{KV}_t, \\quad v^C_t = W^{UV} c^{KV}_t", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "lola_attention": { | |
| "id": "lola_attention", | |
| "category": "E", | |
| "primary_formula": "W \\approx U V, \\quad U \\in \\mathbb{R}^{d_{\\text{out}} \\times r}, \\; V \\in \\mathbb{R}^{r \\times d_{\\text{in}}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "lora_attention": { | |
| "id": "lora_attention", | |
| "category": "E", | |
| "primary_formula": "y = W_0 x + \\frac{\\alpha}{r} B A x", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "lsh_attention": { | |
| "id": "lsh_attention", | |
| "category": "E", | |
| "primary_formula": "\\mathbf{Q} = \\mathbf{X} \\mathbf{W}_Q, \\quad \\mathbf{K} = \\mathbf{X} \\mathbf{W}_K, \\quad \\mathbf{V} = \\mathbf{X} \\mathbf{W}_V \\quad \\in \\mathbb{R}^{B \\times H \\times N \\times d_k}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "mamba": { | |
| "id": "mamba", | |
| "category": "D", | |
| "primary_formula": "[U, Z] = \\operatorname{chunk}(X W_{\\text{in}}, 2) \\in \\mathbb{R}^{B \\times L \\times D}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "mamba_inject": { | |
| "id": "mamba_inject", | |
| "category": "D", | |
| "primary_formula": "h_t = \\bar{A} h_{t-1} + \\bar{B} x_t, \\quad y_t = C h_t + D x_t, \\quad \\bar{A} = \\exp(\\Delta A)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "mamba_svd_flash": { | |
| "id": "mamba_svd_flash", | |
| "category": "D", | |
| "primary_formula": "U_t = \\text{proj}_u(x_t) \\in \\mathbb{R}^{D \\times r}, \\quad V_t = \\text{proj}_v(x_t) \\in \\mathbb{R}^{r \\times D}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "minimal_aurora": { | |
| "id": "minimal_aurora", | |
| "category": "E", | |
| "primary_formula": "W_0 = U \\Sigma V^T, \\quad \\Sigma = \\operatorname{diag}(\\sigma_1, \\sigma_2, \\dots, \\sigma_{\\min(d_{\\text{out}}, d_{\\text{in}})})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "minimal_nova": { | |
| "id": "minimal_nova", | |
| "category": "E", | |
| "primary_formula": "W_0 = U \\Sigma V^T, \\quad \\Sigma = \\operatorname{diag}(\\sigma_1, \\sigma_2, \\dots, \\sigma_{\\min(d_{\\text{out}}, d_{\\text{in}})})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "minimal_prism": { | |
| "id": "minimal_prism", | |
| "category": "E", | |
| "primary_formula": "W_0 = U \\Sigma V^T, \\quad \\Sigma = \\operatorname{diag}(\\sigma_1, \\sigma_2, \\dots, \\sigma_{\\min(d_{\\text{out}}, d_{\\text{in}})})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "minimal_svd": { | |
| "id": "minimal_svd", | |
| "category": "E", | |
| "primary_formula": "A = U_{:, :r} \\text{diag}(\\sqrt{S_{:r}}) \\in \\mathbb{R}^{\\text{out} \\times r}, \\quad B = \\text{diag}(\\sqrt{S_{:r}}) V_{:r, :} \\in \\mathbb{R}^{r \\times \\text{in}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "minimal_svd_v2": { | |
| "id": "minimal_svd_v2", | |
| "category": "E", | |
| "primary_formula": "W = U \\Sigma V^\\top \\approx U_{:, :r} \\Sigma_{:r} V_{:r, :}^\\top = A B", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "mixattention_moska": { | |
| "id": "mixattention_moska", | |
| "category": "B", | |
| "primary_formula": "\\operatorname{Var}(\\mathbf{K}_{b, h, t}) = \\frac{1}{d_k} \\sum_{c=1}^{d_k} \\left(K_{b, h, t, c} - \\bar{K}_{b, h, t}\\right)^2", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "mla_attention": { | |
| "id": "mla_attention", | |
| "category": "E", | |
| "primary_formula": "\\mathbf{c}_{KV} = W_{DKV} X \\in \\mathbb{R}^{B \\times N \\times d_c}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "moba": { | |
| "id": "moba", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "moba_attention": { | |
| "id": "moba_attention", | |
| "category": "B", | |
| "primary_formula": "\\mathbf{Q} = \\mathbf{X} \\mathbf{W}_Q, \\quad \\mathbf{K} = \\mathbf{X} \\mathbf{W}_K, \\quad \\mathbf{V} = \\mathbf{X} \\mathbf{W}_V", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "moe_deltakv_paged": { | |
| "id": "moe_deltakv_paged", | |
| "category": "E", | |
| "primary_formula": "\\Delta \\mathbf{K}_t = \\mathbf{K}_t - \\mathbf{K}_{t-1}, \\quad \\forall t \\in \\{1, \\dots, N-1\\}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "nasc_attention": { | |
| "id": "nasc_attention", | |
| "category": "E", | |
| "primary_formula": "Q_{\\text{loc}}, K_{\\text{loc}}, V_{\\text{loc}} = \\text{SVD-LoRA}(X)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "nexus": { | |
| "id": "nexus", | |
| "category": "B", | |
| "primary_formula": "\\mathbf{Q}_{\\text{sample}} = \\mathbf{Q}[:, :, ::\\sigma, :], \\quad \\mathbf{K}_{\\text{sample}} = \\mathbf{K}[:, :, ::\\sigma, :]", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "nha_attention": { | |
| "id": "nha_attention", | |
| "category": "E", | |
| "primary_formula": "\\mathbf{Y} = \\gamma(\\mathbf{x}) \\mathbf{Y}_{\\text{local}} + (1 - \\gamma(\\mathbf{x})) \\mathbf{Y}_{\\text{global}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "nha_compressor": { | |
| "id": "nha_compressor", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "nsa_attention": { | |
| "id": "nsa_attention", | |
| "category": "E", | |
| "primary_formula": "\\mathbf{Q} = \\mathbf{X} \\mathbf{W}_Q, \\quad \\mathbf{K} = \\mathbf{X} \\mathbf{W}_K, \\quad \\mathbf{V} = \\mathbf{X} \\mathbf{W}_V \\quad \\in \\mathbb{R}^{B \\times H \\times L \\times d_h}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "nystrom_attention": { | |
| "id": "nystrom_attention", | |
| "category": "E", | |
| "primary_formula": "\\hat{A} = \\hat{F}_1 \\hat{F}_2^+ \\hat{F}_3", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "optimized_hybrid_attention": { | |
| "id": "optimized_hybrid_attention", | |
| "category": "E", | |
| "primary_formula": "Q = X W_Q, \\quad K = X W_K, \\quad V = X W_V \\in \\mathbb{R}^{B \\times N \\times H \\cdot d_h}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "orion": { | |
| "id": "orion", | |
| "category": "E", | |
| "primary_formula": "W = U \\Sigma V^T, \\quad \\Sigma = \\text{diag}(\\sigma_1, \\sigma_2, \\dots, \\sigma_{\\min(d_{\\text{out}}, d_{\\text{in}})})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "performer_attention": { | |
| "id": "performer_attention", | |
| "category": "E", | |
| "primary_formula": "\\text{Attn}(Q, K, V) = D^{-1} (\\phi(Q) (\\phi(K)^\\top V)), \\quad \\phi(x) = \\frac{h(x)}{\\sqrt{m}} \\exp\\left(\\omega^\\top x - \\frac{\\|x\\|^2}{2}\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "phantom": { | |
| "id": "phantom", | |
| "category": "E", | |
| "primary_formula": "\\ell_{\\text{row}}(i) = \\frac{\\|U_{i, :}\\|_2^2}{\\sum_{i'=1}^{d_{\\text{out}}} \\|U_{i', :}\\|_2^2}, \\quad \\ell_{\\text{col}}(j) = \\frac{\\|V_{:, j}\\|_2^2}{\\sum_{j'=1}^{d_{\\text{in}}} \\|V_{:, j'}\\|_2^2}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "primal_attention": { | |
| "id": "primal_attention", | |
| "category": "E", | |
| "primary_formula": "F_1 = \\frac{Q \\widetilde{K}^T}{\\sqrt{d}} \\in \\mathbb{R}^{B \\times H \\times N \\times r}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "progressive_token_pruning": { | |
| "id": "progressive_token_pruning", | |
| "category": "E", | |
| "primary_formula": "\\min_{\\text{rank}(\\widehat{W}) \\le r} \\|W - \\widehat{W}\\|_F^2 = \\sum_{i=r+1}^{D} \\sigma_i^2", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "projection_factorized_attn": { | |
| "id": "projection_factorized_attn", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "qcqa_attention": { | |
| "id": "qcqa_attention", | |
| "category": "A", | |
| "primary_formula": "\\pi_{h, j} = \\frac{\\exp\\left(\\frac{q_h W_{\\text{router}, j}}{\\tau}\\right)}{\\sum_{j'=1}^{H_{kv}} \\exp\\left(\\frac{q_h W_{\\text{router}, j'}}{\\tau}\\right)}, \\quad \\forall j \\in \\{1, \\dots, H_{kv}\\}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "rala_attention": { | |
| "id": "rala_attention", | |
| "category": "E", | |
| "primary_formula": "\\text{Attention}_{\\text{RALA}}(Q, K, V) = \\frac{\\phi(Q) (\\phi(K)^\\top V)}{\\phi(Q) (\\phi(K)^\\top \\mathbf{1})} + \\alpha \\frac{\\phi(Q) (U V^\\top)}{\\phi(Q) (\\phi(K)^\\top \\mathbf{1})}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "rala_compressor": { | |
| "id": "rala_compressor", | |
| "category": "E", | |
| "primary_formula": "S_{\\text{KV}} = \\sum_{j=1}^N \\phi(K_j) V_j^T \\in \\mathbb{R}^{B \\times H \\times d \\times d}, \\quad z = \\sum_{j=1}^N \\phi(K_j) \\in \\mathbb{R}^{B \\times H \\times d}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "reap_moe": { | |
| "id": "reap_moe", | |
| "category": "E", | |
| "primary_formula": "\\mathbf{W}_e = \\mathbf{U}_e \\mathbf{\\Sigma}_e \\mathbf{V}_e^\\top, \\quad \\operatorname{rank}(\\mathbf{W}_e) \\le r \\ll \\min(d_{\\text{in}}, d_{\\text{out}})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "regla_attention": { | |
| "id": "regla_attention", | |
| "category": "E", | |
| "primary_formula": "Q = X W_Q, \\quad K = X W_K, \\quad V = X W_V, \\quad G = X W_G", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "replace": { | |
| "id": "replace", | |
| "category": "B", | |
| "primary_formula": "\\min_{\\Theta_{\\text{new}}} \\|f_{\\text{new}}(X; \\Theta_{\\text{new}}) - f_{\\text{old}}(X; \\Theta_{\\text{old}})\\|_F", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "res_svd": { | |
| "id": "res_svd", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "routing_attention": { | |
| "id": "routing_attention", | |
| "category": "B", | |
| "primary_formula": "\\mathbf{Q} = \\mathbf{X} \\mathbf{W}_Q, \\quad \\mathbf{K} = \\mathbf{X} \\mathbf{W}_K, \\quad \\mathbf{V} = \\mathbf{X} \\mathbf{W}_V \\quad \\in \\mathbb{R}^{B \\times H \\times L \\times d_h}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "rpc": { | |
| "id": "rpc", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "rpc_attention": { | |
| "id": "rpc_attention", | |
| "category": "A", | |
| "primary_formula": "\\widetilde{Q} = Q P \\in \\mathbb{R}^{B \\times H \\times L \\times d_p}, \\quad \\widetilde{K} = K P \\in \\mathbb{R}^{B \\times H \\times L \\times d_p}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "salt": { | |
| "id": "salt", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "salt_attention": { | |
| "id": "salt_attention", | |
| "category": "E", | |
| "primary_formula": "W = U_{:, :r} \\operatorname{diag}(\\Sigma_{1:r} \\odot \\gamma) V_{1:r, :}^\\top + A B", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "samba_m3": { | |
| "id": "samba_m3", | |
| "category": "D", | |
| "primary_formula": "\\mathbf{M}^{\\text{window}}_{i, j} = \\begin{cases} 0 & \\text{if } 0 \\le i - j < W \\\\ -\\infty & \\text{otherwise} \\end{cases}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "samba_moe": { | |
| "id": "samba_moe", | |
| "category": "D", | |
| "primary_formula": "\\mathbf{h}_t' = \\mathbf{A} \\mathbf{h}_{t-1}' + \\mathbf{B} \\mathbf{x}_t, \\quad \\mathbf{y}_{\\text{ssm}, t} = \\mathbf{C} \\mathbf{h}_t' + \\mathbf{D} \\mathbf{x}_t", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "simple_ssm": { | |
| "id": "simple_ssm", | |
| "category": "A", | |
| "primary_formula": "[X_{\\text{ssm}}, Z] = \\operatorname{chunk}(X W_{\\text{in}}, 2) \\in \\mathbb{R}^{B \\times L \\times D}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "smart_weighted_ensemble": { | |
| "id": "smart_weighted_ensemble", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "soa_attn": { | |
| "id": "soa_attn", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_{:, :r} \\Sigma_{:r, :r} V_{:, :r}^T, \\quad r=64", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "soa_fused_module": { | |
| "id": "soa_fused_module", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "soa_module": { | |
| "id": "soa_module", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_{:, :r} \\Sigma_{:r, :r} V_{:, :r}^T, \\quad r=64", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "span_expanded_attention": { | |
| "id": "span_expanded_attention", | |
| "category": "E", | |
| "primary_formula": "A = \\operatorname{Softmax}\\left(\\frac{Q K^\\top}{\\sqrt{d}} + M\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "spectra_v2": { | |
| "id": "spectra_v2", | |
| "category": "B", | |
| "primary_formula": "M^{\\text{span}}_{h, i, j} = \\ln\\left( \\sigma\\left( 5.0 \\cdot (S_h - d_{ij}) \\right) + 10^{-5} \\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "spectra_v2_attention": { | |
| "id": "spectra_v2_attention", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "spectral_mamba": { | |
| "id": "spectral_mamba", | |
| "category": "D", | |
| "primary_formula": "Q = X W_Q, \\quad K = X W_K, \\quad V = X W_V", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "spectral_mamba_attention": { | |
| "id": "spectral_mamba_attention", | |
| "category": "D", | |
| "primary_formula": "h_t = \\bar{A} h_{t-1} + \\bar{B} x_t, \\quad y_t = C h_t + D x_t, \\quad \\bar{A} = \\exp(\\Delta A)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "spectron": { | |
| "id": "spectron", | |
| "category": "E", | |
| "primary_formula": "\\min_{\\{r_i\\}} \\sum_{i=1}^M \\left(1 - \\frac{\\sum_{k=1}^{r_i} \\sigma_{i, k}^2}{\\sum_k \\sigma_{i, k}^2}\\right) \\quad \\text{s.t.} \\quad \\sum_{i=1}^M r_i (d_{i, \\text{in}} + d_{i, \\text{out}}) \\le B_{\\text{target}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "spectron_attention": { | |
| "id": "spectron_attention", | |
| "category": "H", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "squeezing_heads": { | |
| "id": "squeezing_heads", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "squeezing_heads_distill": { | |
| "id": "squeezing_heads_distill", | |
| "category": "B", | |
| "primary_formula": "Q = X W_Q, \\quad K = X W_K, \\quad V = X W_V \\in \\mathbb{R}^{B \\times H \\times L \\times d}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "squeezing_heads_distiller": { | |
| "id": "squeezing_heads_distiller", | |
| "category": "B", | |
| "primary_formula": "\\mathcal{L} = \\alpha D_{\\text{KL}}(A_{\\text{teacher}} \\parallel A_{\\text{student}}) + \\beta \\|V_{\\text{teacher}} - V_{\\text{student}} W_{\\text{proj}}\\|_F^2", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "svd_attention": { | |
| "id": "svd_attention", | |
| "category": "E", | |
| "primary_formula": "X_{\\text{valid}} = X \\odot M_{\\text{valid}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "svd_compressed": { | |
| "id": "svd_compressed", | |
| "category": "E", | |
| "primary_formula": "E_i(r) = \\frac{\\sum_{j=1}^r \\sigma_{i, j}^2}{\\sum_{j=1}^{\\min(d_{\\text{in}}, d_{\\text{out}})} \\sigma_{i, j}^2}, \\quad \\operatorname{Cost}_i(r) = 1 - E_i(r)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "svd_compressed_attention": { | |
| "id": "svd_compressed_attention", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "svd_linear_attention": { | |
| "id": "svd_linear_attention", | |
| "category": "D", | |
| "primary_formula": "\\\\text{Attention}(Q, K, V) = \\\\frac{\\\\phi(Q) (\\\\phi(K)^\\\\top V)}{\\\\phi(Q) \\\\sum_j \\\\phi(k_j)^\\\\top + \\\\epsilon}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "svd_linear_decay": { | |
| "id": "svd_linear_decay", | |
| "category": "D", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "svd_llm_v2": { | |
| "id": "svd_llm_v2", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "svd_llm_v2_attention": { | |
| "id": "svd_llm_v2_attention", | |
| "category": "E", | |
| "primary_formula": "\\frac{\\sum_{i=1}^{r^*} \\sigma_i^2}{\\sum_{j=1}^d \\sigma_j^2} \\ge \\tau \\quad (\\tau = 0.92)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "svd_mamba": { | |
| "id": "svd_mamba", | |
| "category": "D", | |
| "primary_formula": "W = U \\Sigma V^\\top \\approx W_{\\text{down}} W_{\\text{up}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "svd_proj_attn": { | |
| "id": "svd_proj_attn", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "svd_projection_attention": { | |
| "id": "svd_projection_attention", | |
| "category": "E", | |
| "primary_formula": "W = U V \\in \\mathbb{R}^{d_{\\text{out}} \\times d_{\\text{in}}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "svd_sequence_attention": { | |
| "id": "svd_sequence_attention", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "template_attention": { | |
| "id": "template_attention", | |
| "category": "B", | |
| "primary_formula": "Q = X W_Q, \\quad K = X W_K, \\quad V = X W_V", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "token_compressed": { | |
| "id": "token_compressed", | |
| "category": "B", | |
| "primary_formula": "\\min_{\\text{rank}(\\widehat{W}) \\le r} \\|W - \\widehat{W}\\|_F^2 = \\sum_{i=r+1}^{D} \\sigma_i^2", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "token_compressed_attention": { | |
| "id": "token_compressed_attention", | |
| "category": "G", | |
| "primary_formula": "\\text{energy}_i = \\|x_i\\|_2, \\quad \\text{score}_i = \\text{ScoreNet}(x_i)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "tpa_attention": { | |
| "id": "tpa_attention", | |
| "category": "E", | |
| "primary_formula": "q_1 = \\text{LoRALinear}_{Q1}(X) \\in \\mathbb{R}^{B \\times L \\times (H \\cdot r)}, \\quad q_2 = \\text{LoRALinear}_{Q2}(X) \\in \\mathbb{R}^{B \\times L \\times (H \\cdot r)}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "tt": { | |
| "id": "tt", | |
| "category": "B", | |
| "primary_formula": "\\text{Output} = \\text{BaseAttn}(X, M)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "unified_rope": { | |
| "id": "unified_rope", | |
| "category": "B", | |
| "primary_formula": "Q = \\operatorname{LoRALinear}_Q(X) \\in \\mathbb{R}^{B \\times L \\times (H \\cdot d_{\\text{head}})}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "upi_mamba": { | |
| "id": "upi_mamba", | |
| "category": "D", | |
| "primary_formula": "[X_{\\text{ssm}}, Z] = X W_{\\text{in}} \\in \\mathbb{R}^{B \\times L \\times 2 D}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "upi_stuffed_mamba": { | |
| "id": "upi_stuffed_mamba", | |
| "category": "D", | |
| "primary_formula": "[X_{\\text{ssm}}, Z] = X W_{\\text{in}} \\in \\mathbb{R}^{B \\times L \\times 2 D}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "vig_attention": { | |
| "id": "vig_attention", | |
| "category": "B", | |
| "primary_formula": "\\text{Total MACs} \\approx 4 B N D^2 + 4 B N D d_k", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "vspec": { | |
| "id": "vspec", | |
| "category": "E", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "weighted_ensemble": { | |
| "id": "weighted_ensemble", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "wgqa_attention": { | |
| "id": "wgqa_attention", | |
| "category": "A", | |
| "primary_formula": "c^{KV}_t = W^{DKV} x_t, \\quad k^C_t = W^{UK} c^{KV}_t, \\quad v^C_t = W^{UV} c^{KV}_t", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "xattention": { | |
| "id": "xattention", | |
| "category": "E", | |
| "primary_formula": "y = \\sum_{i=1}^K G(x)_i \\cdot E_i(x), \\quad G(x) = \\text{Softmax}(\\text{TopK}(x W_g, K=2))", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "zeros_attention": { | |
| "id": "zeros_attention", | |
| "category": "E", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "a3_compressor": { | |
| "id": "a3_compressor", | |
| "category": "E", | |
| "primary_formula": "\\text{FLOPs}_{\\text{dense}} = 2 \\cdot m \\cdot n \\quad \\longrightarrow \\quad \\text{FLOPs}_{\\text{factorized}} = 2 \\cdot r \\cdot (m + n)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "aa_svd_compressor": { | |
| "id": "aa_svd_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "aasvd": { | |
| "id": "aasvd", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top, \\quad \\min_{U, \\Sigma, V} \\left\\| (W - U \\Sigma V^\\top) X^\\top \\right\\|_F^2", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "adaptive_compressor": { | |
| "id": "adaptive_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "adaptive_ranks": { | |
| "id": "adaptive_ranks", | |
| "category": "E", | |
| "primary_formula": "\\\\frac{\\\\sum_{i=1}^r \\\\sigma_i^2}{\\\\sum_{j=1}^d \\\\sigma_j^2} \\\\ge \\\\tau \\\\quad (\\\\tau = 0.92)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "adasvd_compressor": { | |
| "id": "adasvd_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "admm_compressor": { | |
| "id": "admm_compressor", | |
| "category": "E", | |
| "primary_formula": "\\tau = \\text{TopK}_{k} \\left( \\{ |W_{i,j}| \\}_{i=1, j=1}^{N, M} \\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "admm_pruner": { | |
| "id": "admm_pruner", | |
| "category": "G", | |
| "primary_formula": "\\min_{W, Z} \\frac{1}{2} \\|W - W_0\\|_F^2 + g(Z) \\quad \\text{subject to} \\quad W - Z = 0", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "armor_pruner": { | |
| "id": "armor_pruner", | |
| "category": "E", | |
| "primary_formula": "W = W_{\\text{sparse}} + R", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "ast_pruner": { | |
| "id": "ast_pruner", | |
| "category": "G", | |
| "primary_formula": "\\Sigma_X = Q \\Lambda Q^\\top, \\quad Q^\\top Q = I_{d_{\\text{in}}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "astro_quantizer": { | |
| "id": "astro_quantizer", | |
| "category": "H", | |
| "primary_formula": "W = S \\odot (Q_W - Z), \\quad Q_W \\in \\{-8, \\dots, 7\\}, \\quad S = \\text{scale}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "bonsai_pruner": { | |
| "id": "bonsai_pruner", | |
| "category": "G", | |
| "primary_formula": "s_t = s_{\\text{target}} \\cdot \\left( 1 - \\left( 1 - \\frac{t}{T} \\right)^3 \\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "cachenotes_compressor": { | |
| "id": "cachenotes_compressor", | |
| "category": "E", | |
| "primary_formula": "W_{\\text{cpt}} = \\operatorname{diag}(I_{\\text{task}}) W \\in \\mathbb{R}^{d_{\\text{out}} \\times d_{\\text{in}}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "come": { | |
| "id": "come", | |
| "category": "A", | |
| "primary_formula": "M_{\\text{compressed}} = X W_{\\text{bridge}}, \\quad W_{\\text{bridge}} = U_r \\Sigma_r V_r^\\top", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "compot_compressor": { | |
| "id": "compot_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "coopq_quantizer": { | |
| "id": "coopq_quantizer", | |
| "category": "H", | |
| "primary_formula": "b_l \\\\ge b_{\\\\min} = 4, \\\\quad \\\\forall l \\\\in \\\\{1, \\\\dots, L\\\\}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "cr_compressor": { | |
| "id": "cr_compressor", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "cur_compressor": { | |
| "id": "cur_compressor", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "cur_linear": { | |
| "id": "cur_linear", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "cur_moe": { | |
| "id": "cur_moe", | |
| "category": "F", | |
| "primary_formula": "y = \\sum_{i=1}^K G(x)_i \\cdot E_i(x), \\quad G(x) = \\text{Softmax}(\\text{TopK}(x W_g, K=2))", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Router Gating", | |
| "formula": "G(x) = \text{Softmax}(\text{TopK}(x W_g, k))", | |
| "flops": 12288 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Expert Execution", | |
| "formula": "Y = \\sum_{e \\in \text{TopK}} G(x)_e E_e(x)", | |
| "flops": 2359296 | |
| } | |
| ], | |
| "total_flops_per_token": 2371584 | |
| }, | |
| "cur_moe_compressor": { | |
| "id": "cur_moe_compressor", | |
| "category": "F", | |
| "primary_formula": "W \\approx C \\cdot U \\cdot R, \\quad C = W_{[:, J]}, \\; R = W_{[I, :]}, \\; U = C^\\dagger W R^\\dagger", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Router Gating", | |
| "formula": "G(x) = \text{Softmax}(\text{TopK}(x W_g, k))", | |
| "flops": 12288 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Expert Execution", | |
| "formula": "Y = \\sum_{e \\in \text{TopK}} G(x)_e E_e(x)", | |
| "flops": 2359296 | |
| } | |
| ], | |
| "total_flops_per_token": 2371584 | |
| }, | |
| "cure": { | |
| "id": "cure", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "delta_kv": { | |
| "id": "delta_kv", | |
| "category": "A", | |
| "primary_formula": "R_K \\in \\mathbb{R}^{B \\times H \\times 1 \\times M \\times D}, \\quad R_V \\in \\mathbb{R}^{B \\times H \\times 1 \\times M \\times D}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "direct_svd": { | |
| "id": "direct_svd", | |
| "category": "E", | |
| "primary_formula": "\\min_{\\operatorname{rank}(\\hat{W}) \\le r} \\|W - \\hat{W}\\|_F = \\sqrt{\\sum_{i=r+1}^{\\min(d_{\\text{in}}, d_{\\text{out}})} \\sigma_i^2}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "dobi": { | |
| "id": "dobi", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "dobi_svd": { | |
| "id": "dobi_svd", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "dynamic_entropy": { | |
| "id": "dynamic_entropy", | |
| "category": "E", | |
| "primary_formula": "\\widetilde{W} = W C_X^{1/2} = U \\Sigma V^\\top", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "dynamic_rank": { | |
| "id": "dynamic_rank", | |
| "category": "E", | |
| "primary_formula": "H(\\Sigma_l) = -\\sum_{i=1}^d p_i \\ln(p_i), \\quad p_i = \\frac{\\sigma_i}{\\sum_{j=1}^d \\sigma_j}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "enlighten_compressor": { | |
| "id": "enlighten_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "entropy_budget_allocation": { | |
| "id": "entropy_budget_allocation", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "erc_svd": { | |
| "id": "erc_svd", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "faq_quantizer": { | |
| "id": "faq_quantizer", | |
| "category": "H", | |
| "primary_formula": "W = S \\odot (Q_W - Z), \\quad Q_W \\in \\{-8, \\dots, 7\\}, \\quad S = \\text{scale}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "flap": { | |
| "id": "flap", | |
| "category": "G", | |
| "primary_formula": "I(i) = \\frac{1}{N} \\sum_{t=1}^N |X_{t, i} - \\bar{X}_i| \\cdot \\|W_{:, i}\\|_2", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "flat_llm": { | |
| "id": "flat_llm", | |
| "category": "G", | |
| "primary_formula": "\\min_{W_r} \\text{Tr}((W - W_r) R_{XX} (W - W_r)^\\top)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "frandi": { | |
| "id": "frandi", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "frandi_compressor": { | |
| "id": "frandi_compressor", | |
| "category": "B", | |
| "primary_formula": "\\Omega \\sim \\mathcal{N}(0, I), \\; Y = (W W^\\top)^q W \\Omega, \\; Q = \\text{qr}(Y), \\; W \\approx Q (Q^\\top W)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "gfws_d_compressor": { | |
| "id": "gfws_d_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "grecodec": { | |
| "id": "grecodec", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "grecodec_compressor": { | |
| "id": "grecodec_compressor", | |
| "category": "E", | |
| "primary_formula": "\\min_{u, v} \\|W - u v^\\top\\|_F^2 \\implies u = \\frac{W v}{\\|v\\|^2}, \\; v = \\frac{W^\\top u}{\\|u\\|^2}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "hashed_linear": { | |
| "id": "hashed_linear", | |
| "category": "E", | |
| "primary_formula": "h(i, j) = \\left( i \\cdot P_1 + j \\cdot P_2 \\right) \\pmod B", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "hashed_net_compressor": { | |
| "id": "hashed_net_compressor", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "huffman_coder": { | |
| "id": "huffman_coder", | |
| "category": "E", | |
| "primary_formula": "f(a_k) = \\sum_{i=1}^N \\mathbb{I}(w_i = a_k), \\quad p(a_k) = \\frac{f(a_k)}{N}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "hybrid_svd_layer": { | |
| "id": "hybrid_svd_layer", | |
| "category": "E", | |
| "primary_formula": "W = U_r \\Sigma_r V_r^\\top + M_{\\text{sparse}}, \\quad \\|M_{\\text{sparse}}\\|_0 \\le k", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "hybridkv": { | |
| "id": "hybridkv", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "hybridkv_compressor": { | |
| "id": "hybridkv_compressor", | |
| "category": "I", | |
| "primary_formula": "\\sigma_i^2 = \\text{Var}_j(W_{i, j}) = \\frac{1}{D_{\\text{in}}} \\sum_{j=1}^{D_{\\text{in}}} \\left( W_{i, j} - \\mu_i \\right)^2 \\quad \\text{for } i \\in \\{1, \\dots, D_{\\text{out}}\\}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "kmeans_quantize": { | |
| "id": "kmeans_quantize", | |
| "category": "H", | |
| "primary_formula": "W = S \\odot (Q_W - Z), \\quad Q_W \\in \\{-8, \\dots, 7\\}, \\quad S = \\text{scale}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "kronecker_compressor": { | |
| "id": "kronecker_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "kronecker_linear": { | |
| "id": "kronecker_linear", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "kronecker_tt_lora": { | |
| "id": "kronecker_tt_lora", | |
| "category": "E", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "kvtc": { | |
| "id": "kvtc", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "kvtc_compressor": { | |
| "id": "kvtc_compressor", | |
| "category": "H", | |
| "primary_formula": "K_{\\text{comp}} = K V_K^\\top, \\; V_{\\text{comp}} = V V_V^\\top, \\quad V_K, V_V \\in \\mathbb{R}^{r \\times d_k}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "liftr": { | |
| "id": "liftr", | |
| "category": "A", | |
| "primary_formula": "Y = W \\Omega, \\quad Q = \\text{qr}(Y)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "lillama_compressor": { | |
| "id": "lillama_compressor", | |
| "category": "E", | |
| "primary_formula": "C_{YX} = \\frac{1}{N} Y^\\top X = \\frac{1}{N} \\left( X W^\\top + \\mathbf{1}_N b^\\top \\right)^\\top X = W \\left( \\frac{1}{N} X^\\top X \\right) + b \\left( \\frac{1}{N} \\mathbf{1}_N^\\top X \\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "lsc_compressor": { | |
| "id": "lsc_compressor", | |
| "category": "E", | |
| "primary_formula": "\\tilde{W}_{u,v} = \\sum_{n=0}^{N-1} \\sum_{m=0}^{M-1} W_{nm} \\cos\\left(\\frac{\\pi u (2n+1)}{2N}\\right) \\cos\\left(\\frac{\\pi v (2m+1)}{2M}\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "mamba_shedder": { | |
| "id": "mamba_shedder", | |
| "category": "D", | |
| "primary_formula": "s_j = \\sqrt{ \\frac{1}{N_{\\text{tokens}}} \\sum_{i=1}^{N_{\\text{tokens}}} X_{i, j}^2 } \\quad \\text{for } j \\in \\{1, \\dots, D_{\\text{in}}\\}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Forget Gate", | |
| "formula": "f_t = \\exp(-\text{softplus}(W_f x_t))", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Input Gate", | |
| "formula": "i_t = \\sigma(W_i x_t)", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Projection", | |
| "formula": "v_t = W_v x_t", | |
| "flops": 1179648 | |
| }, | |
| { | |
| "id": 4, | |
| "name": "Recurrent State Update", | |
| "formula": "h_t = f_t \\odot h_{t-1} + (1 - f_t) \\odot i_t \\odot v_t", | |
| "flops": 3072 | |
| } | |
| ], | |
| "total_flops_per_token": 3542016 | |
| }, | |
| "mgie_svd": { | |
| "id": "mgie_svd", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "mgie_svd_compressor": { | |
| "id": "mgie_svd_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "mlorq_compressor": { | |
| "id": "mlorq_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx \\text{Quant}(A) \\cdot B^\\top + S \\cdot \\text{scale}, \\quad A \\in \\mathbb{R}^{d_{out} \\times r}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "modegpt": { | |
| "id": "modegpt", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "modegpt_compressor": { | |
| "id": "modegpt_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "nli_softmax": { | |
| "id": "nli_softmax", | |
| "category": "H", | |
| "primary_formula": "W' = Q_2^\\top W Q_1, \\quad W_{sliced} = W'[:d'_1, :d'_2], \\quad Q^\\top Q = I", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "nmf_compressor": { | |
| "id": "nmf_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "oats": { | |
| "id": "oats", | |
| "category": "B", | |
| "primary_formula": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "oats_compressor": { | |
| "id": "oats_compressor", | |
| "category": "E", | |
| "primary_formula": "\\min_{\\text{rank}(\\hat{W}) \\le r} \\|(W - \\hat{W}) L\\|_F^2", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "optimal_subspace_projection": { | |
| "id": "optimal_subspace_projection", | |
| "category": "E", | |
| "primary_formula": "C = \\\\frac{1}{N} X^\\\\top X + \\\\epsilon I", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "padre_attention": { | |
| "id": "padre_attention", | |
| "category": "B", | |
| "primary_formula": "c^{KV}_t = W^{DKV} x_t, \\quad k^C_t = W^{UK} c^{KV}_t, \\quad v^C_t = W^{UV} c^{KV}_t", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "proxsparse": { | |
| "id": "proxsparse", | |
| "category": "E", | |
| "primary_formula": "W = U_r \\Sigma_r V_r^\\top + S", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "proxsparse_pruner": { | |
| "id": "proxsparse_pruner", | |
| "category": "G", | |
| "primary_formula": "W^{(t+1)} = \\text{prox}_{\\lambda \\|\\cdot\\|_1}\\left(W^{(t)} - \\eta \\nabla f(W^{(t)})\\right) = \\text{sign}(W) \\max(0, |W| - \\lambda \\eta)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "pruning_utils": { | |
| "id": "pruning_utils", | |
| "category": "G", | |
| "primary_formula": "S_{i, j} = |W_{i, j}| \\cdot \\|X_{:, j}\\|_2", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "replaceme_compressor": { | |
| "id": "replaceme_compressor", | |
| "category": "E", | |
| "primary_formula": "\\text{CKA}(X_l, X_{l+1}) = \\frac{\\|X_l^\\top X_{l+1}\\|_F^2}{\\|X_l^\\top X_l\\|_F \\|X_{l+1}^\\top X_{l+1}\\|_F}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "roast_hashed_linear": { | |
| "id": "roast_hashed_linear", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "rotational_slicegpt_int4": { | |
| "id": "rotational_slicegpt_int4", | |
| "category": "H", | |
| "primary_formula": "\\\\widetilde{W} = W Q, \\\\quad \\\\widetilde{X} = Q^\\\\top X, \\\\quad Q^\\\\top Q = I", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "rpca": { | |
| "id": "rpca", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "rpca_compressor": { | |
| "id": "rpca_compressor", | |
| "category": "E", | |
| "primary_formula": "\\min_{L, S} \\|L\\|_* + \\lambda \\|S\\|_1 \\quad \\text{s.t.} \\quad W = L + S, \\quad \\text{rank}(L) \\le r", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "saes_svd": { | |
| "id": "saes_svd", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "saes_svd_compressor": { | |
| "id": "saes_svd_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "saes_svd_v2": { | |
| "id": "saes_svd_v2", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top + \\text{SAE}(W - U_r \\Sigma_r V_r^\\top)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "salr": { | |
| "id": "salr", | |
| "category": "E", | |
| "primary_formula": "\\min_{\\text{rank}(\\widehat{W}) \\le r} \\|W - \\widehat{W}\\|_F^2 = \\sum_{i=r+1}^{D} \\sigma_i^2", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "sdrm": { | |
| "id": "sdrm", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "sdrm_compressor": { | |
| "id": "sdrm_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "slicegpt": { | |
| "id": "slicegpt", | |
| "category": "A", | |
| "primary_formula": "y = W x + b = W (I_{d_{\\text{in}}}) x + b = W (Q Q^\\top) x + b = (W Q) (Q^\\top x) + b = \\widetilde{W} \\widetilde{x} + b", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "slicegpt_compressor": { | |
| "id": "slicegpt_compressor", | |
| "category": "A", | |
| "primary_formula": "W_{\\text{pruned}} = (Q^\\top W Q)_{[:, \\mathcal{I}]}, \\quad Q = \\text{eigh}(X X^\\top), \\; |\\mathcal{I}| = (1 - 0.45) d_{in}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "slicegpt_pca": { | |
| "id": "slicegpt_pca", | |
| "category": "E", | |
| "primary_formula": "\\Sigma = \\frac{1}{N - 1} X_c^\\top X_c + \\epsilon I_{d_{\\text{in}}}", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "slim_compressor": { | |
| "id": "slim_compressor", | |
| "category": "E", | |
| "primary_formula": "W_q = \\\\operatorname{Quant}_b(W_0)", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "soka": { | |
| "id": "soka", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "soka_compressor": { | |
| "id": "soka_compressor", | |
| "category": "E", | |
| "primary_formula": "W = U \\\\Sigma V^\\\\top \\\\approx U_r \\\\Sigma_r V_r^\\\\top = A B^\\\\top", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "sparsegpt": { | |
| "id": "sparsegpt", | |
| "category": "E", | |
| "primary_formula": "\\Delta W = -\\frac{w_q}{[H^{-1}]_{qq}} H_{:, q}^{-1}, \\quad H = 2 X X^\\top", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "spectraformer_attention": { | |
| "id": "spectraformer_attention", | |
| "category": "B", | |
| "primary_formula": "c^{KV}_t = W^{DKV} x_t, \\quad k^C_t = W^{UK} c^{KV}_t, \\quad v^C_t = W^{UV} c^{KV}_t", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "subspace_projection": { | |
| "id": "subspace_projection", | |
| "category": "E", | |
| "primary_formula": "\\min_{P, \\{C_k\\}} \\sum_{k=1}^K \\|W_k - P C_k\\|_F^2 \\quad \\text{s.t.} \\quad P^\\top P = I", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "svd_linear": { | |
| "id": "svd_linear", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = (U_r \\sqrt{\\Sigma_r}) (\\sqrt{\\Sigma_r} V_r^\\top) = A B^\\top", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "swift_svd": { | |
| "id": "swift_svd", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "swift_svd_act": { | |
| "id": "swift_svd_act", | |
| "category": "E", | |
| "primary_formula": "\\min_{A, B} \\left\\|(W - A B) \\Sigma_{XX}^{1/2}\\right\\|_F^2, \\quad \\Sigma_{XX} = \\frac{1}{N} X X^\\top", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "swift_svd_compressor": { | |
| "id": "swift_svd_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "tlinformer_attention": { | |
| "id": "tlinformer_attention", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "tomoe": { | |
| "id": "tomoe", | |
| "category": "F", | |
| "primary_formula": "y = \\sum_{i=1}^K G(x)_i \\cdot E_i(x), \\quad G(x) = \\text{Softmax}(\\text{TopK}(x W_g, K=2))", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Router Gating", | |
| "formula": "G(x) = \text{Softmax}(\text{TopK}(x W_g, k))", | |
| "flops": 12288 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Expert Execution", | |
| "formula": "Y = \\sum_{e \\in \text{TopK}} G(x)_e E_e(x)", | |
| "flops": 2359296 | |
| } | |
| ], | |
| "total_flops_per_token": 2371584 | |
| }, | |
| "tomoe_compressor": { | |
| "id": "tomoe_compressor", | |
| "category": "F", | |
| "primary_formula": "y = \\sum_{i=1}^K G(x)_i \\cdot E_i(x), \\quad G(x) = \\text{Softmax}(\\text{TopK}(x W_g, K=2))", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Router Gating", | |
| "formula": "G(x) = \text{Softmax}(\text{TopK}(x W_g, k))", | |
| "flops": 12288 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Expert Execution", | |
| "formula": "Y = \\sum_{e \\in \text{TopK}} G(x)_e E_e(x)", | |
| "flops": 2359296 | |
| } | |
| ], | |
| "total_flops_per_token": 2371584 | |
| }, | |
| "tt_compressor": { | |
| "id": "tt_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "tt_linear": { | |
| "id": "tt_linear", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "uniql_compressor": { | |
| "id": "uniql_compressor", | |
| "category": "E", | |
| "primary_formula": "W \\approx U_r \\Sigma_r V_r^\\top = A B^\\top, \\quad r \\ll \\min(d_{in}, d_{out})", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "Low-Rank V-Factor", | |
| "formula": "h = X V_r^\top, \\quad V_r \\in \\mathbb{R}^{r \times d_{in}}", | |
| "flops": 98304 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Low-Rank U-Factor", | |
| "formula": "Y = h U_r^\top, \\quad U_r \\in \\mathbb{R}^{d_{out} \times r}", | |
| "flops": 98304 | |
| } | |
| ], | |
| "total_flops_per_token": 196608 | |
| }, | |
| "zerokv_neo": { | |
| "id": "zerokv_neo", | |
| "category": "A", | |
| "primary_formula": "c^{KV}_t = W^{DKV} x_t, \\quad k^C_t = W^{UK} c^{KV}_t, \\quad v^C_t = W^{UV} c^{KV}_t", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| }, | |
| "zerokv_neo_compressor": { | |
| "id": "zerokv_neo_compressor", | |
| "category": "A", | |
| "primary_formula": "c^{KV}_t = W^{DKV} x_t, \\quad k^C_t = W^{UK} c^{KV}_t, \\quad v^C_t = W^{UV} c^{KV}_t", | |
| "steps": [ | |
| { | |
| "id": 1, | |
| "name": "QKV Projections", | |
| "formula": "Q, K, V = X W_Q, X W_K, X W_V", | |
| "flops": 3538944 | |
| }, | |
| { | |
| "id": 2, | |
| "name": "Attention Materialization", | |
| "formula": "A = \text{Softmax}(Q K^\top / \\sqrt{d_k})", | |
| "flops": 786432 | |
| }, | |
| { | |
| "id": 3, | |
| "name": "Value Aggregation", | |
| "formula": "O = A V", | |
| "flops": 786432 | |
| } | |
| ], | |
| "total_flops_per_token": 5111808 | |
| } | |
| } |
Xet Storage Details
- Size:
- 169 kB
- Xet hash:
- fce7b5ae6c62a64ba2e0f70001e74f5f1158cfa0fd514c01ee25a5e072c32544
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.