| { | |
| "id": "gqa", | |
| "name": "Gqa", | |
| "file": "Codes/framework/attentions/gqa.py", | |
| "type": "Attention Module", | |
| "category": "Dynamic & Efficient Attention", | |
| "paradigm": "sparse", | |
| "icon": "\ud83c\udf1f", | |
| "description": "Production-grade Dynamic & Efficient Attention module.", | |
| "classes": [], | |
| "methods_count": 0, | |
| "math_latex": "K_{\\text{shared}} = \\text{repeat\\_interleave}(K, H_Q / H_{KV}, \\text{dim}=\\text{heads})", | |
| "components": [ | |
| { | |
| "id": "sparse_mask", | |
| "name": "Dynamic Top-K Attention Sparsity Mask", | |
| "role": "O(N sqrt{N}) Token Filtering", | |
| "shape_transform": "[Seq \u00d7 Seq] Sparse", | |
| "color": "#38bdf8", | |
| "math_latex": "M_{ij} = \\mathbb{I}(|S_{ij}| \\ge \\tau), \\quad S_{ij} = \\frac{q_i k_j^\\top}{\\sqrt{d}}" | |
| }, | |
| { | |
| "id": "kernel_attn", | |
| "name": "Linearized Positive Kernel Projection", | |
| "role": "Softmax Approximation", | |
| "shape_transform": "[\\phi(Q) \\cdot \\phi(K)^T]", | |
| "color": "#7dd3fc", | |
| "math_latex": "\\text{Attn}(Q,K,V) = \\frac{\\phi(Q) (\\phi(K)^\\top V)}{\\phi(Q) \\sum_j \\phi(K_j)^\\top}" | |
| }, | |
| { | |
| "id": "out_norm", | |
| "name": "Output Layer Normalization", | |
| "role": "Attention Scale Stabilization", | |
| "shape_transform": "[d_{in}]", | |
| "color": "#bae6fd", | |
| "math_latex": "\\text{LN}(y) = \\frac{y - \\mu}{\\sqrt{\\sigma^2 + \\epsilon}} \\odot \\gamma + \\beta" | |
| } | |
| ], | |
| "model_projections": { | |
| "distilbert": { | |
| "hidden_size": 768, | |
| "intermediate_size": 3072, | |
| "baseline_layer_params": 6488064, | |
| "compressed_layer_params": 3568435, | |
| "savings_ratio": 1.82 | |
| }, | |
| "roberta": { | |
| "hidden_size": 768, | |
| "intermediate_size": 3072, | |
| "baseline_layer_params": 6488064, | |
| "compressed_layer_params": 3568435, | |
| "savings_ratio": 1.82 | |
| }, | |
| "gpt2": { | |
| "hidden_size": 768, | |
| "intermediate_size": 3072, | |
| "baseline_layer_params": 6488064, | |
| "compressed_layer_params": 3568435, | |
| "savings_ratio": 1.82 | |
| }, | |
| "qwen": { | |
| "hidden_size": 896, | |
| "intermediate_size": 4864, | |
| "baseline_layer_params": 11124736, | |
| "compressed_layer_params": 6118604, | |
| "savings_ratio": 1.82 | |
| } | |
| } | |
| } |
Xet Storage Details
- Size:
- 2.22 kB
- Xet hash:
- 2f8e6be8a4182157198d631d17dff3d0747c06f10bb9ab14a211d3a11f273abd
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.