| { | |
| "id": "adaptive_span_inject", | |
| "name": "Deprecated stub. Re-exports base module for registry compatibility.", | |
| "file": "Codes/framework/attentions/adaptive_span_inject.py", | |
| "type": "Attention Module", | |
| "category": "Dynamic & Efficient Attention", | |
| "paradigm": "sparse", | |
| "icon": "\ud83c\udf1f", | |
| "description": "Deprecated stub. Re-exports base module for registry compatibility.", | |
| "classes": [], | |
| "methods_count": 0, | |
| "math_latex": "S_{ij} = \\frac{Q_i K_j^\\top}{\\sqrt{d}}", | |
| "components": [ | |
| { | |
| "id": "sparse_mask", | |
| "name": "Dynamic Top-K Attention Sparsity Mask", | |
| "role": "O(N sqrt{N}) Token Filtering", | |
| "shape_transform": "[Seq \u00d7 Seq] Sparse", | |
| "color": "#38bdf8", | |
| "math_latex": "M_{ij} = \\mathbb{I}(|S_{ij}| \\ge \\tau), \\quad S_{ij} = \\frac{q_i k_j^\\top}{\\sqrt{d}}" | |
| }, | |
| { | |
| "id": "kernel_attn", | |
| "name": "Linearized Positive Kernel Projection", | |
| "role": "Softmax Approximation", | |
| "shape_transform": "[\\phi(Q) \\cdot \\phi(K)^T]", | |
| "color": "#7dd3fc", | |
| "math_latex": "\\text{Attn}(Q,K,V) = \\frac{\\phi(Q) (\\phi(K)^\\top V)}{\\phi(Q) \\sum_j \\phi(K_j)^\\top}" | |
| }, | |
| { | |
| "id": "out_norm", | |
| "name": "Output Layer Normalization", | |
| "role": "Attention Scale Stabilization", | |
| "shape_transform": "[d_{in}]", | |
| "color": "#bae6fd", | |
| "math_latex": "\\text{LN}(y) = \\frac{y - \\mu}{\\sqrt{\\sigma^2 + \\epsilon}} \\odot \\gamma + \\beta" | |
| } | |
| ], | |
| "model_projections": { | |
| "distilbert": { | |
| "hidden_size": 768, | |
| "intermediate_size": 3072, | |
| "baseline_layer_params": 6488064, | |
| "compressed_layer_params": 3568435, | |
| "savings_ratio": 1.82 | |
| }, | |
| "roberta": { | |
| "hidden_size": 768, | |
| "intermediate_size": 3072, | |
| "baseline_layer_params": 6488064, | |
| "compressed_layer_params": 3568435, | |
| "savings_ratio": 1.82 | |
| }, | |
| "gpt2": { | |
| "hidden_size": 768, | |
| "intermediate_size": 3072, | |
| "baseline_layer_params": 6488064, | |
| "compressed_layer_params": 3568435, | |
| "savings_ratio": 1.82 | |
| }, | |
| "qwen": { | |
| "hidden_size": 896, | |
| "intermediate_size": 4864, | |
| "baseline_layer_params": 11124736, | |
| "compressed_layer_params": 6118604, | |
| "savings_ratio": 1.82 | |
| } | |
| } | |
| } |
Xet Storage Details
- Size:
- 2.28 kB
- Xet hash:
- 19a2c39b8bb1cdaff6a089a188e2b9012c108e5c8bb91f3327a3847d0b6015ef
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.