| { | |
| "id": "cur_compressor", | |
| "name": "CUR Compressor: replaces Linear and Conv1D layers with CUR decomposition.", | |
| "file": "Codes/framework/compression_engine/cur_compressor.py", | |
| "type": "Compression Engine", | |
| "category": "Dynamic & Efficient Attention", | |
| "paradigm": "sparse", | |
| "icon": "\ud83c\udf1f", | |
| "description": "CUR Compressor: replaces Linear and Conv1D layers with CUR decomposition. Uses leverage score sampling or energy sampling for column/row selection. Registered in COMPRESSOR_REGISTRY as 'cur' and 'cur_compressor'.", | |
| "classes": [ | |
| "CURCompressor" | |
| ], | |
| "methods_count": 6, | |
| "math_latex": "A_{ij} = \\text{TopK}\\left(\\text{Softmax}\\left(\\frac{Q_i K_j^\\top}{\\sqrt{d}}\\right)\\right)", | |
| "components": [ | |
| { | |
| "id": "sparse_mask", | |
| "name": "Dynamic Top-K Attention Sparsity Mask", | |
| "role": "O(N sqrt{N}) Token Filtering", | |
| "shape_transform": "[Seq \u00d7 Seq] Sparse", | |
| "color": "#38bdf8", | |
| "math_latex": "M_{ij} = \\mathbb{I}(|S_{ij}| \\ge \\tau), \\quad S_{ij} = \\frac{q_i k_j^\\top}{\\sqrt{d}}" | |
| }, | |
| { | |
| "id": "kernel_attn", | |
| "name": "Linearized Positive Kernel Projection", | |
| "role": "Softmax Approximation", | |
| "shape_transform": "[\\phi(Q) \\cdot \\phi(K)^T]", | |
| "color": "#7dd3fc", | |
| "math_latex": "\\text{Attn}(Q,K,V) = \\frac{\\phi(Q) (\\phi(K)^\\top V)}{\\phi(Q) \\sum_j \\phi(K_j)^\\top}" | |
| }, | |
| { | |
| "id": "out_norm", | |
| "name": "Output Layer Normalization", | |
| "role": "Attention Scale Stabilization", | |
| "shape_transform": "[d_{in}]", | |
| "color": "#bae6fd", | |
| "math_latex": "\\text{LN}(y) = \\frac{y - \\mu}{\\sqrt{\\sigma^2 + \\epsilon}} \\odot \\gamma + \\beta" | |
| } | |
| ], | |
| "model_projections": { | |
| "distilbert": { | |
| "hidden_size": 768, | |
| "intermediate_size": 3072, | |
| "baseline_layer_params": 6488064, | |
| "compressed_layer_params": 3568435, | |
| "savings_ratio": 1.82 | |
| }, | |
| "roberta": { | |
| "hidden_size": 768, | |
| "intermediate_size": 3072, | |
| "baseline_layer_params": 6488064, | |
| "compressed_layer_params": 3568435, | |
| "savings_ratio": 1.82 | |
| }, | |
| "gpt2": { | |
| "hidden_size": 768, | |
| "intermediate_size": 3072, | |
| "baseline_layer_params": 6488064, | |
| "compressed_layer_params": 3568435, | |
| "savings_ratio": 1.82 | |
| }, | |
| "qwen": { | |
| "hidden_size": 896, | |
| "intermediate_size": 4864, | |
| "baseline_layer_params": 11124736, | |
| "compressed_layer_params": 6118604, | |
| "savings_ratio": 1.82 | |
| } | |
| } | |
| } |
Xet Storage Details
- Size:
- 2.51 kB
- Xet hash:
- 5147633a79c4b7cea4a39216bc6db85699970bd5295a2c1215f6ef97dc1b5991
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.