| { | |
| "id": "fused_svd_kernel", | |
| "name": "Fused SVD-Linear Layer with SRAM-Resident RMSNorm & Dual-Projection.", | |
| "file": "Codes/framework/attentions/fused_svd_kernel.py", | |
| "type": "Attention Module", | |
| "category": "Low-Rank SVD & Factorization", | |
| "paradigm": "svd", | |
| "icon": "\u26a1", | |
| "description": "Fused SVD-Linear Layer with SRAM-Resident RMSNorm & Dual-Projection. RESEARCH FIX: Eliminates GPU High-Bandwidth Memory (HBM) roundtrip latency by fusing: 1. Down-projection: Z = X @ V^T (hidden_dim -> rank) 2. In-SRAM RMSNorm: Z_norm = Z * rsqrt(mean(Z^2) + eps) * weight 3. Up-projection: Y = Z_norm @ U^T + bias (rank -> out_dim) The intermediate activation matrix Z \\in R^{B x L", | |
| "classes": [ | |
| "_FusedSVDLinearAutogradFunction", | |
| "FusedSVDLinear", | |
| "FusedSVDKernelAttention" | |
| ], | |
| "methods_count": 23, | |
| "math_latex": "\\min_{\\text{rank}(\\widehat{W}) \\le r} \\|W - \\widehat{W}\\|_F^2 = \\sum_{i=r+1}^{D} \\sigma_i^2", | |
| "components": [ | |
| { | |
| "id": "v_factor", | |
| "name": "Factor V (Low-Rank Subspace Projection)", | |
| "role": "Projection to Rank r", | |
| "shape_transform": "[r \u00d7 d_in]", | |
| "color": "#3acbb8", | |
| "math_latex": "z_t = x_t V_r^\\top, \\quad V_r \\in \\mathbb{R}^{r \\times d_{in}}" | |
| }, | |
| { | |
| "id": "u_factor", | |
| "name": "Factor U (Full-Rank Expansion Projection)", | |
| "role": "Expansion to Output Space", | |
| "shape_transform": "[d_out \u00d7 r]", | |
| "color": "#2dd4bf", | |
| "math_latex": "y_t = z_t (\\Sigma_r U_r^\\top) + b, \\quad U_r \\in \\mathbb{R}^{d_{out} \\times r}" | |
| }, | |
| { | |
| "id": "adapter", | |
| "name": "Residual LoRA / Norm Calibration", | |
| "role": "Fidelity Compensation", | |
| "shape_transform": "[d_out \u00d7 d_in] Residual", | |
| "color": "#5eead4", | |
| "math_latex": "\\Delta y_t = x_t (B A)^\\top \\cdot \\frac{\\alpha}{r}, \\quad A \\in \\mathbb{R}^{r \\times d_{in}}, \\; B \\in \\mathbb{R}^{d_{out} \\times r}" | |
| } | |
| ], | |
| "model_projections": { | |
| "distilbert": { | |
| "hidden_size": 768, | |
| "intermediate_size": 3072, | |
| "baseline_layer_params": 6488064, | |
| "compressed_layer_params": 786432, | |
| "savings_ratio": 8.25 | |
| }, | |
| "roberta": { | |
| "hidden_size": 768, | |
| "intermediate_size": 3072, | |
| "baseline_layer_params": 6488064, | |
| "compressed_layer_params": 786432, | |
| "savings_ratio": 8.25 | |
| }, | |
| "gpt2": { | |
| "hidden_size": 768, | |
| "intermediate_size": 3072, | |
| "baseline_layer_params": 6488064, | |
| "compressed_layer_params": 786432, | |
| "savings_ratio": 8.25 | |
| }, | |
| "qwen": { | |
| "hidden_size": 896, | |
| "intermediate_size": 4864, | |
| "baseline_layer_params": 11124736, | |
| "compressed_layer_params": 1081344, | |
| "savings_ratio": 10.29 | |
| } | |
| } | |
| } |
Xet Storage Details
- Size:
- 2.77 kB
- Xet hash:
- f2c11e4bdfb35fedc394f730143f899db5ab68437f5e986f97d0036f197c0d22
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.