Download Model/omvt/compressor.py from Montlok/DoL-1.5-Flash: direct link, hf CLI and curl.
- Browser
- Download file 2.72 kB
-
https://huggingface.co/Montlok/DoL-1.5-Flash/resolve/main/Model/omvt/compressor.py
- Command line
-
hf download hf://Montlok/DoL-1.5-Flash/Model/omvt/compressor.py
-
curl -L -o compressor.py https://huggingface.co/Montlok/DoL-1.5-Flash/resolve/main/Model/omvt/compressor.py
2.72 kB
| # -*- coding: utf-8 -*- | |
| """Perceiver-style compressor for OMVT.""" | |
| from __future__ import annotations | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| class _CrossAttn(nn.Module): | |
| def __init__(self, d_model: int, n_heads: int) -> None: | |
| super().__init__() | |
| if d_model % n_heads != 0: | |
| raise ValueError("d_model must be divisible by n_heads") | |
| self.n_heads = n_heads | |
| self.head_dim = d_model // n_heads | |
| self.q_proj = nn.Linear(d_model, d_model) | |
| self.kv_proj = nn.Linear(d_model, d_model * 2) | |
| self.out = nn.Linear(d_model, d_model) | |
| self.norm_q = nn.LayerNorm(d_model) | |
| self.norm_kv = nn.LayerNorm(d_model) | |
| def forward(self, latents: torch.Tensor, ctx: torch.Tensor) -> torch.Tensor: | |
| bsz, n_lat, d = latents.shape | |
| q = self.q_proj(self.norm_q(latents)) | |
| kv = self.kv_proj(self.norm_kv(ctx)) | |
| k, v = kv.chunk(2, dim=-1) | |
| q = q.reshape(bsz, n_lat, self.n_heads, self.head_dim).transpose(1, 2) | |
| k = k.reshape(bsz, ctx.shape[1], self.n_heads, self.head_dim).transpose(1, 2) | |
| v = v.reshape(bsz, ctx.shape[1], self.n_heads, self.head_dim).transpose(1, 2) | |
| attn = F.scaled_dot_product_attention(q, k, v) | |
| attn = attn.transpose(1, 2).reshape(bsz, n_lat, d) | |
| return self.out(attn) | |
| class PerceiverCompressor(nn.Module): | |
| def __init__(self, omvt_cfg) -> None: | |
| super().__init__() | |
| self.cfg = omvt_cfg | |
| self.latents = nn.Parameter( | |
| torch.randn(omvt_cfg.compress_to, omvt_cfg.d_vision) * 0.02 | |
| ) | |
| self.blocks = nn.ModuleList() | |
| for _ in range(omvt_cfg.compressor_layers): | |
| self.blocks.append( | |
| nn.ModuleDict( | |
| { | |
| "cross": _CrossAttn(omvt_cfg.d_vision, omvt_cfg.compressor_heads), | |
| "ffn": nn.Sequential( | |
| nn.LayerNorm(omvt_cfg.d_vision), | |
| nn.Linear(omvt_cfg.d_vision, omvt_cfg.vision_ffn_hidden), | |
| nn.GELU(), | |
| nn.Linear(omvt_cfg.vision_ffn_hidden, omvt_cfg.d_vision), | |
| ), | |
| } | |
| ) | |
| ) | |
| self.final_norm = nn.LayerNorm(omvt_cfg.d_vision) | |
| def forward(self, patches: torch.Tensor) -> torch.Tensor: | |
| bsz = patches.shape[0] | |
| latents = self.latents.unsqueeze(0).expand(bsz, -1, -1) | |
| for block in self.blocks: | |
| latents = latents + block["cross"](latents, patches) | |
| latents = latents + block["ffn"](latents) | |
| return self.final_norm(latents) | |
| __all__ = ["PerceiverCompressor"] | |