DoL-1.5-Flash / Model /omvt /compressor.py
GABIRI's picture
Add model implementation source
af0f593 verified
Raw History Blame Contribute Delete
2.72 kB
# -*- coding: utf-8 -*-
"""Perceiver-style compressor for OMVT."""
from __future__ import annotations
import torch
import torch.nn as nn
import torch.nn.functional as F
class _CrossAttn(nn.Module):
def __init__(self, d_model: int, n_heads: int) -> None:
super().__init__()
if d_model % n_heads != 0:
raise ValueError("d_model must be divisible by n_heads")
self.n_heads = n_heads
self.head_dim = d_model // n_heads
self.q_proj = nn.Linear(d_model, d_model)
self.kv_proj = nn.Linear(d_model, d_model * 2)
self.out = nn.Linear(d_model, d_model)
self.norm_q = nn.LayerNorm(d_model)
self.norm_kv = nn.LayerNorm(d_model)
def forward(self, latents: torch.Tensor, ctx: torch.Tensor) -> torch.Tensor:
bsz, n_lat, d = latents.shape
q = self.q_proj(self.norm_q(latents))
kv = self.kv_proj(self.norm_kv(ctx))
k, v = kv.chunk(2, dim=-1)
q = q.reshape(bsz, n_lat, self.n_heads, self.head_dim).transpose(1, 2)
k = k.reshape(bsz, ctx.shape[1], self.n_heads, self.head_dim).transpose(1, 2)
v = v.reshape(bsz, ctx.shape[1], self.n_heads, self.head_dim).transpose(1, 2)
attn = F.scaled_dot_product_attention(q, k, v)
attn = attn.transpose(1, 2).reshape(bsz, n_lat, d)
return self.out(attn)
class PerceiverCompressor(nn.Module):
def __init__(self, omvt_cfg) -> None:
super().__init__()
self.cfg = omvt_cfg
self.latents = nn.Parameter(
torch.randn(omvt_cfg.compress_to, omvt_cfg.d_vision) * 0.02
)
self.blocks = nn.ModuleList()
for _ in range(omvt_cfg.compressor_layers):
self.blocks.append(
nn.ModuleDict(
{
"cross": _CrossAttn(omvt_cfg.d_vision, omvt_cfg.compressor_heads),
"ffn": nn.Sequential(
nn.LayerNorm(omvt_cfg.d_vision),
nn.Linear(omvt_cfg.d_vision, omvt_cfg.vision_ffn_hidden),
nn.GELU(),
nn.Linear(omvt_cfg.vision_ffn_hidden, omvt_cfg.d_vision),
),
}
)
)
self.final_norm = nn.LayerNorm(omvt_cfg.d_vision)
def forward(self, patches: torch.Tensor) -> torch.Tensor:
bsz = patches.shape[0]
latents = self.latents.unsqueeze(0).expand(bsz, -1, -1)
for block in self.blocks:
latents = latents + block["cross"](latents, patches)
latents = latents + block["ffn"](latents)
return self.final_norm(latents)
__all__ = ["PerceiverCompressor"]