projector.py

import json
from pathlib import Path

import torch
from huggingface_hub import hf_hub_download
from safetensors.torch import load_file
from torch import nn


class Projector(nn.Module):
    def __init__(self, config: dict):
        super().__init__()
        self.spatial_merge_size = 2
        self.linear_1 = nn.Linear(config["input_size"], config["hidden_size"], bias=True)
        self.linear_2 = nn.Linear(config["hidden_size"], config["output_size"], bias=True)
        self.merge_linear_1 = nn.Linear(config["merged_hidden_size"], config["hidden_size"], bias=True)
        self.merge_linear_2 = nn.Linear(config["hidden_size"], config["output_size"], bias=True)

    def forward(self, image_features: torch.Tensor) -> torch.Tensor:
        hidden = torch.nn.functional.gelu(self.linear_1(image_features))
        hidden = self.linear_2(hidden)
        hidden = hidden.reshape(hidden.shape[0] // (self.spatial_merge_size ** 2), -1)
        return self.merge_linear_2(torch.nn.functional.gelu(self.merge_linear_1(hidden)))


def load_projector(model: str | Path) -> Projector:
    path = Path(model)
    if path.is_dir():
        config_path = path / "projector_config.json"
        weights_path = path / "projector.safetensors"
    else:
        config_path = Path(hf_hub_download(str(model), "projector_config.json"))
        weights_path = Path(hf_hub_download(str(model), "projector.safetensors"))
    projector = Projector(json.loads(config_path.read_text()))
    projector.load_state_dict(load_file(weights_path))
    return projector