TEI / app /model_loader.py
Akash297's picture
Update app/model_loader.py
285ec1b verified
Raw History Blame Contribute Delete
5.81 kB
import torch
import torch.nn as nn
import numpy as np
from PIL import Image
import requests
from io import BytesIO
from torchvision import transforms, models
from transformers import DistilBertTokenizer, DistilBertModel
import cv2
import mediapipe as mp
from emoji import distinct_emoji_list
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# === Model Definition ===
class HybridMultimodalModel(nn.Module):
def __init__(self, num_classes=3):
super().__init__()
self.bert = DistilBertModel.from_pretrained("distilbert-base-uncased")
self.emoji_proj = nn.Linear(300, 768)
self.gate = nn.Sequential(nn.Linear(768 + 300, 768), nn.Sigmoid())
self.text_emoji_fc = nn.Sequential(nn.Linear(768 + 300, 256), nn.ReLU())
resnet = models.resnet18(pretrained=True)
resnet.conv1 = nn.Conv2d(4, 64, 7, 2, 3, bias=False) # for RGB + attn
self.image_model = nn.Sequential(*list(resnet.children())[:-1])
self.image_fc = nn.Linear(512, 256)
self.cross_attn = nn.MultiheadAttention(256, 4, batch_first=True)
self.fusion_proj = nn.Sequential(
nn.Linear(256, 256), nn.ReLU(), nn.Dropout(0.2), nn.Linear(256, 256)
)
self.classifier = nn.Sequential(
nn.Linear(256, 128), nn.ReLU(), nn.Dropout(0.3), nn.Linear(128, num_classes)
)
def forward(self, input_ids, attention_mask, emoji_vec, image, attn_map):
text_feat = self.bert(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state[:, 0]
emoji_proj = self.emoji_proj(emoji_vec)
alpha = self.gate(torch.cat((text_feat, emoji_vec), dim=1))
gated = alpha * emoji_proj
text_emoji_feat = self.text_emoji_fc(torch.cat((text_feat + gated, emoji_vec), dim=1))
x = torch.cat((image, attn_map), dim=1)
img_feat = self.image_model(x).squeeze()
if len(img_feat.shape) == 1:
img_feat = img_feat.unsqueeze(0)
img_feat = self.image_fc(img_feat)
multimodal = torch.stack([text_emoji_feat, img_feat], dim=1)
attn_out, _ = self.cross_attn(multimodal, multimodal, multimodal)
fused = attn_out.mean(dim=1)
fused = self.fusion_proj(fused) + fused
return self.classifier(fused)
# === Model Wrapper with Prediction ===
class EmotionModel:
def __init__(self):
self.device = device
self.tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
self.model = HybridMultimodalModel().to(self.device)
self.model.load_state_dict(torch.load("app/hybrid_multimodal_model.pth", map_location=self.device))
self.model.eval()
self.emoji_vecs = self.load_emoji_vectors("app/emoji2vec.txt")
self.transform = transforms.Compose([
transforms.Grayscale(num_output_channels=3),
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize([0.5]*3, [0.5]*3)
])
self.mp_face_mesh = mp.solutions.face_mesh.FaceMesh(static_image_mode=True)
def load_emoji_vectors(self, path):
emoji_vecs = {}
with open(path, 'r', encoding='utf-8') as f:
for line in f:
parts = line.strip().split()
if len(parts) == 301:
emoji_vecs[parts[0]] = np.array([float(x) for x in parts[1:]], dtype=np.float32)
return emoji_vecs
def get_emoji_vector(self, text):
emojis = distinct_emoji_list(text)
vecs = [self.emoji_vecs[e] for e in emojis if e in self.emoji_vecs]
return np.mean(vecs, axis=0).astype(np.float32) if vecs else np.zeros(300, dtype=np.float32)
def get_landmark_attention(self, image_np):
results = self.mp_face_mesh.process(image_np)
attention_map = np.zeros((image_np.shape[0], image_np.shape[1]), dtype=np.float32)
if results.multi_face_landmarks:
for landmark in results.multi_face_landmarks[0].landmark:
x = int(landmark.x * image_np.shape[1])
y = int(landmark.y * image_np.shape[0])
if 0 <= x < image_np.shape[1] and 0 <= y < image_np.shape[0]:
cv2.circle(attention_map, (x, y), 4, 1.0, -1)
attention_map = cv2.GaussianBlur(attention_map, (31, 31), 0)
attention_map = attention_map / np.max(attention_map) if np.max(attention_map) != 0 else attention_map
attention_map = cv2.resize(attention_map, (224, 224))
return attention_map.astype(np.float32)
def predict(self, text, image_url):
# --- Text ---
inputs = self.tokenizer(text, return_tensors='pt', truncation=True, padding='max_length', max_length=128)
input_ids = inputs['input_ids'].to(self.device)
attention_mask = inputs['attention_mask'].to(self.device)
# --- Emoji ---
emoji_vec = self.get_emoji_vector(text)
emoji_tensor = torch.tensor(emoji_vec, dtype=torch.float32).unsqueeze(0).to(self.device)
# --- Image ---
response = requests.get(image_url)
image = Image.open(BytesIO(response.content)).convert("RGB")
image_np = np.array(image)
img_tensor = self.transform(image).unsqueeze(0).to(self.device)
# --- Attention Map from MediaPipe ---
attn_map = self.get_landmark_attention(image_np)
attn_tensor = torch.tensor(attn_map, dtype=torch.float32).unsqueeze(0).unsqueeze(0).to(self.device)
# --- Final Prediction ---
with torch.no_grad():
output = self.model(input_ids, attention_mask, emoji_tensor, img_tensor, attn_tensor)
prediction = torch.argmax(output, dim=1).item()
label_map = {0: "Negative", 1: "Neutral", 2: "Positive"}
return label_map[prediction]