import torch import torch.nn as nn import numpy as np from PIL import Image import requests from io import BytesIO from torchvision import transforms, models from transformers import DistilBertTokenizer, DistilBertModel import cv2 import mediapipe as mp from emoji import distinct_emoji_list device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # === Model Definition === class HybridMultimodalModel(nn.Module): def __init__(self, num_classes=3): super().__init__() self.bert = DistilBertModel.from_pretrained("distilbert-base-uncased") self.emoji_proj = nn.Linear(300, 768) self.gate = nn.Sequential(nn.Linear(768 + 300, 768), nn.Sigmoid()) self.text_emoji_fc = nn.Sequential(nn.Linear(768 + 300, 256), nn.ReLU()) resnet = models.resnet18(pretrained=True) resnet.conv1 = nn.Conv2d(4, 64, 7, 2, 3, bias=False) # for RGB + attn self.image_model = nn.Sequential(*list(resnet.children())[:-1]) self.image_fc = nn.Linear(512, 256) self.cross_attn = nn.MultiheadAttention(256, 4, batch_first=True) self.fusion_proj = nn.Sequential( nn.Linear(256, 256), nn.ReLU(), nn.Dropout(0.2), nn.Linear(256, 256) ) self.classifier = nn.Sequential( nn.Linear(256, 128), nn.ReLU(), nn.Dropout(0.3), nn.Linear(128, num_classes) ) def forward(self, input_ids, attention_mask, emoji_vec, image, attn_map): text_feat = self.bert(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state[:, 0] emoji_proj = self.emoji_proj(emoji_vec) alpha = self.gate(torch.cat((text_feat, emoji_vec), dim=1)) gated = alpha * emoji_proj text_emoji_feat = self.text_emoji_fc(torch.cat((text_feat + gated, emoji_vec), dim=1)) x = torch.cat((image, attn_map), dim=1) img_feat = self.image_model(x).squeeze() if len(img_feat.shape) == 1: img_feat = img_feat.unsqueeze(0) img_feat = self.image_fc(img_feat) multimodal = torch.stack([text_emoji_feat, img_feat], dim=1) attn_out, _ = self.cross_attn(multimodal, multimodal, multimodal) fused = attn_out.mean(dim=1) fused = self.fusion_proj(fused) + fused return self.classifier(fused) # === Model Wrapper with Prediction === class EmotionModel: def __init__(self): self.device = device self.tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') self.model = HybridMultimodalModel().to(self.device) self.model.load_state_dict(torch.load("app/hybrid_multimodal_model.pth", map_location=self.device)) self.model.eval() self.emoji_vecs = self.load_emoji_vectors("app/emoji2vec.txt") self.transform = transforms.Compose([ transforms.Grayscale(num_output_channels=3), transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize([0.5]*3, [0.5]*3) ]) self.mp_face_mesh = mp.solutions.face_mesh.FaceMesh(static_image_mode=True) def load_emoji_vectors(self, path): emoji_vecs = {} with open(path, 'r', encoding='utf-8') as f: for line in f: parts = line.strip().split() if len(parts) == 301: emoji_vecs[parts[0]] = np.array([float(x) for x in parts[1:]], dtype=np.float32) return emoji_vecs def get_emoji_vector(self, text): emojis = distinct_emoji_list(text) vecs = [self.emoji_vecs[e] for e in emojis if e in self.emoji_vecs] return np.mean(vecs, axis=0).astype(np.float32) if vecs else np.zeros(300, dtype=np.float32) def get_landmark_attention(self, image_np): results = self.mp_face_mesh.process(image_np) attention_map = np.zeros((image_np.shape[0], image_np.shape[1]), dtype=np.float32) if results.multi_face_landmarks: for landmark in results.multi_face_landmarks[0].landmark: x = int(landmark.x * image_np.shape[1]) y = int(landmark.y * image_np.shape[0]) if 0 <= x < image_np.shape[1] and 0 <= y < image_np.shape[0]: cv2.circle(attention_map, (x, y), 4, 1.0, -1) attention_map = cv2.GaussianBlur(attention_map, (31, 31), 0) attention_map = attention_map / np.max(attention_map) if np.max(attention_map) != 0 else attention_map attention_map = cv2.resize(attention_map, (224, 224)) return attention_map.astype(np.float32) def predict(self, text, image_url): # --- Text --- inputs = self.tokenizer(text, return_tensors='pt', truncation=True, padding='max_length', max_length=128) input_ids = inputs['input_ids'].to(self.device) attention_mask = inputs['attention_mask'].to(self.device) # --- Emoji --- emoji_vec = self.get_emoji_vector(text) emoji_tensor = torch.tensor(emoji_vec, dtype=torch.float32).unsqueeze(0).to(self.device) # --- Image --- response = requests.get(image_url) image = Image.open(BytesIO(response.content)).convert("RGB") image_np = np.array(image) img_tensor = self.transform(image).unsqueeze(0).to(self.device) # --- Attention Map from MediaPipe --- attn_map = self.get_landmark_attention(image_np) attn_tensor = torch.tensor(attn_map, dtype=torch.float32).unsqueeze(0).unsqueeze(0).to(self.device) # --- Final Prediction --- with torch.no_grad(): output = self.model(input_ids, attention_mask, emoji_tensor, img_tensor, attn_tensor) prediction = torch.argmax(output, dim=1).item() label_map = {0: "Negative", 1: "Neutral", 2: "Positive"} return label_map[prediction]