Download app/model_loader.py from Akash297/TEI: direct link, hf CLI and curl.
- Browser
- Download file 5.81 kB
-
https://huggingface.co/spaces/Akash297/TEI/resolve/main/app/model_loader.py
- Command line
-
hf download hf://spaces/Akash297/TEI/app/model_loader.py
-
curl -L -o model_loader.py https://huggingface.co/spaces/Akash297/TEI/resolve/main/app/model_loader.py
5.81 kB
| import torch | |
| import torch.nn as nn | |
| import numpy as np | |
| from PIL import Image | |
| import requests | |
| from io import BytesIO | |
| from torchvision import transforms, models | |
| from transformers import DistilBertTokenizer, DistilBertModel | |
| import cv2 | |
| import mediapipe as mp | |
| from emoji import distinct_emoji_list | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| # === Model Definition === | |
| class HybridMultimodalModel(nn.Module): | |
| def __init__(self, num_classes=3): | |
| super().__init__() | |
| self.bert = DistilBertModel.from_pretrained("distilbert-base-uncased") | |
| self.emoji_proj = nn.Linear(300, 768) | |
| self.gate = nn.Sequential(nn.Linear(768 + 300, 768), nn.Sigmoid()) | |
| self.text_emoji_fc = nn.Sequential(nn.Linear(768 + 300, 256), nn.ReLU()) | |
| resnet = models.resnet18(pretrained=True) | |
| resnet.conv1 = nn.Conv2d(4, 64, 7, 2, 3, bias=False) # for RGB + attn | |
| self.image_model = nn.Sequential(*list(resnet.children())[:-1]) | |
| self.image_fc = nn.Linear(512, 256) | |
| self.cross_attn = nn.MultiheadAttention(256, 4, batch_first=True) | |
| self.fusion_proj = nn.Sequential( | |
| nn.Linear(256, 256), nn.ReLU(), nn.Dropout(0.2), nn.Linear(256, 256) | |
| ) | |
| self.classifier = nn.Sequential( | |
| nn.Linear(256, 128), nn.ReLU(), nn.Dropout(0.3), nn.Linear(128, num_classes) | |
| ) | |
| def forward(self, input_ids, attention_mask, emoji_vec, image, attn_map): | |
| text_feat = self.bert(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state[:, 0] | |
| emoji_proj = self.emoji_proj(emoji_vec) | |
| alpha = self.gate(torch.cat((text_feat, emoji_vec), dim=1)) | |
| gated = alpha * emoji_proj | |
| text_emoji_feat = self.text_emoji_fc(torch.cat((text_feat + gated, emoji_vec), dim=1)) | |
| x = torch.cat((image, attn_map), dim=1) | |
| img_feat = self.image_model(x).squeeze() | |
| if len(img_feat.shape) == 1: | |
| img_feat = img_feat.unsqueeze(0) | |
| img_feat = self.image_fc(img_feat) | |
| multimodal = torch.stack([text_emoji_feat, img_feat], dim=1) | |
| attn_out, _ = self.cross_attn(multimodal, multimodal, multimodal) | |
| fused = attn_out.mean(dim=1) | |
| fused = self.fusion_proj(fused) + fused | |
| return self.classifier(fused) | |
| # === Model Wrapper with Prediction === | |
| class EmotionModel: | |
| def __init__(self): | |
| self.device = device | |
| self.tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') | |
| self.model = HybridMultimodalModel().to(self.device) | |
| self.model.load_state_dict(torch.load("app/hybrid_multimodal_model.pth", map_location=self.device)) | |
| self.model.eval() | |
| self.emoji_vecs = self.load_emoji_vectors("app/emoji2vec.txt") | |
| self.transform = transforms.Compose([ | |
| transforms.Grayscale(num_output_channels=3), | |
| transforms.Resize((224, 224)), | |
| transforms.ToTensor(), | |
| transforms.Normalize([0.5]*3, [0.5]*3) | |
| ]) | |
| self.mp_face_mesh = mp.solutions.face_mesh.FaceMesh(static_image_mode=True) | |
| def load_emoji_vectors(self, path): | |
| emoji_vecs = {} | |
| with open(path, 'r', encoding='utf-8') as f: | |
| for line in f: | |
| parts = line.strip().split() | |
| if len(parts) == 301: | |
| emoji_vecs[parts[0]] = np.array([float(x) for x in parts[1:]], dtype=np.float32) | |
| return emoji_vecs | |
| def get_emoji_vector(self, text): | |
| emojis = distinct_emoji_list(text) | |
| vecs = [self.emoji_vecs[e] for e in emojis if e in self.emoji_vecs] | |
| return np.mean(vecs, axis=0).astype(np.float32) if vecs else np.zeros(300, dtype=np.float32) | |
| def get_landmark_attention(self, image_np): | |
| results = self.mp_face_mesh.process(image_np) | |
| attention_map = np.zeros((image_np.shape[0], image_np.shape[1]), dtype=np.float32) | |
| if results.multi_face_landmarks: | |
| for landmark in results.multi_face_landmarks[0].landmark: | |
| x = int(landmark.x * image_np.shape[1]) | |
| y = int(landmark.y * image_np.shape[0]) | |
| if 0 <= x < image_np.shape[1] and 0 <= y < image_np.shape[0]: | |
| cv2.circle(attention_map, (x, y), 4, 1.0, -1) | |
| attention_map = cv2.GaussianBlur(attention_map, (31, 31), 0) | |
| attention_map = attention_map / np.max(attention_map) if np.max(attention_map) != 0 else attention_map | |
| attention_map = cv2.resize(attention_map, (224, 224)) | |
| return attention_map.astype(np.float32) | |
| def predict(self, text, image_url): | |
| # --- Text --- | |
| inputs = self.tokenizer(text, return_tensors='pt', truncation=True, padding='max_length', max_length=128) | |
| input_ids = inputs['input_ids'].to(self.device) | |
| attention_mask = inputs['attention_mask'].to(self.device) | |
| # --- Emoji --- | |
| emoji_vec = self.get_emoji_vector(text) | |
| emoji_tensor = torch.tensor(emoji_vec, dtype=torch.float32).unsqueeze(0).to(self.device) | |
| # --- Image --- | |
| response = requests.get(image_url) | |
| image = Image.open(BytesIO(response.content)).convert("RGB") | |
| image_np = np.array(image) | |
| img_tensor = self.transform(image).unsqueeze(0).to(self.device) | |
| # --- Attention Map from MediaPipe --- | |
| attn_map = self.get_landmark_attention(image_np) | |
| attn_tensor = torch.tensor(attn_map, dtype=torch.float32).unsqueeze(0).unsqueeze(0).to(self.device) | |
| # --- Final Prediction --- | |
| with torch.no_grad(): | |
| output = self.model(input_ids, attention_mask, emoji_tensor, img_tensor, attn_tensor) | |
| prediction = torch.argmax(output, dim=1).item() | |
| label_map = {0: "Negative", 1: "Neutral", 2: "Positive"} | |
| return label_map[prediction] | |