Spaces:
Build error
Build error
Upload 3 files
Browse files- app.py +11 -30
- config.json +2 -1
- model.py +21 -29
app.py
CHANGED
|
@@ -4,12 +4,9 @@ import gradio as gr
|
|
| 4 |
from PIL import Image
|
| 5 |
import pickle
|
| 6 |
import json
|
| 7 |
-
from huggingface_hub import hf_hub_download
|
| 8 |
-
from model import image_captioning_model
|
| 9 |
-
from torchvision import models
|
| 10 |
|
| 11 |
-
|
| 12 |
-
# login(token="hf_YOUR_API_TOKEN") # Замените на ваш токен или уберите для публичного Space
|
| 13 |
|
| 14 |
# Загрузка файлов
|
| 15 |
repo_id = "Pumponchik/ImageCaption"
|
|
@@ -34,16 +31,7 @@ model_params = {
|
|
| 34 |
# Инициализация модели
|
| 35 |
model = image_captioning_model(**model_params)
|
| 36 |
|
| 37 |
-
# Загрузка
|
| 38 |
-
weights = models.ResNet50_Weights.DEFAULT
|
| 39 |
-
resnet = models.resnet50(weights=weights, progress=True)
|
| 40 |
-
model.img_fe.resnet.load_state_dict(resnet.state_dict())
|
| 41 |
-
|
| 42 |
-
# Замораживаем веса ResNet
|
| 43 |
-
for param in model.img_fe.resnet.parameters():
|
| 44 |
-
param.requires_grad = False
|
| 45 |
-
|
| 46 |
-
# Загрузка остальной части state_dict с игнорированием отсутствующих ключей
|
| 47 |
model.load_state_dict(torch.load(model_path, map_location="cpu"), strict=False)
|
| 48 |
|
| 49 |
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
@@ -54,14 +42,9 @@ model.eval()
|
|
| 54 |
with open(tokenizer_path, "rb") as f:
|
| 55 |
tokenizer = pickle.load(f)
|
| 56 |
|
| 57 |
-
#
|
| 58 |
-
resnet_transform = weights.transforms()
|
| 59 |
with open(transform_path, "rb") as f:
|
| 60 |
-
|
| 61 |
-
# Комбинируем или используем transform из весов ResNet, если custom_transform отсутствует
|
| 62 |
-
transform = custom_transform if custom_transform else resnet_transform
|
| 63 |
-
|
| 64 |
-
title_width = 43
|
| 65 |
|
| 66 |
def wrap_text(text, max_width):
|
| 67 |
words = text.split(" ")
|
|
@@ -79,13 +62,12 @@ def generate(model, image, tokenizer, max_seq_len=20, top_p=None, top_k=1):
|
|
| 79 |
|
| 80 |
model.eval()
|
| 81 |
device = next(model.parameters()).device
|
| 82 |
-
image = transform(image).to(device).unsqueeze(0)
|
| 83 |
batch_size = image.size(0)
|
| 84 |
|
| 85 |
with torch.no_grad():
|
| 86 |
-
img_emb = model.
|
| 87 |
-
h_0 = model.
|
| 88 |
-
h_0 = h_0.unsqueeze(0).repeat(model.text_features.num_layers, 1, 1)
|
| 89 |
c_0 = h_0.clone()
|
| 90 |
|
| 91 |
input_token = torch.full((batch_size,), tokenizer.tok_to_ind['<BOS>'], dtype=torch.long, device=device)
|
|
@@ -93,9 +75,8 @@ def generate(model, image, tokenizer, max_seq_len=20, top_p=None, top_k=1):
|
|
| 93 |
mask = torch.ones(batch_size, dtype=torch.bool, device=device)
|
| 94 |
|
| 95 |
for _ in range(max_seq_len):
|
| 96 |
-
input_emb = model.
|
| 97 |
-
|
| 98 |
-
output, (h_0, c_0) = model.text_features.rnn(input_emb, (h_0, c_0))
|
| 99 |
logits = model.fc_out(output.squeeze(1))
|
| 100 |
probs = F.softmax(logits, dim=-1)
|
| 101 |
|
|
@@ -146,7 +127,7 @@ def generate_caption(image):
|
|
| 146 |
image = image.convert("RGB")
|
| 147 |
max_seq_len = full_config.get("max_len", 20)
|
| 148 |
caption = generate(model, image, tokenizer, max_seq_len=max_seq_len, top_k=5)
|
| 149 |
-
caption = wrap_text(caption,
|
| 150 |
return caption
|
| 151 |
|
| 152 |
# Создание Gradio-интерфейса
|
|
|
|
| 4 |
from PIL import Image
|
| 5 |
import pickle
|
| 6 |
import json
|
| 7 |
+
from huggingface_hub import hf_hub_download
|
|
|
|
|
|
|
| 8 |
|
| 9 |
+
from model import image_captioning_model
|
|
|
|
| 10 |
|
| 11 |
# Загрузка файлов
|
| 12 |
repo_id = "Pumponchik/ImageCaption"
|
|
|
|
| 31 |
# Инициализация модели
|
| 32 |
model = image_captioning_model(**model_params)
|
| 33 |
|
| 34 |
+
# Загрузка весов модели
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
model.load_state_dict(torch.load(model_path, map_location="cpu"), strict=False)
|
| 36 |
|
| 37 |
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
|
|
| 42 |
with open(tokenizer_path, "rb") as f:
|
| 43 |
tokenizer = pickle.load(f)
|
| 44 |
|
| 45 |
+
# Загрузка трансформаций
|
|
|
|
| 46 |
with open(transform_path, "rb") as f:
|
| 47 |
+
transform = pickle.load(f)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
|
| 49 |
def wrap_text(text, max_width):
|
| 50 |
words = text.split(" ")
|
|
|
|
| 62 |
|
| 63 |
model.eval()
|
| 64 |
device = next(model.parameters()).device
|
| 65 |
+
image = transform(image).to(device).unsqueeze(0)
|
| 66 |
batch_size = image.size(0)
|
| 67 |
|
| 68 |
with torch.no_grad():
|
| 69 |
+
img_emb = model.img_fe(image)
|
| 70 |
+
h_0 = model.img_to_hidden(img_emb).unsqueeze(0).repeat(model.text_fe.num_layers, 1, 1)
|
|
|
|
| 71 |
c_0 = h_0.clone()
|
| 72 |
|
| 73 |
input_token = torch.full((batch_size,), tokenizer.tok_to_ind['<BOS>'], dtype=torch.long, device=device)
|
|
|
|
| 75 |
mask = torch.ones(batch_size, dtype=torch.bool, device=device)
|
| 76 |
|
| 77 |
for _ in range(max_seq_len):
|
| 78 |
+
input_emb = model.text_fe.embedding(input_token).unsqueeze(1)
|
| 79 |
+
output, (h_0, c_0) = model.text_fe.rnn(input_emb, (h_0, c_0))
|
|
|
|
| 80 |
logits = model.fc_out(output.squeeze(1))
|
| 81 |
probs = F.softmax(logits, dim=-1)
|
| 82 |
|
|
|
|
| 127 |
image = image.convert("RGB")
|
| 128 |
max_seq_len = full_config.get("max_len", 20)
|
| 129 |
caption = generate(model, image, tokenizer, max_seq_len=max_seq_len, top_k=5)
|
| 130 |
+
caption = wrap_text(caption, 43)
|
| 131 |
return caption
|
| 132 |
|
| 133 |
# Создание Gradio-интерфейса
|
config.json
CHANGED
|
@@ -2,5 +2,6 @@
|
|
| 2 |
"vocab_size": 3478,
|
| 3 |
"embedding_dim": 300,
|
| 4 |
"hidden_size": 1024,
|
| 5 |
-
"num_layers": 1
|
|
|
|
| 6 |
}
|
|
|
|
| 2 |
"vocab_size": 3478,
|
| 3 |
"embedding_dim": 300,
|
| 4 |
"hidden_size": 1024,
|
| 5 |
+
"num_layers": 1,
|
| 6 |
+
"max_len": 20
|
| 7 |
}
|
model.py
CHANGED
|
@@ -5,21 +5,33 @@ import torchvision.models as models
|
|
| 5 |
class img_fe_class(nn.Module):
|
| 6 |
def __init__(self):
|
| 7 |
super(img_fe_class, self).__init__()
|
| 8 |
-
|
| 9 |
-
self.resnet = models.resnet50(weights=
|
| 10 |
-
|
| 11 |
for param in self.resnet.parameters():
|
| 12 |
param.requires_grad = False
|
| 13 |
-
|
| 14 |
def forward(self, imgs):
|
| 15 |
-
|
| 16 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
|
| 18 |
class text_fe_class(nn.Module):
|
| 19 |
def __init__(self, vocab_size, embedding_dim, hidden_size, num_layers):
|
| 20 |
super(text_fe_class, self).__init__()
|
| 21 |
self.embedding = nn.Embedding(vocab_size, embedding_dim)
|
| 22 |
self.rnn = nn.LSTM(embedding_dim, hidden_size, num_layers, batch_first=True)
|
|
|
|
| 23 |
|
| 24 |
def forward(self, captions, hidden):
|
| 25 |
embeddings = self.embedding(captions)
|
|
@@ -32,32 +44,12 @@ class image_captioning_model(nn.Module):
|
|
| 32 |
self.img_fe = img_fe_class()
|
| 33 |
self.text_fe = text_fe_class(vocab_size, embedding_dim, hidden_size, num_layers)
|
| 34 |
self.fc_out = nn.Linear(hidden_size, vocab_size)
|
| 35 |
-
self.hidden_size = hidden_size
|
| 36 |
-
self.num_layers = num_layers
|
| 37 |
self.img_to_hidden = nn.Linear(2048, hidden_size) # 2048 - размер выхода ResNet50
|
| 38 |
|
| 39 |
def forward(self, imgs, captions):
|
| 40 |
img_features = self.img_fe(imgs)
|
| 41 |
-
hidden = self.img_to_hidden(img_features).unsqueeze(0).repeat(self.num_layers, 1, 1)
|
| 42 |
-
cell = torch.zeros_like(hidden)
|
| 43 |
outputs, _ = self.text_fe(captions, (hidden, cell))
|
| 44 |
outputs = self.fc_out(outputs)
|
| 45 |
-
return outputs
|
| 46 |
-
|
| 47 |
-
def generate_caption(self, img, tokenizer, max_len=30, device="cuda"):
|
| 48 |
-
self.eval()
|
| 49 |
-
with torch.no_grad():
|
| 50 |
-
img = img.unsqueeze(0).to(device)
|
| 51 |
-
img_features = self.img_fe(img)
|
| 52 |
-
hidden = self.img_to_hidden(img_features).unsqueeze(0).repeat(self.num_layers, 1, 1)
|
| 53 |
-
cell = torch.zeros_like(hidden)
|
| 54 |
-
caption = [tokenizer.tok_to_ind["<BOS>"]]
|
| 55 |
-
for _ in range(max_len):
|
| 56 |
-
inputs = torch.LongTensor(caption[-1]).unsqueeze(0).unsqueeze(0).to(device)
|
| 57 |
-
outputs, (hidden, cell) = self.text_fe(inputs, (hidden, cell))
|
| 58 |
-
outputs = self.fc_out(outputs[:, -1, :])
|
| 59 |
-
next_token = outputs.argmax(dim=-1).item()
|
| 60 |
-
caption.append(next_token)
|
| 61 |
-
if next_token == tokenizer.tok_to_ind["<EOS>"]:
|
| 62 |
-
break
|
| 63 |
-
return tokenizer.decode(caption)
|
|
|
|
| 5 |
class img_fe_class(nn.Module):
|
| 6 |
def __init__(self):
|
| 7 |
super(img_fe_class, self).__init__()
|
| 8 |
+
# Загружаем полную модель ResNet50 с предобученными весами
|
| 9 |
+
self.resnet = models.resnet50(weights=models.ResNet50_Weights.DEFAULT)
|
| 10 |
+
# Замораживаем все параметры
|
| 11 |
for param in self.resnet.parameters():
|
| 12 |
param.requires_grad = False
|
| 13 |
+
|
| 14 |
def forward(self, imgs):
|
| 15 |
+
# Используем все слои до avgpool
|
| 16 |
+
x = self.resnet.conv1(imgs)
|
| 17 |
+
x = self.resnet.bn1(x)
|
| 18 |
+
x = self.resnet.relu(x)
|
| 19 |
+
x = self.resnet.maxpool(x)
|
| 20 |
+
|
| 21 |
+
x = self.resnet.layer1(x)
|
| 22 |
+
x = self.resnet.layer2(x)
|
| 23 |
+
x = self.resnet.layer3(x)
|
| 24 |
+
x = self.resnet.layer4(x)
|
| 25 |
+
|
| 26 |
+
x = self.resnet.avgpool(x)
|
| 27 |
+
return torch.flatten(x, 1)
|
| 28 |
|
| 29 |
class text_fe_class(nn.Module):
|
| 30 |
def __init__(self, vocab_size, embedding_dim, hidden_size, num_layers):
|
| 31 |
super(text_fe_class, self).__init__()
|
| 32 |
self.embedding = nn.Embedding(vocab_size, embedding_dim)
|
| 33 |
self.rnn = nn.LSTM(embedding_dim, hidden_size, num_layers, batch_first=True)
|
| 34 |
+
self.num_layers = num_layers
|
| 35 |
|
| 36 |
def forward(self, captions, hidden):
|
| 37 |
embeddings = self.embedding(captions)
|
|
|
|
| 44 |
self.img_fe = img_fe_class()
|
| 45 |
self.text_fe = text_fe_class(vocab_size, embedding_dim, hidden_size, num_layers)
|
| 46 |
self.fc_out = nn.Linear(hidden_size, vocab_size)
|
|
|
|
|
|
|
| 47 |
self.img_to_hidden = nn.Linear(2048, hidden_size) # 2048 - размер выхода ResNet50
|
| 48 |
|
| 49 |
def forward(self, imgs, captions):
|
| 50 |
img_features = self.img_fe(imgs)
|
| 51 |
+
hidden = self.img_to_hidden(img_features).unsqueeze(0).repeat(self.text_fe.num_layers, 1, 1)
|
| 52 |
+
cell = torch.zeros_like(hidden)
|
| 53 |
outputs, _ = self.text_fe(captions, (hidden, cell))
|
| 54 |
outputs = self.fc_out(outputs)
|
| 55 |
+
return outputs
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|