Pumponchik commited on
Commit
15012bd
·
verified ·
1 Parent(s): 69c39e7

Upload 3 files

Browse files
Files changed (3) hide show
  1. app.py +11 -30
  2. config.json +2 -1
  3. model.py +21 -29
app.py CHANGED
@@ -4,12 +4,9 @@ import gradio as gr
4
  from PIL import Image
5
  import pickle
6
  import json
7
- from huggingface_hub import hf_hub_download, login
8
- from model import image_captioning_model
9
- from torchvision import models
10
 
11
- # Аутентификация
12
- # login(token="hf_YOUR_API_TOKEN") # Замените на ваш токен или уберите для публичного Space
13
 
14
  # Загрузка файлов
15
  repo_id = "Pumponchik/ImageCaption"
@@ -34,16 +31,7 @@ model_params = {
34
  # Инициализация модели
35
  model = image_captioning_model(**model_params)
36
 
37
- # Загрузка предобученных весов ResNet с вашим подходом
38
- weights = models.ResNet50_Weights.DEFAULT
39
- resnet = models.resnet50(weights=weights, progress=True)
40
- model.img_fe.resnet.load_state_dict(resnet.state_dict())
41
-
42
- # Замораживаем веса ResNet
43
- for param in model.img_fe.resnet.parameters():
44
- param.requires_grad = False
45
-
46
- # Загрузка остальной части state_dict с игнорированием отсутствующих ключей
47
  model.load_state_dict(torch.load(model_path, map_location="cpu"), strict=False)
48
 
49
  device = "cuda" if torch.cuda.is_available() else "cpu"
@@ -54,14 +42,9 @@ model.eval()
54
  with open(tokenizer_path, "rb") as f:
55
  tokenizer = pickle.load(f)
56
 
57
- # Обновление transform с аугментациями из весов ResNet
58
- resnet_transform = weights.transforms()
59
  with open(transform_path, "rb") as f:
60
- custom_transform = pickle.load(f)
61
- # Комбинируем или используем transform из весов ResNet, если custom_transform отсутствует
62
- transform = custom_transform if custom_transform else resnet_transform
63
-
64
- title_width = 43
65
 
66
  def wrap_text(text, max_width):
67
  words = text.split(" ")
@@ -79,13 +62,12 @@ def generate(model, image, tokenizer, max_seq_len=20, top_p=None, top_k=1):
79
 
80
  model.eval()
81
  device = next(model.parameters()).device
82
- image = transform(image).to(device).unsqueeze(0) # Применяем transform перед подачей в модель
83
  batch_size = image.size(0)
84
 
85
  with torch.no_grad():
86
- img_emb = model.img_features(image)
87
- h_0 = model.text_features.img_to_hidden(img_emb)
88
- h_0 = h_0.unsqueeze(0).repeat(model.text_features.num_layers, 1, 1)
89
  c_0 = h_0.clone()
90
 
91
  input_token = torch.full((batch_size,), tokenizer.tok_to_ind['<BOS>'], dtype=torch.long, device=device)
@@ -93,9 +75,8 @@ def generate(model, image, tokenizer, max_seq_len=20, top_p=None, top_k=1):
93
  mask = torch.ones(batch_size, dtype=torch.bool, device=device)
94
 
95
  for _ in range(max_seq_len):
96
- input_emb = model.text_features.embed(input_token).unsqueeze(1)
97
- input_emb = model.text_features.text_to_hidden(input_emb)
98
- output, (h_0, c_0) = model.text_features.rnn(input_emb, (h_0, c_0))
99
  logits = model.fc_out(output.squeeze(1))
100
  probs = F.softmax(logits, dim=-1)
101
 
@@ -146,7 +127,7 @@ def generate_caption(image):
146
  image = image.convert("RGB")
147
  max_seq_len = full_config.get("max_len", 20)
148
  caption = generate(model, image, tokenizer, max_seq_len=max_seq_len, top_k=5)
149
- caption = wrap_text(caption, title_width)
150
  return caption
151
 
152
  # Создание Gradio-интерфейса
 
4
  from PIL import Image
5
  import pickle
6
  import json
7
+ from huggingface_hub import hf_hub_download
 
 
8
 
9
+ from model import image_captioning_model
 
10
 
11
  # Загрузка файлов
12
  repo_id = "Pumponchik/ImageCaption"
 
31
  # Инициализация модели
32
  model = image_captioning_model(**model_params)
33
 
34
+ # Загрузка весов модели
 
 
 
 
 
 
 
 
 
35
  model.load_state_dict(torch.load(model_path, map_location="cpu"), strict=False)
36
 
37
  device = "cuda" if torch.cuda.is_available() else "cpu"
 
42
  with open(tokenizer_path, "rb") as f:
43
  tokenizer = pickle.load(f)
44
 
45
+ # Загрузка трансформаций
 
46
  with open(transform_path, "rb") as f:
47
+ transform = pickle.load(f)
 
 
 
 
48
 
49
  def wrap_text(text, max_width):
50
  words = text.split(" ")
 
62
 
63
  model.eval()
64
  device = next(model.parameters()).device
65
+ image = transform(image).to(device).unsqueeze(0)
66
  batch_size = image.size(0)
67
 
68
  with torch.no_grad():
69
+ img_emb = model.img_fe(image)
70
+ h_0 = model.img_to_hidden(img_emb).unsqueeze(0).repeat(model.text_fe.num_layers, 1, 1)
 
71
  c_0 = h_0.clone()
72
 
73
  input_token = torch.full((batch_size,), tokenizer.tok_to_ind['<BOS>'], dtype=torch.long, device=device)
 
75
  mask = torch.ones(batch_size, dtype=torch.bool, device=device)
76
 
77
  for _ in range(max_seq_len):
78
+ input_emb = model.text_fe.embedding(input_token).unsqueeze(1)
79
+ output, (h_0, c_0) = model.text_fe.rnn(input_emb, (h_0, c_0))
 
80
  logits = model.fc_out(output.squeeze(1))
81
  probs = F.softmax(logits, dim=-1)
82
 
 
127
  image = image.convert("RGB")
128
  max_seq_len = full_config.get("max_len", 20)
129
  caption = generate(model, image, tokenizer, max_seq_len=max_seq_len, top_k=5)
130
+ caption = wrap_text(caption, 43)
131
  return caption
132
 
133
  # Создание Gradio-интерфейса
config.json CHANGED
@@ -2,5 +2,6 @@
2
  "vocab_size": 3478,
3
  "embedding_dim": 300,
4
  "hidden_size": 1024,
5
- "num_layers": 1
 
6
  }
 
2
  "vocab_size": 3478,
3
  "embedding_dim": 300,
4
  "hidden_size": 1024,
5
+ "num_layers": 1,
6
+ "max_len": 20
7
  }
model.py CHANGED
@@ -5,21 +5,33 @@ import torchvision.models as models
5
  class img_fe_class(nn.Module):
6
  def __init__(self):
7
  super(img_fe_class, self).__init__()
8
- weights = models.ResNet50_Weights.DEFAULT
9
- self.resnet = models.resnet50(weights=weights)
10
- self.resnet = nn.Sequential(*list(self.resnet.children())[:-1])
11
  for param in self.resnet.parameters():
12
  param.requires_grad = False
13
-
14
  def forward(self, imgs):
15
- features = self.resnet(imgs)
16
- return features.reshape(features.size(0), -1)
 
 
 
 
 
 
 
 
 
 
 
17
 
18
  class text_fe_class(nn.Module):
19
  def __init__(self, vocab_size, embedding_dim, hidden_size, num_layers):
20
  super(text_fe_class, self).__init__()
21
  self.embedding = nn.Embedding(vocab_size, embedding_dim)
22
  self.rnn = nn.LSTM(embedding_dim, hidden_size, num_layers, batch_first=True)
 
23
 
24
  def forward(self, captions, hidden):
25
  embeddings = self.embedding(captions)
@@ -32,32 +44,12 @@ class image_captioning_model(nn.Module):
32
  self.img_fe = img_fe_class()
33
  self.text_fe = text_fe_class(vocab_size, embedding_dim, hidden_size, num_layers)
34
  self.fc_out = nn.Linear(hidden_size, vocab_size)
35
- self.hidden_size = hidden_size
36
- self.num_layers = num_layers
37
  self.img_to_hidden = nn.Linear(2048, hidden_size) # 2048 - размер выхода ResNet50
38
 
39
  def forward(self, imgs, captions):
40
  img_features = self.img_fe(imgs)
41
- hidden = self.img_to_hidden(img_features).unsqueeze(0).repeat(self.num_layers, 1, 1)
42
- cell = torch.zeros_like(hidden) # Для LSTM нужно также cell state
43
  outputs, _ = self.text_fe(captions, (hidden, cell))
44
  outputs = self.fc_out(outputs)
45
- return outputs
46
-
47
- def generate_caption(self, img, tokenizer, max_len=30, device="cuda"):
48
- self.eval()
49
- with torch.no_grad():
50
- img = img.unsqueeze(0).to(device)
51
- img_features = self.img_fe(img)
52
- hidden = self.img_to_hidden(img_features).unsqueeze(0).repeat(self.num_layers, 1, 1)
53
- cell = torch.zeros_like(hidden)
54
- caption = [tokenizer.tok_to_ind["<BOS>"]]
55
- for _ in range(max_len):
56
- inputs = torch.LongTensor(caption[-1]).unsqueeze(0).unsqueeze(0).to(device)
57
- outputs, (hidden, cell) = self.text_fe(inputs, (hidden, cell))
58
- outputs = self.fc_out(outputs[:, -1, :])
59
- next_token = outputs.argmax(dim=-1).item()
60
- caption.append(next_token)
61
- if next_token == tokenizer.tok_to_ind["<EOS>"]:
62
- break
63
- return tokenizer.decode(caption)
 
5
  class img_fe_class(nn.Module):
6
  def __init__(self):
7
  super(img_fe_class, self).__init__()
8
+ # Загружаем полную модель ResNet50 с предобученными весами
9
+ self.resnet = models.resnet50(weights=models.ResNet50_Weights.DEFAULT)
10
+ # Замораживаем все параметры
11
  for param in self.resnet.parameters():
12
  param.requires_grad = False
13
+
14
  def forward(self, imgs):
15
+ # Используем все слои до avgpool
16
+ x = self.resnet.conv1(imgs)
17
+ x = self.resnet.bn1(x)
18
+ x = self.resnet.relu(x)
19
+ x = self.resnet.maxpool(x)
20
+
21
+ x = self.resnet.layer1(x)
22
+ x = self.resnet.layer2(x)
23
+ x = self.resnet.layer3(x)
24
+ x = self.resnet.layer4(x)
25
+
26
+ x = self.resnet.avgpool(x)
27
+ return torch.flatten(x, 1)
28
 
29
  class text_fe_class(nn.Module):
30
  def __init__(self, vocab_size, embedding_dim, hidden_size, num_layers):
31
  super(text_fe_class, self).__init__()
32
  self.embedding = nn.Embedding(vocab_size, embedding_dim)
33
  self.rnn = nn.LSTM(embedding_dim, hidden_size, num_layers, batch_first=True)
34
+ self.num_layers = num_layers
35
 
36
  def forward(self, captions, hidden):
37
  embeddings = self.embedding(captions)
 
44
  self.img_fe = img_fe_class()
45
  self.text_fe = text_fe_class(vocab_size, embedding_dim, hidden_size, num_layers)
46
  self.fc_out = nn.Linear(hidden_size, vocab_size)
 
 
47
  self.img_to_hidden = nn.Linear(2048, hidden_size) # 2048 - размер выхода ResNet50
48
 
49
  def forward(self, imgs, captions):
50
  img_features = self.img_fe(imgs)
51
+ hidden = self.img_to_hidden(img_features).unsqueeze(0).repeat(self.text_fe.num_layers, 1, 1)
52
+ cell = torch.zeros_like(hidden)
53
  outputs, _ = self.text_fe(captions, (hidden, cell))
54
  outputs = self.fc_out(outputs)
55
+ return outputs