Spaces:
Build error
Build error
Upload 3 files
Browse files- app.py +3 -3
- tokenizer.pkl +2 -2
- tokenizer.py +10 -13
app.py
CHANGED
|
@@ -7,6 +7,7 @@ import json
|
|
| 7 |
from huggingface_hub import hf_hub_download
|
| 8 |
|
| 9 |
from model import image_captioning_model
|
|
|
|
| 10 |
|
| 11 |
# Загрузка файлов
|
| 12 |
repo_id = "Pumponchik/ImageCaption"
|
|
@@ -16,6 +17,7 @@ config_path = hf_hub_download(repo_id=repo_id, filename="config.json", repo_type
|
|
| 16 |
tokenizer_path = hf_hub_download(repo_id=repo_id, filename="tokenizer.pkl", repo_type=repo_type)
|
| 17 |
transform_path = hf_hub_download(repo_id=repo_id, filename="image_transforms.pkl", repo_type=repo_type)
|
| 18 |
|
|
|
|
| 19 |
# Загрузка конфигурации
|
| 20 |
with open(config_path, "r") as f:
|
| 21 |
full_config = json.load(f)
|
|
@@ -38,9 +40,7 @@ device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
| 38 |
model.to(device)
|
| 39 |
model.eval()
|
| 40 |
|
| 41 |
-
|
| 42 |
-
with open(tokenizer_path, "rb") as f:
|
| 43 |
-
tokenizer = pickle.load(f)
|
| 44 |
|
| 45 |
# Загрузка трансформаций
|
| 46 |
with open(transform_path, "rb") as f:
|
|
|
|
| 7 |
from huggingface_hub import hf_hub_download
|
| 8 |
|
| 9 |
from model import image_captioning_model
|
| 10 |
+
from tokenizer import CaptionTokenizer # Важно!
|
| 11 |
|
| 12 |
# Загрузка файлов
|
| 13 |
repo_id = "Pumponchik/ImageCaption"
|
|
|
|
| 17 |
tokenizer_path = hf_hub_download(repo_id=repo_id, filename="tokenizer.pkl", repo_type=repo_type)
|
| 18 |
transform_path = hf_hub_download(repo_id=repo_id, filename="image_transforms.pkl", repo_type=repo_type)
|
| 19 |
|
| 20 |
+
|
| 21 |
# Загрузка конфигурации
|
| 22 |
with open(config_path, "r") as f:
|
| 23 |
full_config = json.load(f)
|
|
|
|
| 40 |
model.to(device)
|
| 41 |
model.eval()
|
| 42 |
|
| 43 |
+
tokenizer = CaptionTokenizer.load('tokenizer.pkl')
|
|
|
|
|
|
|
| 44 |
|
| 45 |
# Загрузка трансформаций
|
| 46 |
with open(transform_path, "rb") as f:
|
tokenizer.pkl
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0e23f1eb7e56c299c90b074ba771a953a38434d9a7355b4d4b1263460c6ec83c
|
| 3 |
+
size 106
|
tokenizer.py
CHANGED
|
@@ -1,26 +1,23 @@
|
|
| 1 |
-
from collections import defaultdict
|
| 2 |
import pickle
|
| 3 |
|
| 4 |
class CaptionTokenizer:
|
| 5 |
-
def __init__(self
|
| 6 |
-
self.tok_to_ind =
|
| 7 |
-
self.ind_to_tok =
|
| 8 |
-
self.vocab_freq = vocab_freq
|
| 9 |
-
self.min_freq = min_freq
|
| 10 |
-
self.vocab_size = len(tok_to_ind)
|
| 11 |
|
| 12 |
def encode(self, text):
|
| 13 |
-
return [self.tok_to_ind
|
| 14 |
-
|
| 15 |
-
|
| 16 |
def decode(self, indices):
|
| 17 |
-
return
|
| 18 |
|
| 19 |
def save(self, path):
|
| 20 |
with open(path, 'wb') as f:
|
| 21 |
-
pickle.dump(self, f)
|
| 22 |
|
| 23 |
@classmethod
|
| 24 |
def load(cls, path):
|
|
|
|
| 25 |
with open(path, 'rb') as f:
|
| 26 |
-
|
|
|
|
|
|
|
|
|
| 1 |
import pickle
|
| 2 |
|
| 3 |
class CaptionTokenizer:
|
| 4 |
+
def __init__(self):
|
| 5 |
+
self.tok_to_ind = {'<PAD>': 0, '<BOS>': 1, '<EOS>': 2, '<UNK>': 3}
|
| 6 |
+
self.ind_to_tok = {v: k for k, v in self.tok_to_ind.items()}
|
|
|
|
|
|
|
|
|
|
| 7 |
|
| 8 |
def encode(self, text):
|
| 9 |
+
return [self.tok_to_ind.get(word, 3) for word in text.split()]
|
| 10 |
+
|
|
|
|
| 11 |
def decode(self, indices):
|
| 12 |
+
return ' '.join([self.ind_to_tok.get(idx, '<UNK>') for idx in indices])
|
| 13 |
|
| 14 |
def save(self, path):
|
| 15 |
with open(path, 'wb') as f:
|
| 16 |
+
pickle.dump(self.__dict__, f)
|
| 17 |
|
| 18 |
@classmethod
|
| 19 |
def load(cls, path):
|
| 20 |
+
tokenizer = cls()
|
| 21 |
with open(path, 'rb') as f:
|
| 22 |
+
tokenizer.__dict__ = pickle.load(f)
|
| 23 |
+
return tokenizer
|