Pumponchik commited on
Commit
bbb5cc7
·
verified ·
1 Parent(s): bb43d66

Upload 3 files

Browse files
Files changed (3) hide show
  1. app.py +3 -3
  2. tokenizer.pkl +2 -2
  3. tokenizer.py +10 -13
app.py CHANGED
@@ -7,6 +7,7 @@ import json
7
  from huggingface_hub import hf_hub_download
8
 
9
  from model import image_captioning_model
 
10
 
11
  # Загрузка файлов
12
  repo_id = "Pumponchik/ImageCaption"
@@ -16,6 +17,7 @@ config_path = hf_hub_download(repo_id=repo_id, filename="config.json", repo_type
16
  tokenizer_path = hf_hub_download(repo_id=repo_id, filename="tokenizer.pkl", repo_type=repo_type)
17
  transform_path = hf_hub_download(repo_id=repo_id, filename="image_transforms.pkl", repo_type=repo_type)
18
 
 
19
  # Загрузка конфигурации
20
  with open(config_path, "r") as f:
21
  full_config = json.load(f)
@@ -38,9 +40,7 @@ device = "cuda" if torch.cuda.is_available() else "cpu"
38
  model.to(device)
39
  model.eval()
40
 
41
- # Загрузка токенизатора
42
- with open(tokenizer_path, "rb") as f:
43
- tokenizer = pickle.load(f)
44
 
45
  # Загрузка трансформаций
46
  with open(transform_path, "rb") as f:
 
7
  from huggingface_hub import hf_hub_download
8
 
9
  from model import image_captioning_model
10
+ from tokenizer import CaptionTokenizer # Важно!
11
 
12
  # Загрузка файлов
13
  repo_id = "Pumponchik/ImageCaption"
 
17
  tokenizer_path = hf_hub_download(repo_id=repo_id, filename="tokenizer.pkl", repo_type=repo_type)
18
  transform_path = hf_hub_download(repo_id=repo_id, filename="image_transforms.pkl", repo_type=repo_type)
19
 
20
+
21
  # Загрузка конфигурации
22
  with open(config_path, "r") as f:
23
  full_config = json.load(f)
 
40
  model.to(device)
41
  model.eval()
42
 
43
+ tokenizer = CaptionTokenizer.load('tokenizer.pkl')
 
 
44
 
45
  # Загрузка трансформаций
46
  with open(transform_path, "rb") as f:
tokenizer.pkl CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:42542fedd1f4ec828d692b165dca24f65a48f44e4a89283a408ac9ed702e384b
3
- size 139663
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0e23f1eb7e56c299c90b074ba771a953a38434d9a7355b4d4b1263460c6ec83c
3
+ size 106
tokenizer.py CHANGED
@@ -1,26 +1,23 @@
1
- from collections import defaultdict
2
  import pickle
3
 
4
  class CaptionTokenizer:
5
- def __init__(self, tok_to_ind, ind_to_tok, vocab_freq, min_freq=5):
6
- self.tok_to_ind = tok_to_ind
7
- self.ind_to_tok = ind_to_tok
8
- self.vocab_freq = vocab_freq
9
- self.min_freq = min_freq
10
- self.vocab_size = len(tok_to_ind)
11
 
12
  def encode(self, text):
13
- return [self.tok_to_ind[tok] if tok in self.tok_to_ind and self.vocab_freq[tok] >= self.min_freq else 0
14
- for tok in text.split()]
15
-
16
  def decode(self, indices):
17
- return " ".join([self.ind_to_tok[idx] for idx in indices if idx in self.ind_to_tok and idx != 0])
18
 
19
  def save(self, path):
20
  with open(path, 'wb') as f:
21
- pickle.dump(self, f)
22
 
23
  @classmethod
24
  def load(cls, path):
 
25
  with open(path, 'rb') as f:
26
- return pickle.load(f)
 
 
 
1
  import pickle
2
 
3
  class CaptionTokenizer:
4
+ def __init__(self):
5
+ self.tok_to_ind = {'<PAD>': 0, '<BOS>': 1, '<EOS>': 2, '<UNK>': 3}
6
+ self.ind_to_tok = {v: k for k, v in self.tok_to_ind.items()}
 
 
 
7
 
8
  def encode(self, text):
9
+ return [self.tok_to_ind.get(word, 3) for word in text.split()]
10
+
 
11
  def decode(self, indices):
12
+ return ' '.join([self.ind_to_tok.get(idx, '<UNK>') for idx in indices])
13
 
14
  def save(self, path):
15
  with open(path, 'wb') as f:
16
+ pickle.dump(self.__dict__, f)
17
 
18
  @classmethod
19
  def load(cls, path):
20
+ tokenizer = cls()
21
  with open(path, 'rb') as f:
22
+ tokenizer.__dict__ = pickle.load(f)
23
+ return tokenizer