小能豆子 commited on
Commit
2c31f17
·
verified ·
1 Parent(s): 4ad863d

Upload 6 files

Browse files
Files changed (5) hide show
  1. README.md +33 -0
  2. config.json +49 -0
  3. generation_config.json +6 -0
  4. model.safetensors +3 -0
  5. tokenizer.json +277 -0
README.md ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: bsd-3-clause
3
+ ---
4
+ Mirror of the base ProGen2-small model (with slightly modified configuration and forward pass) by [Nijkamp, et al.](https://arxiv.org/abs/2206.13517).
5
+
6
+ See also my github [repo](https://github.com/hugohrban/ProGen2-finetuning/tree/main) for an example of finetuning this model.
7
+
8
+ Example usage:
9
+
10
+ ```python
11
+ from transformers import AutoModelForCausalLM
12
+ from tokenizers import Tokenizer
13
+ import torch
14
+ import torch.nn.functional as F
15
+
16
+ # load model and tokenizer
17
+ model = AutoModelForCausalLM.from_pretrained("hugohrban/progen2-small", trust_remote_code=True)
18
+ tokenizer = Tokenizer.from_pretrained("hugohrban/progen2-small")
19
+ tokenizer.no_padding()
20
+
21
+ # prepare input
22
+ prompt = "1MEVVIVTGMSGAGK"
23
+ input_ids = torch.tensor(tokenizer.encode(prompt).ids).to(model.device)
24
+
25
+ # forward pass
26
+ logits = model(input_ids).logits
27
+
28
+ # print output probabilities
29
+ next_token_logits = logits[-1, :]
30
+ next_token_probs = F.softmax(next_token_logits, dim=-1)
31
+ for i in range(tokenizer.get_vocab_size(with_added_tokens=False)):
32
+ print(f"{tokenizer.id_to_token(i)}: {100 * next_token_probs[i].item():.2f} %")
33
+ ```
config.json ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "hugohrban/progen2-small",
3
+ "activation_function": "gelu_new",
4
+ "architectures": [
5
+ "ProGenForCausalLM"
6
+ ],
7
+ "attn_pdrop": 0.0,
8
+ "auto_map": {
9
+ "AutoConfig": "configuration_progen.ProGenConfig",
10
+ "AutoModelForCausalLM": "modeling_progen.ProGenForCausalLM"
11
+ },
12
+ "bos_token_id": 1,
13
+ "embd_pdrop": 0.0,
14
+ "embed_dim": 1024,
15
+ "eos_token_id": 2,
16
+ "gradient_checkpointing": false,
17
+ "initializer_range": 0.02,
18
+ "layer_norm_epsilon": 1e-05,
19
+ "model_type": "progen",
20
+ "n_embd": 1024,
21
+ "n_head": 16,
22
+ "n_inner": null,
23
+ "n_layer": 12,
24
+ "n_positions": 1024,
25
+ "resid_pdrop": 0.0,
26
+ "rotary_dim": 44,
27
+ "scale_attn_weights": true,
28
+ "summary_activation": null,
29
+ "summary_first_dropout": 0.1,
30
+ "summary_proj_to_labels": true,
31
+ "summary_type": "cls_index",
32
+ "summary_use_proj": true,
33
+ "task_specific_params": {
34
+ "text-generation": {
35
+ "do_sample": true,
36
+ "max_length": 50,
37
+ "temperature": 1.0
38
+ }
39
+ },
40
+ "tokenizer_class": "GPT2Tokenizer",
41
+ "torch_dtype": "float32",
42
+ "transformers_version": "4.40.0",
43
+ "use_cache": true,
44
+
45
+ "vocab_size": 44,
46
+ "vocab_size_emb": 32,
47
+ "vocab_size_lm_head": 32
48
+
49
+ }
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "transformers_version": "4.40.0"
6
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12b6d31fa3127e4c68ca488a322980e39cb3e1d406d6362d2d8fe273725bff79
3
+ size 604604816
tokenizer.json ADDED
@@ -0,0 +1,277 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [
6
+ {
7
+ "id": 0,
8
+ "special": true,
9
+ "content": "<|pad|>",
10
+ "single_word": false,
11
+ "lstrip": false,
12
+ "rstrip": false,
13
+ "normalized": false
14
+ },
15
+ {
16
+ "id": 1,
17
+ "special": true,
18
+ "content": "<|bos|>",
19
+ "single_word": false,
20
+ "lstrip": false,
21
+ "rstrip": false,
22
+ "normalized": false
23
+ },
24
+ {
25
+ "id": 2,
26
+ "special": true,
27
+ "content": "<|eos|>",
28
+ "single_word": false,
29
+ "lstrip": false,
30
+ "rstrip": false,
31
+ "normalized": false
32
+ },
33
+ {
34
+ "id": 25,
35
+ "special": true,
36
+ "content": "<|antifungal|>",
37
+ "single_word": false,
38
+ "lstrip": false,
39
+ "rstrip": false,
40
+ "normalized": false
41
+ },
42
+ {
43
+ "id": 26,
44
+ "special": true,
45
+ "content": "<|antibacterial|>",
46
+ "single_word": false,
47
+ "lstrip": false,
48
+ "rstrip": false,
49
+ "normalized": false
50
+ },
51
+ {
52
+ "id": 27,
53
+ "special": true,
54
+ "content": "<|antibiofilm|>",
55
+ "single_word": false,
56
+ "lstrip": false,
57
+ "rstrip": false,
58
+ "normalized": false
59
+ },
60
+ {
61
+ "id": 28,
62
+ "special": true,
63
+ "content": "<|anti-diabetic|>",
64
+ "single_word": false,
65
+ "lstrip": false,
66
+ "rstrip": false,
67
+ "normalized": false
68
+ },
69
+ {
70
+ "id": 29,
71
+ "special": true,
72
+ "content": "<|anti-MRSA|>",
73
+ "single_word": false,
74
+ "lstrip": false,
75
+ "rstrip": false,
76
+ "normalized": false
77
+ },
78
+ {
79
+ "id": 30,
80
+ "special": true,
81
+ "content": "<|anti-parasite|>",
82
+ "single_word": false,
83
+ "lstrip": false,
84
+ "rstrip": false,
85
+ "normalized": false
86
+ },
87
+ {
88
+ "id": 31,
89
+ "special": true,
90
+ "content": "<|anti-TB|>",
91
+ "single_word": false,
92
+ "lstrip": false,
93
+ "rstrip": false,
94
+ "normalized": false
95
+ },
96
+ {
97
+ "id": 32,
98
+ "special": true,
99
+ "content": "<|anti-toxin|>",
100
+ "single_word": false,
101
+ "lstrip": false,
102
+ "rstrip": false,
103
+ "normalized": false
104
+ },
105
+ {
106
+ "id": 33,
107
+ "special": true,
108
+ "content": "<|anticancer|>",
109
+ "single_word": false,
110
+ "lstrip": false,
111
+ "rstrip": false,
112
+ "normalized": false
113
+ },
114
+ {
115
+ "id": 34,
116
+ "special": true,
117
+ "content": "<|antiHIV|>",
118
+ "single_word": false,
119
+ "lstrip": false,
120
+ "rstrip": false,
121
+ "normalized": false
122
+ },
123
+ {
124
+ "id": 35,
125
+ "special": true,
126
+ "content": "<|antimalarial|>",
127
+ "single_word": false,
128
+ "lstrip": false,
129
+ "rstrip": false,
130
+ "normalized": false
131
+ },
132
+ {
133
+ "id": 36,
134
+ "special": true,
135
+ "content": "<|antioxidant|>",
136
+ "single_word": false,
137
+ "lstrip": false,
138
+ "rstrip": false,
139
+ "normalized": false
140
+ },
141
+ {
142
+ "id": 37,
143
+ "special": true,
144
+ "content": "<|antiviral|>",
145
+ "single_word": false,
146
+ "lstrip": false,
147
+ "rstrip": false,
148
+ "normalized": false
149
+ },
150
+ {
151
+ "id": 38,
152
+ "special": true,
153
+ "content": "<|insecticidal|>",
154
+ "single_word": false,
155
+ "lstrip": false,
156
+ "rstrip": false,
157
+ "normalized": false
158
+ },
159
+ {
160
+ "id": 39,
161
+ "special": true,
162
+ "content": "<|ion_channel|>",
163
+ "single_word": false,
164
+ "lstrip": false,
165
+ "rstrip": false,
166
+ "normalized": false
167
+ },
168
+ {
169
+ "id": 40,
170
+ "special": true,
171
+ "content": "<|protease_inhibitors|>",
172
+ "single_word": false,
173
+ "lstrip": false,
174
+ "rstrip": false,
175
+ "normalized": false
176
+ },
177
+ {
178
+ "id": 41,
179
+ "special": true,
180
+ "content": "<|spermicidal|>",
181
+ "single_word": false,
182
+ "lstrip": false,
183
+ "rstrip": false,
184
+ "normalized": false
185
+ },
186
+ {
187
+ "id": 42,
188
+ "special": true,
189
+ "content": "<|surface_immobilized|>",
190
+ "single_word": false,
191
+ "lstrip": false,
192
+ "rstrip": false,
193
+ "normalized": false
194
+ },
195
+ {
196
+ "id": 43,
197
+ "special": true,
198
+ "content": "<|wound_healing|>",
199
+ "single_word": false,
200
+ "lstrip": false,
201
+ "rstrip": false,
202
+ "normalized": false
203
+ }
204
+
205
+ ],
206
+ "normalizer": null,
207
+ "pre_tokenizer": {
208
+ "type": "ByteLevel",
209
+ "add_prefix_space": false,
210
+ "trim_offsets": true
211
+ },
212
+ "post_processor": {
213
+ "type": "ByteLevel",
214
+ "add_prefix_space": true,
215
+ "trim_offsets": true
216
+ },
217
+ "decoder": {
218
+ "type": "ByteLevel",
219
+ "add_prefix_space": true,
220
+ "trim_offsets": true
221
+ },
222
+ "model": {
223
+ "type": "BPE",
224
+ "dropout": null,
225
+ "unk_token": null,
226
+ "continuing_subword_prefix": null,
227
+ "end_of_word_suffix": null,
228
+ "fuse_unk": false,
229
+ "vocab": {
230
+ "<|pad|>": 0,
231
+ "<|bos|>": 1,
232
+ "<|eos|>": 2,
233
+ "1": 3,
234
+ "2": 4,
235
+ "A": 5,
236
+ "C": 6,
237
+ "D": 7,
238
+ "E": 8,
239
+ "F": 9,
240
+ "G": 10,
241
+ "H": 11,
242
+ "I": 12,
243
+ "K": 13,
244
+ "L": 14,
245
+ "M": 15,
246
+ "N": 16,
247
+ "P": 17,
248
+ "Q": 18,
249
+ "R": 19,
250
+ "S": 20,
251
+ "T": 21,
252
+ "V": 22,
253
+ "W": 23,
254
+ "Y": 24,
255
+ "<|antifungal|>": 25,
256
+ "<|antibacterial|>": 26,
257
+ "<|antibiofilm|>": 27,
258
+ "<|anti-diabetic|>": 28,
259
+ "<|anti-MRSA|>": 29,
260
+ "<|anti-parasite|>": 30,
261
+ "<|anti-TB|>": 31,
262
+ "<|anti-toxin|>": 32,
263
+ "<|anticancer|>": 33,
264
+ "<|antiHIV|>": 34,
265
+ "<|antimalarial|>": 35,
266
+ "<|antioxidant|>": 36,
267
+ "<|antiviral|>": 37,
268
+ "<|insecticidal|>": 38,
269
+ "<|ion_channel|>": 39,
270
+ "<|protease_inhibitors|>": 40,
271
+ "<|spermicidal|>": 41,
272
+ "<|surface_immobilized|>": 42,
273
+ "<|wound_healing|>": 43
274
+ },
275
+ "merges": []
276
+ }
277
+ }