Delete tokenizer
Browse files
tokenizer/tokenizer.pkl
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:0ca760f09eb2dd9e980654f61e75867c04f1a9afd9245a031a0fa50025b2bc88
|
| 3 |
-
size 147083
|
|
|
|
|
|
|
|
|
|
|
|
tokenizer/tokenizer_config.json
DELETED
|
@@ -1,7 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"vocab_size": 12288,
|
| 3 |
-
"bos_id": 12287,
|
| 4 |
-
"eos_id": 12287,
|
| 5 |
-
"pad_id": 12287,
|
| 6 |
-
"unk_id": 12287
|
| 7 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|