File size: 643 Bytes
aa3647a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | {
"version": "1.0",
"model": "BPE",
"vocab": {
"<|begoftext|>": 0,
"<|endoftext|>": 1,
"<pad>": 2,
"<unk>": 3,
"the": 4,
"a": 5,
"and": 6,
"to": 7,
"of": 8,
"in": 9,
"I": 10,
"is": 11,
"it": 12,
".": 13,
",": 14,
"th": 15,
"an": 16,
"ing": 17,
"er": 18,
"on": 19
},
"merges": [
"t h",
"th e",
"a n",
"a nd",
"i n",
"o f",
"to k",
"i s",
"in g",
"e r",
"o n"
],
"special_tokens": {
"pad_token": "<pad>",
"bos_token": "<|begoftext|>",
"eos_token": "<|endoftext|>",
"unk_token": "<unk>"
}
} |