disease_detection / data /tokenizer /tokenizer.json
razz-123's picture
Upload folder using huggingface_hub
042eb00 verified
Raw History Blame Contribute Delete
2.71 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<PAD>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "<UNK>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "<BOS>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "<EOS>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"post_processor": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": null,
"end_of_word_suffix": null,
"fuse_unk": false,
"byte_fallback": false,
"ignore_merges": false,
"vocab": {
"<PAD>": 0,
"<UNK>": 1,
"<BOS>": 2,
"<EOS>": 3,
".": 4,
"T": 5,
"a": 6,
"d": 7,
"e": 8,
"g": 9,
"h": 10,
"i": 11,
"l": 12,
"m": 13,
"n": 14,
"o": 15,
"p": 16,
"r": 17,
"s": 18,
"t": 19,
"v": 20,
"x": 21,
"»": 22,
"¿": 23,
"ï": 24,
"Ċ": 25,
"č": 26,
"Ġ": 27,
"is": 28,
"Ġt": 29,
"Th": 30,
"ex": 31,
"in": 32,
"»¿": 33,
"": 34,
"čĊ": 35,
"Ġis": 36,
"Ġ": 37,
"Ġtex": 38,
"This": 39,
"Ġtext": 40
},
"merges": [
[
"i",
"s"
],
[
"Ġ",
"t"
],
[
"T",
"h"
],
[
"e",
"x"
],
[
"i",
"n"
],
[
"»",
"¿"
],
[
"ï",
"»¿"
],
[
"č",
"Ċ"
],
[
"Ġ",
"is"
],
[
"Ġ",
""
],
[
"Ġt",
"ex"
],
[
"Th",
"is"
],
[
"Ġtex",
"t"
]
]
}
}