File size: 988 Bytes
8f2ec8e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | {
"boundary_policy": "raw-text-then-one-document-EOS-v1",
"corpus_fingerprint": "a89f69d6f63cfa4385da233693a445404234a18194170fe1d2b4a74bf501f5d6",
"format": "sml-v2-tokenizer-v1",
"recipe": {
"corpus": "a89f69d6f63cfa4385da233693a445404234a18194170fe1d2b4a74bf501f5d6",
"min_frequency": 2,
"sample_manifest": "30ead6f502f11e99b479e3711a81ce7198787408d0700153615fc3255b244f46",
"specials": [
"<|pad|>",
"<|doc_end|>",
"<|turn_start|>",
"<|turn_end|>"
],
"tokenizers": "0.22.2",
"version": 1,
"vocab_size": 32000
},
"sha256": {
"corpus.json": "e4c47b7b6aba08f948de7c2c7cb94c58dbd09dde2ded6ac185c60856ee686d27",
"tokenizer.json": "e095f0413a11fa91fbb58f37d00e4c6bcda59beb89b28f6f680db1436a7e8891",
"tokenizer_config.json": "619f500553367eb76f5be00a98625130ec857b9c367561334984096c029ba2a9"
},
"special_ids": {
"eos": 1,
"pad": 0,
"turn_end": 3,
"turn_start": 2
},
"vocab_size": 32000
}
|