| { | |
| "boundary_policy": "raw-text-then-one-document-EOS-v1", | |
| "corpus_fingerprint": "a89f69d6f63cfa4385da233693a445404234a18194170fe1d2b4a74bf501f5d6", | |
| "format": "sml-v2-tokenizer-v1", | |
| "recipe": { | |
| "corpus": "a89f69d6f63cfa4385da233693a445404234a18194170fe1d2b4a74bf501f5d6", | |
| "min_frequency": 2, | |
| "sample_manifest": "30ead6f502f11e99b479e3711a81ce7198787408d0700153615fc3255b244f46", | |
| "specials": [ | |
| "<|pad|>", | |
| "<|doc_end|>", | |
| "<|turn_start|>", | |
| "<|turn_end|>" | |
| ], | |
| "tokenizers": "0.22.2", | |
| "version": 1, | |
| "vocab_size": 32000 | |
| }, | |
| "sha256": { | |
| "corpus.json": "e4c47b7b6aba08f948de7c2c7cb94c58dbd09dde2ded6ac185c60856ee686d27", | |
| "tokenizer.json": "e095f0413a11fa91fbb58f37d00e4c6bcda59beb89b28f6f680db1436a7e8891", | |
| "tokenizer_config.json": "619f500553367eb76f5be00a98625130ec857b9c367561334984096c029ba2a9" | |
| }, | |
| "special_ids": { | |
| "eos": 1, | |
| "pad": 0, | |
| "turn_end": 3, | |
| "turn_start": 2 | |
| }, | |
| "vocab_size": 32000 | |
| } | |