| { | |
| "vocab_size": 8192, | |
| "pre_tokenizer": "' ?[A-Za-z_][A-Za-z_0-9]*| ?[0-9]{1,3}|\\s+(?!\\S)|\\s+| ?[^A-Za-z_0-9\\s]+", | |
| "own": { | |
| "avg_tokens": 164.8715, | |
| "p50": 152, | |
| "p95": 278 | |
| }, | |
| "roundtrip_failures": 0, | |
| "gpt2": { | |
| "avg_tokens": 144.1685, | |
| "p50": 131, | |
| "p95": 248 | |
| } | |
| } |