File size: 300 Bytes
3737a7c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 | {
"vocab_size": 8192,
"pre_tokenizer": "' ?[A-Za-z_][A-Za-z_0-9]*| ?[0-9]{1,3}|\\s+(?!\\S)|\\s+| ?[^A-Za-z_0-9\\s]+",
"own": {
"avg_tokens": 164.8715,
"p50": 152,
"p95": 278
},
"roundtrip_failures": 0,
"gpt2": {
"avg_tokens": 144.1685,
"p50": 131,
"p95": 248
}
} |