| { | |
| "byte": { | |
| "english": { | |
| "characters": 77, | |
| "bytes": 77, | |
| "tokens": 77, | |
| "characters_per_token": 1.0, | |
| "roundtrip": true | |
| }, | |
| "hindi": { | |
| "characters": 72, | |
| "bytes": 188, | |
| "tokens": 188, | |
| "characters_per_token": 0.3829787234042553, | |
| "roundtrip": true | |
| }, | |
| "hinglish": { | |
| "characters": 70, | |
| "bytes": 70, | |
| "tokens": 70, | |
| "characters_per_token": 1.0, | |
| "roundtrip": true | |
| }, | |
| "python": { | |
| "characters": 70, | |
| "bytes": 70, | |
| "tokens": 70, | |
| "characters_per_token": 1.0, | |
| "roundtrip": true | |
| }, | |
| "json": { | |
| "characters": 61, | |
| "bytes": 61, | |
| "tokens": 61, | |
| "characters_per_token": 1.0, | |
| "roundtrip": true | |
| }, | |
| "math": { | |
| "characters": 45, | |
| "bytes": 57, | |
| "tokens": 57, | |
| "characters_per_token": 0.7894736842105263, | |
| "roundtrip": true | |
| }, | |
| "shell": { | |
| "characters": 38, | |
| "bytes": 38, | |
| "tokens": 38, | |
| "characters_per_token": 1.0, | |
| "roundtrip": true | |
| }, | |
| "xml_url": { | |
| "characters": 66, | |
| "bytes": 67, | |
| "tokens": 67, | |
| "characters_per_token": 0.9850746268656716, | |
| "roundtrip": true | |
| } | |
| }, | |
| "qwen_bpe": { | |
| "english": { | |
| "tokens": 13, | |
| "characters_per_token": 5.923076923076923, | |
| "roundtrip": true | |
| }, | |
| "hindi": { | |
| "tokens": 48, | |
| "characters_per_token": 1.5, | |
| "roundtrip": true | |
| }, | |
| "hinglish": { | |
| "tokens": 21, | |
| "characters_per_token": 3.3333333333333335, | |
| "roundtrip": true | |
| }, | |
| "python": { | |
| "tokens": 20, | |
| "characters_per_token": 3.5, | |
| "roundtrip": true | |
| }, | |
| "json": { | |
| "tokens": 14, | |
| "characters_per_token": 4.357142857142857, | |
| "roundtrip": true | |
| }, | |
| "math": { | |
| "tokens": 35, | |
| "characters_per_token": 1.2857142857142858, | |
| "roundtrip": true | |
| }, | |
| "shell": { | |
| "tokens": 11, | |
| "characters_per_token": 3.4545454545454546, | |
| "roundtrip": true | |
| }, | |
| "xml_url": { | |
| "tokens": 21, | |
| "characters_per_token": 3.142857142857143, | |
| "roundtrip": true | |
| } | |
| }, | |
| "limitation": "Eight public examples, not a representative benchmark or vocabulary-size study" | |
| } |