| { |
| "passed": true, |
| "verified": "2026-10-20T10:05:10.712512+00:00", |
| "vocab_size": 32000, |
| "byte_pieces": 256, |
| "special_ids": { |
| "unk": 0, |
| "bos": 1, |
| "eos": 2, |
| "pad": 3 |
| }, |
| "by_source": { |
| "kanripo": { |
| "samples": 128, |
| "characters": 244506, |
| "tokens": 197796, |
| "unknown_tokens": 0, |
| "byte_tokens": 13491, |
| "roundtrip_matches": 128, |
| "characters_per_token": 1.236152399441849, |
| "byte_token_fraction": 0.0682066371412971 |
| }, |
| "minhon": { |
| "samples": 128, |
| "characters": 38835, |
| "tokens": 26888, |
| "unknown_tokens": 0, |
| "byte_tokens": 3017, |
| "roundtrip_matches": 128, |
| "characters_per_token": 1.4443246057720915, |
| "byte_token_fraction": 0.11220618863433501 |
| }, |
| "ndl": { |
| "samples": 128, |
| "characters": 213716, |
| "tokens": 156139, |
| "unknown_tokens": 0, |
| "byte_tokens": 677, |
| "roundtrip_matches": 128, |
| "characters_per_token": 1.368754763383908, |
| "byte_token_fraction": 0.004335880209300687 |
| }, |
| "wikisource": { |
| "samples": 128, |
| "characters": 219399, |
| "tokens": 155118, |
| "unknown_tokens": 0, |
| "byte_tokens": 3874, |
| "roundtrip_matches": 128, |
| "characters_per_token": 1.414400649827873, |
| "byte_token_fraction": 0.024974535514898335 |
| }, |
| "_probes": { |
| "samples": 9, |
| "characters": 102, |
| "tokens": 126, |
| "unknown_tokens": 0, |
| "byte_tokens": 57, |
| "roundtrip_matches": 9, |
| "characters_per_token": 0.8095238095238095, |
| "byte_token_fraction": 0.4523809523809524 |
| } |
| }, |
| "failures": [], |
| "native_u2581_roundtrip": false, |
| "required_wrapper": "scripts/sentencepiece_unigram.py:Tokenizer for literal U+2581", |
| "evaluation_limit": "Diagnostic training-distribution sample, not held-out evaluation" |
| } |
|
|
