| { | |
| "mode": "pretokenized", | |
| "sampled_sequences": 20000, | |
| "total_sequences": 483367, | |
| "unk_rate": 3.4e-05, | |
| "total_unk_tokens": 340, | |
| "seq_len": { | |
| "mean": 494.9529, | |
| "p50": 512.0, | |
| "p95": 512.0, | |
| "max": 512 | |
| }, | |
| "share_over_max_seq_len": 0.0, | |
| "starts_with_bos_rate": 1.0, | |
| "ends_with_eos_rate": 1.0 | |
| } |