| { |
| "selected": { |
| "vocab": 24576, |
| "ffn": 2496, |
| "parameters": 149333081, |
| "weighted_tokens_per_byte": 0.21982151683772885, |
| "measurements": { |
| "fineweb_edu": { |
| "tokens": 108258, |
| "bytes": 500330, |
| "bytes_per_token": 4.621644589776275 |
| }, |
| "dclm_edu": { |
| "tokens": 113785, |
| "bytes": 502955, |
| "bytes_per_token": 4.42022234916729 |
| }, |
| "cosmopedia": { |
| "tokens": 96667, |
| "bytes": 500651, |
| "bytes_per_token": 5.179130416791666 |
| }, |
| "finemath": { |
| "tokens": 149111, |
| "bytes": 509813, |
| "bytes_per_token": 3.4190167056756375 |
| }, |
| "edge_cases": { |
| "tokens": 67, |
| "bytes": 130, |
| "bytes_per_token": 1.9402985074626866 |
| } |
| }, |
| "fit_seconds": 6.842154794954695 |
| }, |
| "candidates": [ |
| { |
| "vocab": 16384, |
| "ffn": 2592, |
| "parameters": 148927577, |
| "weighted_tokens_per_byte": 0.2296576476963851, |
| "measurements": { |
| "fineweb_edu": { |
| "tokens": 113155, |
| "bytes": 500330, |
| "bytes_per_token": 4.421634041801069 |
| }, |
| "dclm_edu": { |
| "tokens": 118553, |
| "bytes": 502955, |
| "bytes_per_token": 4.242448525132219 |
| }, |
| "cosmopedia": { |
| "tokens": 101369, |
| "bytes": 500651, |
| "bytes_per_token": 4.938896506821612 |
| }, |
| "finemath": { |
| "tokens": 155627, |
| "bytes": 509813, |
| "bytes_per_token": 3.275864727842855 |
| }, |
| "edge_cases": { |
| "tokens": 72, |
| "bytes": 130, |
| "bytes_per_token": 1.8055555555555556 |
| } |
| }, |
| "fit_seconds": 9.067338421940804 |
| }, |
| { |
| "vocab": 24576, |
| "ffn": 2496, |
| "parameters": 149333081, |
| "weighted_tokens_per_byte": 0.21982151683772885, |
| "measurements": { |
| "fineweb_edu": { |
| "tokens": 108258, |
| "bytes": 500330, |
| "bytes_per_token": 4.621644589776275 |
| }, |
| "dclm_edu": { |
| "tokens": 113785, |
| "bytes": 502955, |
| "bytes_per_token": 4.42022234916729 |
| }, |
| "cosmopedia": { |
| "tokens": 96667, |
| "bytes": 500651, |
| "bytes_per_token": 5.179130416791666 |
| }, |
| "finemath": { |
| "tokens": 149111, |
| "bytes": 509813, |
| "bytes_per_token": 3.4190167056756375 |
| }, |
| "edge_cases": { |
| "tokens": 67, |
| "bytes": 130, |
| "bytes_per_token": 1.9402985074626866 |
| } |
| }, |
| "fit_seconds": 6.842154794954695 |
| }, |
| { |
| "vocab": 32768, |
| "ffn": 2384, |
| "parameters": 149019737, |
| "weighted_tokens_per_byte": 0.2144127319251325, |
| "measurements": { |
| "fineweb_edu": { |
| "tokens": 105600, |
| "bytes": 500330, |
| "bytes_per_token": 4.737973484848485 |
| }, |
| "dclm_edu": { |
| "tokens": 111026, |
| "bytes": 502955, |
| "bytes_per_token": 4.530065029812836 |
| }, |
| "cosmopedia": { |
| "tokens": 94072, |
| "bytes": 500651, |
| "bytes_per_token": 5.321998044051365 |
| }, |
| "finemath": { |
| "tokens": 145642, |
| "bytes": 509813, |
| "bytes_per_token": 3.50045316598234 |
| }, |
| "edge_cases": { |
| "tokens": 63, |
| "bytes": 130, |
| "bytes_per_token": 2.0634920634920637 |
| } |
| }, |
| "fit_seconds": 7.331924270954914 |
| } |
| ], |
| "criterion": "Smallest vocabulary within 3% of best weighted held-out token compression; provisional pending speed check.", |
| "tokenizer_training_sha256": "c15bad040cf3eaf9e0053539bff3039601f9a4db7e89a0acd6db0e03e4cced49", |
| "tokenizer_sha256": "d1ffe47f6fd8cc19ba7c3e5d2537699d6a99d09b584d3319944f4e40fe6c44ea", |
| "normalization": "none; byte fallback alphabet; preserve Unicode and whitespace; digit groups max3", |
| "parameters_cap": 149500000, |
| "data_mix": { |
| "fineweb_edu": 0.65, |
| "dclm_edu": 0.2, |
| "cosmopedia": 0.1, |
| "finemath": 0.05 |
| }, |
| "status": "roundtrip_passed" |
| } |
