File size: 7,849 Bytes
ebd2f40 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 | {
"audit_drain_documents": 0,
"contract": "D:\\llm\\hummingbird\\configs\\flightmix_10b_balanced_corpus_contract.yaml",
"contract_sha256": "73cd71ff4f7849d939606bd77434df4697d590bdfe2202ba1681b267c63f6786",
"contract_validation": {
"contract": "D:\\llm\\hummingbird\\configs\\flightmix_10b_balanced_corpus_contract.yaml",
"curriculum": "constant_mixture",
"pilot_only": false,
"source_count": 9,
"source_share_sum": 1.0,
"source_tokens": {
"code": 100000000,
"cosmopedia_v2": 1500000000,
"dense_pdf_exposition": 300000000,
"general_web": 2050000000,
"general_web_v2": 3450000000,
"natural_text": 1000000000,
"science_exposition": 400000000,
"science_pdf_qa": 200000000,
"worked_mathematics": 1000000000
},
"status": "passed",
"train_tokens": 10000000000
},
"curriculum": [
{
"document_boundary": true,
"end_token": 10000000000,
"per_source_tokens": {
"code": 100000000,
"cosmopedia_v2": 1500000000,
"dense_pdf_exposition": 300000000,
"general_web": 2050000000,
"general_web_v2": 3450000000,
"natural_text": 1000000000,
"science_exposition": 400000000,
"science_pdf_qa": 200000000,
"worked_mathematics": 1000000000
},
"phase": "constant_mixture",
"start_token": 0,
"tokens": 10000000000
}
],
"document_end_id": 8,
"dtype": "uint16",
"elapsed_seconds": 1166.1,
"format_version": 5,
"mix_policy": "three_phase_exact_token_curriculum",
"per_source_documents_all_splits": {
"code": {
"held_out": 269,
"train": 110098,
"validation": 257
},
"cosmopedia_v2": {
"held_out": 3687,
"train": 1482589,
"validation": 3700
},
"dense_pdf_exposition": {
"held_out": 516,
"train": 211198,
"validation": 508
},
"general_web": {
"held_out": 5209,
"train": 2085750,
"validation": 5245
},
"general_web_v2": {
"held_out": 8644,
"train": 3366937,
"validation": 8545
},
"natural_text": {
"held_out": 2344,
"train": 927051,
"validation": 2356
},
"science_exposition": {
"held_out": 877,
"train": 338512,
"validation": 862
},
"science_pdf_qa": {
"held_out": 1732,
"train": 680086,
"validation": 1688
},
"worked_mathematics": {
"held_out": 2041,
"train": 831431,
"validation": 2117
}
},
"per_source_tokens_all_splits": {
"code": {
"held_out": 243511,
"train": 100000000,
"validation": 250292
},
"cosmopedia_v2": {
"held_out": 3750173,
"train": 1500000000,
"validation": 3750712
},
"dense_pdf_exposition": {
"held_out": 751319,
"train": 300000000,
"validation": 750424
},
"general_web": {
"held_out": 5123293,
"train": 2050000000,
"validation": 5125099
},
"general_web_v2": {
"held_out": 8626828,
"train": 3450000000,
"validation": 8625843
},
"natural_text": {
"held_out": 2500508,
"train": 1000000000,
"validation": 2501686
},
"science_exposition": {
"held_out": 1001743,
"train": 400000000,
"validation": 995177
},
"science_pdf_qa": {
"held_out": 500686,
"train": 200000000,
"validation": 500188
},
"worked_mathematics": {
"held_out": 2501939,
"train": 1000000000,
"validation": 2500579
}
},
"per_source_train_tokens": {
"code": 100000000,
"cosmopedia_v2": 1500000000,
"dense_pdf_exposition": 300000000,
"general_web": 2050000000,
"general_web_v2": 3450000000,
"natural_text": 1000000000,
"science_exposition": 400000000,
"science_pdf_qa": 200000000,
"worked_mathematics": 1000000000
},
"phase_boundaries": [
10000000000
],
"source_files": {
"code": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_code.jsonl",
"cosmopedia_v2": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_cosmopedia_v2.jsonl",
"dense_pdf_exposition": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_dense_pdf_exposition.jsonl",
"general_web": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_general_web.jsonl",
"general_web_v2": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_general_web_v2.jsonl",
"natural_text": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_natural_text.jsonl",
"science_exposition": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_science_exposition.jsonl",
"science_pdf_qa": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_science_pdf_qa.jsonl",
"worked_mathematics": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_worked_mathematics.jsonl"
},
"source_order": [
"general_web",
"general_web_v2",
"cosmopedia_v2",
"worked_mathematics",
"natural_text",
"science_exposition",
"science_pdf_qa",
"dense_pdf_exposition",
"code"
],
"source_rows_read": {
"code": 110629,
"cosmopedia_v2": 1489976,
"dense_pdf_exposition": 212222,
"general_web": 2096207,
"general_web_v2": 3384126,
"natural_text": 931751,
"science_exposition": 340254,
"science_pdf_qa": 683506,
"worked_mathematics": 835589
},
"split_policy": {
"hash": "sha256_first_8_bytes_big_endian",
"held_out_buckets": [
5,
6,
7,
8,
9
],
"modulus": 1000,
"validation_buckets": [
0,
1,
2,
3,
4
]
},
"splits": {
"held_out": {
"document_ends_file": "held_out_document_ends.npy",
"documents": 25319,
"quota_tail_fragments": 1,
"token_file": "held_out_tokens.uint16",
"tokens": 25000000
},
"train": {
"document_ends_file": "train_document_ends.npy",
"documents": 10033652,
"quota_tail_fragments": 9,
"token_file": "train_tokens.uint16",
"tokens": 10000000000
},
"validation": {
"document_ends_file": "validation_document_ends.npy",
"documents": 25278,
"quota_tail_fragments": 1,
"token_file": "validation_tokens.uint16",
"tokens": 25000000
}
},
"tokenized_source_files": {
"code": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_code.uint16",
"cosmopedia_v2": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_cosmopedia_v2.uint16",
"dense_pdf_exposition": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_dense_pdf_exposition.uint16",
"general_web": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_general_web.uint16",
"general_web_v2": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_general_web_v2.uint16",
"natural_text": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_natural_text.uint16",
"science_exposition": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_science_exposition.uint16",
"science_pdf_qa": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_science_pdf_qa.uint16",
"worked_mathematics": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_worked_mathematics.uint16"
},
"tokenizer_path": "D:\\llm\\hummingbird\\artifacts\\runs\\e3_tokenizers\\tok_4k_digit",
"tokenizer_sha256": "a788340e936445a7c6a463b11e87f8025be1e63c2fd47a072901328d08ef8818"
}
|