| { | |
| "audit_drain_documents": 0, | |
| "contract": "D:\\llm\\hummingbird\\configs\\flightmix_10b_balanced_corpus_contract.yaml", | |
| "contract_sha256": "73cd71ff4f7849d939606bd77434df4697d590bdfe2202ba1681b267c63f6786", | |
| "contract_validation": { | |
| "contract": "D:\\llm\\hummingbird\\configs\\flightmix_10b_balanced_corpus_contract.yaml", | |
| "curriculum": "constant_mixture", | |
| "pilot_only": false, | |
| "source_count": 9, | |
| "source_share_sum": 1.0, | |
| "source_tokens": { | |
| "code": 100000000, | |
| "cosmopedia_v2": 1500000000, | |
| "dense_pdf_exposition": 300000000, | |
| "general_web": 2050000000, | |
| "general_web_v2": 3450000000, | |
| "natural_text": 1000000000, | |
| "science_exposition": 400000000, | |
| "science_pdf_qa": 200000000, | |
| "worked_mathematics": 1000000000 | |
| }, | |
| "status": "passed", | |
| "train_tokens": 10000000000 | |
| }, | |
| "curriculum": [ | |
| { | |
| "document_boundary": true, | |
| "end_token": 10000000000, | |
| "per_source_tokens": { | |
| "code": 100000000, | |
| "cosmopedia_v2": 1500000000, | |
| "dense_pdf_exposition": 300000000, | |
| "general_web": 2050000000, | |
| "general_web_v2": 3450000000, | |
| "natural_text": 1000000000, | |
| "science_exposition": 400000000, | |
| "science_pdf_qa": 200000000, | |
| "worked_mathematics": 1000000000 | |
| }, | |
| "phase": "constant_mixture", | |
| "start_token": 0, | |
| "tokens": 10000000000 | |
| } | |
| ], | |
| "document_end_id": 8, | |
| "dtype": "uint16", | |
| "elapsed_seconds": 1166.1, | |
| "format_version": 5, | |
| "mix_policy": "three_phase_exact_token_curriculum", | |
| "per_source_documents_all_splits": { | |
| "code": { | |
| "held_out": 269, | |
| "train": 110098, | |
| "validation": 257 | |
| }, | |
| "cosmopedia_v2": { | |
| "held_out": 3687, | |
| "train": 1482589, | |
| "validation": 3700 | |
| }, | |
| "dense_pdf_exposition": { | |
| "held_out": 516, | |
| "train": 211198, | |
| "validation": 508 | |
| }, | |
| "general_web": { | |
| "held_out": 5209, | |
| "train": 2085750, | |
| "validation": 5245 | |
| }, | |
| "general_web_v2": { | |
| "held_out": 8644, | |
| "train": 3366937, | |
| "validation": 8545 | |
| }, | |
| "natural_text": { | |
| "held_out": 2344, | |
| "train": 927051, | |
| "validation": 2356 | |
| }, | |
| "science_exposition": { | |
| "held_out": 877, | |
| "train": 338512, | |
| "validation": 862 | |
| }, | |
| "science_pdf_qa": { | |
| "held_out": 1732, | |
| "train": 680086, | |
| "validation": 1688 | |
| }, | |
| "worked_mathematics": { | |
| "held_out": 2041, | |
| "train": 831431, | |
| "validation": 2117 | |
| } | |
| }, | |
| "per_source_tokens_all_splits": { | |
| "code": { | |
| "held_out": 243511, | |
| "train": 100000000, | |
| "validation": 250292 | |
| }, | |
| "cosmopedia_v2": { | |
| "held_out": 3750173, | |
| "train": 1500000000, | |
| "validation": 3750712 | |
| }, | |
| "dense_pdf_exposition": { | |
| "held_out": 751319, | |
| "train": 300000000, | |
| "validation": 750424 | |
| }, | |
| "general_web": { | |
| "held_out": 5123293, | |
| "train": 2050000000, | |
| "validation": 5125099 | |
| }, | |
| "general_web_v2": { | |
| "held_out": 8626828, | |
| "train": 3450000000, | |
| "validation": 8625843 | |
| }, | |
| "natural_text": { | |
| "held_out": 2500508, | |
| "train": 1000000000, | |
| "validation": 2501686 | |
| }, | |
| "science_exposition": { | |
| "held_out": 1001743, | |
| "train": 400000000, | |
| "validation": 995177 | |
| }, | |
| "science_pdf_qa": { | |
| "held_out": 500686, | |
| "train": 200000000, | |
| "validation": 500188 | |
| }, | |
| "worked_mathematics": { | |
| "held_out": 2501939, | |
| "train": 1000000000, | |
| "validation": 2500579 | |
| } | |
| }, | |
| "per_source_train_tokens": { | |
| "code": 100000000, | |
| "cosmopedia_v2": 1500000000, | |
| "dense_pdf_exposition": 300000000, | |
| "general_web": 2050000000, | |
| "general_web_v2": 3450000000, | |
| "natural_text": 1000000000, | |
| "science_exposition": 400000000, | |
| "science_pdf_qa": 200000000, | |
| "worked_mathematics": 1000000000 | |
| }, | |
| "phase_boundaries": [ | |
| 10000000000 | |
| ], | |
| "source_files": { | |
| "code": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_code.jsonl", | |
| "cosmopedia_v2": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_cosmopedia_v2.jsonl", | |
| "dense_pdf_exposition": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_dense_pdf_exposition.jsonl", | |
| "general_web": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_general_web.jsonl", | |
| "general_web_v2": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_general_web_v2.jsonl", | |
| "natural_text": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_natural_text.jsonl", | |
| "science_exposition": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_science_exposition.jsonl", | |
| "science_pdf_qa": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_science_pdf_qa.jsonl", | |
| "worked_mathematics": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\cleaned_worked_mathematics.jsonl" | |
| }, | |
| "source_order": [ | |
| "general_web", | |
| "general_web_v2", | |
| "cosmopedia_v2", | |
| "worked_mathematics", | |
| "natural_text", | |
| "science_exposition", | |
| "science_pdf_qa", | |
| "dense_pdf_exposition", | |
| "code" | |
| ], | |
| "source_rows_read": { | |
| "code": 110629, | |
| "cosmopedia_v2": 1489976, | |
| "dense_pdf_exposition": 212222, | |
| "general_web": 2096207, | |
| "general_web_v2": 3384126, | |
| "natural_text": 931751, | |
| "science_exposition": 340254, | |
| "science_pdf_qa": 683506, | |
| "worked_mathematics": 835589 | |
| }, | |
| "split_policy": { | |
| "hash": "sha256_first_8_bytes_big_endian", | |
| "held_out_buckets": [ | |
| 5, | |
| 6, | |
| 7, | |
| 8, | |
| 9 | |
| ], | |
| "modulus": 1000, | |
| "validation_buckets": [ | |
| 0, | |
| 1, | |
| 2, | |
| 3, | |
| 4 | |
| ] | |
| }, | |
| "splits": { | |
| "held_out": { | |
| "document_ends_file": "held_out_document_ends.npy", | |
| "documents": 25319, | |
| "quota_tail_fragments": 1, | |
| "token_file": "held_out_tokens.uint16", | |
| "tokens": 25000000 | |
| }, | |
| "train": { | |
| "document_ends_file": "train_document_ends.npy", | |
| "documents": 10033652, | |
| "quota_tail_fragments": 9, | |
| "token_file": "train_tokens.uint16", | |
| "tokens": 10000000000 | |
| }, | |
| "validation": { | |
| "document_ends_file": "validation_document_ends.npy", | |
| "documents": 25278, | |
| "quota_tail_fragments": 1, | |
| "token_file": "validation_tokens.uint16", | |
| "tokens": 25000000 | |
| } | |
| }, | |
| "tokenized_source_files": { | |
| "code": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_code.uint16", | |
| "cosmopedia_v2": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_cosmopedia_v2.uint16", | |
| "dense_pdf_exposition": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_dense_pdf_exposition.uint16", | |
| "general_web": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_general_web.uint16", | |
| "general_web_v2": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_general_web_v2.uint16", | |
| "natural_text": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_natural_text.uint16", | |
| "science_exposition": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_science_exposition.uint16", | |
| "science_pdf_qa": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_science_pdf_qa.uint16", | |
| "worked_mathematics": "E:\\hummingbird\\flightmix_10b_balanced\\materialized\\tokenized_worked_mathematics.uint16" | |
| }, | |
| "tokenizer_path": "D:\\llm\\hummingbird\\artifacts\\runs\\e3_tokenizers\\tok_4k_digit", | |
| "tokenizer_sha256": "a788340e936445a7c6a463b11e87f8025be1e63c2fd47a072901328d08ef8818" | |
| } | |