{ "complete": true, "samples": 37335232, "tokens": 20500262423, "max_len": 2048, "unique": "exact-duplicate token sequences removed", "sources": [ { "name": "openhermes", "kept": 995501, "dups": 138, "tokens": 459820675 }, { "name": "tulu3", "kept": 888089, "dups": 13572, "tokens": 603189885 }, { "name": "openthoughts2", "kept": 259093, "dups": 0, "tokens": 377825471 }, { "name": "smoltalk", "kept": 933756, "dups": 101377, "tokens": 873563095 }, { "name": "openmath2", "kept": 13914553, "dups": 52433, "tokens": 6441662552 }, { "name": "webinstruct", "kept": 2334508, "dups": 44, "tokens": 981812042 }, { "name": "opencode", "kept": 490943, "dups": 55807, "tokens": 477872658 }, { "name": "infinity_7m", "kept": 5871991, "dups": 1516899, "tokens": 2396828009 }, { "name": "magpie", "kept": 946437, "dups": 52539, "tokens": 799903974 }, { "name": "opencodeinstruct", "kept": 4974125, "dups": 25637, "tokens": 2686545810 }, { "name": "opc_stage1", "kept": 3235843, "dups": 948864, "tokens": 2291932185 }, { "name": "opc_stage2", "kept": 173388, "dups": 260331, "tokens": 65874877 }, { "name": "codefeedback", "kept": 12670, "dups": 143267, "tokens": 8034830 }, { "name": "magicoder_evol", "kept": 0, "dups": 108780, "tokens": 0 }, { "name": "magicoder_oss", "kept": 45571, "dups": 29619, "tokens": 24773193 }, { "name": "infinity_gen", "kept": 1325695, "dups": 69975, "tokens": 1141556193 }, { "name": "orca_agent", "kept": 933069, "dups": 10410, "tokens": 869066974 } ] }