File size: 1,753 Bytes
7b5e0a0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | {
"complete": true,
"samples": 37335232,
"tokens": 20500262423,
"max_len": 2048,
"unique": "exact-duplicate token sequences removed",
"sources": [
{
"name": "openhermes",
"kept": 995501,
"dups": 138,
"tokens": 459820675
},
{
"name": "tulu3",
"kept": 888089,
"dups": 13572,
"tokens": 603189885
},
{
"name": "openthoughts2",
"kept": 259093,
"dups": 0,
"tokens": 377825471
},
{
"name": "smoltalk",
"kept": 933756,
"dups": 101377,
"tokens": 873563095
},
{
"name": "openmath2",
"kept": 13914553,
"dups": 52433,
"tokens": 6441662552
},
{
"name": "webinstruct",
"kept": 2334508,
"dups": 44,
"tokens": 981812042
},
{
"name": "opencode",
"kept": 490943,
"dups": 55807,
"tokens": 477872658
},
{
"name": "infinity_7m",
"kept": 5871991,
"dups": 1516899,
"tokens": 2396828009
},
{
"name": "magpie",
"kept": 946437,
"dups": 52539,
"tokens": 799903974
},
{
"name": "opencodeinstruct",
"kept": 4974125,
"dups": 25637,
"tokens": 2686545810
},
{
"name": "opc_stage1",
"kept": 3235843,
"dups": 948864,
"tokens": 2291932185
},
{
"name": "opc_stage2",
"kept": 173388,
"dups": 260331,
"tokens": 65874877
},
{
"name": "codefeedback",
"kept": 12670,
"dups": 143267,
"tokens": 8034830
},
{
"name": "magicoder_evol",
"kept": 0,
"dups": 108780,
"tokens": 0
},
{
"name": "magicoder_oss",
"kept": 45571,
"dups": 29619,
"tokens": 24773193
},
{
"name": "infinity_gen",
"kept": 1325695,
"dups": 69975,
"tokens": 1141556193
},
{
"name": "orca_agent",
"kept": 933069,
"dups": 10410,
"tokens": 869066974
}
]
} |