NiftyOracleAI's picture
Upload folder using huggingface_hub
7b5e0a0 verified
Raw History Blame Contribute Delete
1.75 kB
{
"complete": true,
"samples": 37335232,
"tokens": 20500262423,
"max_len": 2048,
"unique": "exact-duplicate token sequences removed",
"sources": [
{
"name": "openhermes",
"kept": 995501,
"dups": 138,
"tokens": 459820675
},
{
"name": "tulu3",
"kept": 888089,
"dups": 13572,
"tokens": 603189885
},
{
"name": "openthoughts2",
"kept": 259093,
"dups": 0,
"tokens": 377825471
},
{
"name": "smoltalk",
"kept": 933756,
"dups": 101377,
"tokens": 873563095
},
{
"name": "openmath2",
"kept": 13914553,
"dups": 52433,
"tokens": 6441662552
},
{
"name": "webinstruct",
"kept": 2334508,
"dups": 44,
"tokens": 981812042
},
{
"name": "opencode",
"kept": 490943,
"dups": 55807,
"tokens": 477872658
},
{
"name": "infinity_7m",
"kept": 5871991,
"dups": 1516899,
"tokens": 2396828009
},
{
"name": "magpie",
"kept": 946437,
"dups": 52539,
"tokens": 799903974
},
{
"name": "opencodeinstruct",
"kept": 4974125,
"dups": 25637,
"tokens": 2686545810
},
{
"name": "opc_stage1",
"kept": 3235843,
"dups": 948864,
"tokens": 2291932185
},
{
"name": "opc_stage2",
"kept": 173388,
"dups": 260331,
"tokens": 65874877
},
{
"name": "codefeedback",
"kept": 12670,
"dups": 143267,
"tokens": 8034830
},
{
"name": "magicoder_evol",
"kept": 0,
"dups": 108780,
"tokens": 0
},
{
"name": "magicoder_oss",
"kept": 45571,
"dups": 29619,
"tokens": 24773193
},
{
"name": "infinity_gen",
"kept": 1325695,
"dups": 69975,
"tokens": 1141556193
},
{
"name": "orca_agent",
"kept": 933069,
"dups": 10410,
"tokens": 869066974
}
]
}