File size: 1,753 Bytes
7b5e0a0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
{
 "complete": true,
 "samples": 37335232,
 "tokens": 20500262423,
 "max_len": 2048,
 "unique": "exact-duplicate token sequences removed",
 "sources": [
  {
   "name": "openhermes",
   "kept": 995501,
   "dups": 138,
   "tokens": 459820675
  },
  {
   "name": "tulu3",
   "kept": 888089,
   "dups": 13572,
   "tokens": 603189885
  },
  {
   "name": "openthoughts2",
   "kept": 259093,
   "dups": 0,
   "tokens": 377825471
  },
  {
   "name": "smoltalk",
   "kept": 933756,
   "dups": 101377,
   "tokens": 873563095
  },
  {
   "name": "openmath2",
   "kept": 13914553,
   "dups": 52433,
   "tokens": 6441662552
  },
  {
   "name": "webinstruct",
   "kept": 2334508,
   "dups": 44,
   "tokens": 981812042
  },
  {
   "name": "opencode",
   "kept": 490943,
   "dups": 55807,
   "tokens": 477872658
  },
  {
   "name": "infinity_7m",
   "kept": 5871991,
   "dups": 1516899,
   "tokens": 2396828009
  },
  {
   "name": "magpie",
   "kept": 946437,
   "dups": 52539,
   "tokens": 799903974
  },
  {
   "name": "opencodeinstruct",
   "kept": 4974125,
   "dups": 25637,
   "tokens": 2686545810
  },
  {
   "name": "opc_stage1",
   "kept": 3235843,
   "dups": 948864,
   "tokens": 2291932185
  },
  {
   "name": "opc_stage2",
   "kept": 173388,
   "dups": 260331,
   "tokens": 65874877
  },
  {
   "name": "codefeedback",
   "kept": 12670,
   "dups": 143267,
   "tokens": 8034830
  },
  {
   "name": "magicoder_evol",
   "kept": 0,
   "dups": 108780,
   "tokens": 0
  },
  {
   "name": "magicoder_oss",
   "kept": 45571,
   "dups": 29619,
   "tokens": 24773193
  },
  {
   "name": "infinity_gen",
   "kept": 1325695,
   "dups": 69975,
   "tokens": 1141556193
  },
  {
   "name": "orca_agent",
   "kept": 933069,
   "dups": 10410,
   "tokens": 869066974
  }
 ]
}