rishanthrajendhran commited on
Commit
fdfe97d
·
verified ·
1 Parent(s): f2a69ea

Add calibrated thresholds (IdeaLens-1M calibration split)

Browse files
Files changed (1) hide show
  1. thresholds.json +258 -0
thresholds.json ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "rishanthrajendhran/IdeaLens-ModernBERT-L",
3
+ "input": "outline",
4
+ "flag_rule": "flag the document as AI when P(human) < cut",
5
+ "calibration": {
6
+ "data": "IdeaLens-1M, calibration split (human documents only)",
7
+ "scores_from": "the training run's stored predictions on the calibration split",
8
+ "n_humans": 80000,
9
+ "n_per_format": {
10
+ "Academic Writing": 10000,
11
+ "Creative Writing": 10000,
12
+ "Knowledge Article": 10000,
13
+ "News Article": 10000,
14
+ "Nonfiction Writing": 10000,
15
+ "Personal About Page": 10000,
16
+ "Personal Blog": 10000,
17
+ "User Reviews": 10000
18
+ },
19
+ "n_per_topic": {
20
+ "Art & Design": 3338,
21
+ "Crime & Law": 3931,
22
+ "Education & Jobs": 3974,
23
+ "Entertainment": 4692,
24
+ "Fashion & Beauty": 2999,
25
+ "Finance & Business": 4855,
26
+ "Food & Dining": 2696,
27
+ "Games": 4257,
28
+ "Hardware": 2253,
29
+ "Health": 5069,
30
+ "History": 2554,
31
+ "Home & Hobbies": 4192,
32
+ "Industrial": 2299,
33
+ "Literature": 3571,
34
+ "Politics": 3728,
35
+ "Religion": 3617,
36
+ "Science & Tech.": 4154,
37
+ "Social Life": 3126,
38
+ "Software": 2257,
39
+ "Software Dev.": 2468,
40
+ "Sports & Fitness": 4224,
41
+ "Transportation": 2810,
42
+ "Travel": 2936
43
+ }
44
+ },
45
+ "method": {
46
+ "global": "the target-FPR quantile of P(human) over all calibration humans",
47
+ "per_group": "the group's own quantile, shrunk toward the global cut with weight n / (n + 2500)",
48
+ "min_group_humans": 200,
49
+ "estimability": "a cut at FPR q is given only when q * n >= 25 calibration humans",
50
+ "missing_cut": "no cut for a group means none could be estimated; do not substitute the global cut"
51
+ },
52
+ "fpr_targets": [
53
+ 0.001,
54
+ 0.005,
55
+ 0.01,
56
+ 0.02,
57
+ 0.05,
58
+ 0.1,
59
+ 0.2
60
+ ],
61
+ "global": {
62
+ "0.001": 0.005139531451277435,
63
+ "0.005": 0.024237636476755142,
64
+ "0.01": 0.05541279539465904,
65
+ "0.02": 0.182997182905674,
66
+ "0.05": 0.6486834049224853,
67
+ "0.1": 0.8966140151023865,
68
+ "0.2": 0.9773707389831543
69
+ },
70
+ "per_format": {
71
+ "0.005": {
72
+ "Academic Writing": 0.03953759199380875,
73
+ "Creative Writing": 0.05771391174197197,
74
+ "Knowledge Article": 0.019235945463180543,
75
+ "News Article": 0.019345112003386022,
76
+ "Nonfiction Writing": 0.016097568452358248,
77
+ "Personal About Page": 0.04852480654418469,
78
+ "Personal Blog": 0.03317504641413689,
79
+ "User Reviews": 0.02776208963990212
80
+ },
81
+ "0.01": {
82
+ "Academic Writing": 0.12872246548533442,
83
+ "Creative Writing": 0.1665476575791836,
84
+ "Knowledge Article": 0.03676119011640549,
85
+ "News Article": 0.046824600577354436,
86
+ "Nonfiction Writing": 0.029045478478074076,
87
+ "Personal About Page": 0.12370659753680231,
88
+ "Personal Blog": 0.11285716816782951,
89
+ "User Reviews": 0.08841504457592966
90
+ },
91
+ "0.02": {
92
+ "Academic Writing": 0.37497584372758863,
93
+ "Creative Writing": 0.36964609414339067,
94
+ "Knowledge Article": 0.08466875571012497,
95
+ "News Article": 0.16145060497522357,
96
+ "Nonfiction Writing": 0.07510314583778382,
97
+ "Personal About Page": 0.31876455098390594,
98
+ "Personal Blog": 0.3270333116650582,
99
+ "User Reviews": 0.2942755115628243
100
+ },
101
+ "0.05": {
102
+ "Academic Writing": 0.7944207191467285,
103
+ "Creative Writing": 0.7330899429321289,
104
+ "Knowledge Article": 0.3667260706424713,
105
+ "News Article": 0.5772577738761903,
106
+ "Nonfiction Writing": 0.4114002323150635,
107
+ "Personal About Page": 0.6979178619384766,
108
+ "Personal Blog": 0.7110324096679688,
109
+ "User Reviews": 0.7446396446228027
110
+ },
111
+ "0.1": {
112
+ "Academic Writing": 0.9471324324607849,
113
+ "Creative Writing": 0.9085190653800964,
114
+ "Knowledge Article": 0.8029681372642518,
115
+ "News Article": 0.8600945830345154,
116
+ "Nonfiction Writing": 0.8666762709617615,
117
+ "Personal About Page": 0.8915582060813904,
118
+ "Personal Blog": 0.9003047823905945,
119
+ "User Reviews": 0.9365684390068054
120
+ },
121
+ "0.2": {
122
+ "Academic Writing": 0.9886121273040771,
123
+ "Creative Writing": 0.9766665458679199,
124
+ "Knowledge Article": 0.9701791286468506,
125
+ "News Article": 0.9669312953948974,
126
+ "Nonfiction Writing": 0.9753310680389404,
127
+ "Personal About Page": 0.9737387657165527,
128
+ "Personal Blog": 0.9732376575469971,
129
+ "User Reviews": 0.9861243724822998
130
+ }
131
+ },
132
+ "per_topic": {
133
+ "0.005": {
134
+ "Health": 0.023053244813589616
135
+ },
136
+ "0.01": {
137
+ "Art & Design": 0.06763211139087584,
138
+ "Crime & Law": 0.07036863836068664,
139
+ "Education & Jobs": 0.05330997003780395,
140
+ "Entertainment": 0.04512200002518127,
141
+ "Fashion & Beauty": 0.088135457826969,
142
+ "Finance & Business": 0.04148502488309716,
143
+ "Food & Dining": 0.05131299507598449,
144
+ "Games": 0.05958534698312023,
145
+ "Health": 0.044195629122912736,
146
+ "History": 0.11171335421877035,
147
+ "Home & Hobbies": 0.05877323033714537,
148
+ "Literature": 0.09868350075013241,
149
+ "Politics": 0.07981549680108586,
150
+ "Religion": 0.05817645865597424,
151
+ "Science & Tech.": 0.05320098759011352,
152
+ "Social Life": 0.04563931021024928,
153
+ "Sports & Fitness": 0.06428856616490729,
154
+ "Transportation": 0.07430894629724619,
155
+ "Travel": 0.0492034640790918
156
+ },
157
+ "0.02": {
158
+ "Art & Design": 0.2667693970327142,
159
+ "Crime & Law": 0.2786540611408266,
160
+ "Education & Jobs": 0.1945583799670131,
161
+ "Entertainment": 0.17524809834921323,
162
+ "Fashion & Beauty": 0.2666449849335448,
163
+ "Finance & Business": 0.10400221619777822,
164
+ "Food & Dining": 0.21489607520957835,
165
+ "Games": 0.18023701900990916,
166
+ "Hardware": 0.25904285000019167,
167
+ "Health": 0.11940725700124312,
168
+ "History": 0.388731863553174,
169
+ "Home & Hobbies": 0.23846134369859992,
170
+ "Industrial": 0.14381403822728955,
171
+ "Literature": 0.2830624036244242,
172
+ "Politics": 0.22826410531634006,
173
+ "Religion": 0.21331345031312893,
174
+ "Science & Tech.": 0.15791790899542096,
175
+ "Social Life": 0.14721397015571044,
176
+ "Software": 0.14573120189892136,
177
+ "Software Dev.": 0.18237710482808323,
178
+ "Sports & Fitness": 0.2365606066013716,
179
+ "Transportation": 0.22459357514781017,
180
+ "Travel": 0.24334309230160414
181
+ },
182
+ "0.05": {
183
+ "Art & Design": 0.7016386904100638,
184
+ "Crime & Law": 0.7753899506732179,
185
+ "Education & Jobs": 0.5901288853020885,
186
+ "Entertainment": 0.6629933400069249,
187
+ "Fashion & Beauty": 0.6879606136594562,
188
+ "Finance & Business": 0.4055291469428589,
189
+ "Food & Dining": 0.6740288405210996,
190
+ "Games": 0.6245980470759533,
191
+ "Hardware": 0.7054242893893569,
192
+ "Health": 0.48469294387655726,
193
+ "History": 0.7848231397507264,
194
+ "Home & Hobbies": 0.6597234686762005,
195
+ "Industrial": 0.6432633246499118,
196
+ "Literature": 0.7456914604230525,
197
+ "Politics": 0.7118498336143346,
198
+ "Religion": 0.6700943714051888,
199
+ "Science & Tech.": 0.6269058667327106,
200
+ "Social Life": 0.6001305766867184,
201
+ "Software": 0.5572515561183712,
202
+ "Software Dev.": 0.6317542892003405,
203
+ "Sports & Fitness": 0.6906096807623959,
204
+ "Transportation": 0.7103921012521464,
205
+ "Travel": 0.6828169851859235
206
+ },
207
+ "0.1": {
208
+ "Art & Design": 0.909681719706948,
209
+ "Crime & Law": 0.9384883085169886,
210
+ "Education & Jobs": 0.8584615583345409,
211
+ "Entertainment": 0.9046775788996986,
212
+ "Fashion & Beauty": 0.9033018975414825,
213
+ "Finance & Business": 0.7865695125306245,
214
+ "Food & Dining": 0.909378945689829,
215
+ "Games": 0.8758573694025886,
216
+ "Hardware": 0.9099728331374239,
217
+ "Health": 0.8523264182308482,
218
+ "History": 0.9392392507291001,
219
+ "Home & Hobbies": 0.8975603867702975,
220
+ "Industrial": 0.9076183274006392,
221
+ "Literature": 0.9243477025665889,
222
+ "Politics": 0.9148298466672634,
223
+ "Religion": 0.9027840338482934,
224
+ "Science & Tech.": 0.9010606672694312,
225
+ "Social Life": 0.8690233408366493,
226
+ "Software": 0.8665858072745392,
227
+ "Software Dev.": 0.8897073692649651,
228
+ "Sports & Fitness": 0.9008747022184569,
229
+ "Transportation": 0.9145631512455124,
230
+ "Travel": 0.900314012768516
231
+ },
232
+ "0.2": {
233
+ "Art & Design": 0.978772696851663,
234
+ "Crime & Law": 0.9863508726200667,
235
+ "Education & Jobs": 0.9701707525720263,
236
+ "Entertainment": 0.9775946031887088,
237
+ "Fashion & Beauty": 0.9786560668059101,
238
+ "Finance & Business": 0.952645285732482,
239
+ "Food & Dining": 0.9798511684262082,
240
+ "Games": 0.9683222689941929,
241
+ "Hardware": 0.97939984327413,
242
+ "Health": 0.9728951033173787,
243
+ "History": 0.9864768436599448,
244
+ "Home & Hobbies": 0.97890669246384,
245
+ "Industrial": 0.9814533342543084,
246
+ "Literature": 0.9840160129274887,
247
+ "Politics": 0.9806345429417409,
248
+ "Religion": 0.9790018851000518,
249
+ "Science & Tech.": 0.9793746653784134,
250
+ "Social Life": 0.9643663023656431,
251
+ "Software": 0.972314228825091,
252
+ "Software Dev.": 0.9730703673308982,
253
+ "Sports & Fitness": 0.9773271697394412,
254
+ "Transportation": 0.9812334339497453,
255
+ "Travel": 0.9784419171682082
256
+ }
257
+ }
258
+ }