Nabidnur commited on
Commit
8efc456
·
verified ·
1 Parent(s): aa3111c

RACE-Net weights, card and code

Browse files
Files changed (5) hide show
  1. README.md +8 -1
  2. analysis.json +396 -288
  3. racenet_analysis.py +3 -0
  4. racenet_scaling.png +2 -2
  5. selection.json +54 -2
README.md CHANGED
@@ -80,7 +80,13 @@ the choice and are shown for transparency.
80
 
81
  | checkpoint | generator | steps | synthetic acc. (v1 / v2) | A0 dev (selects) | SO-101 | Panda | Panda, no attack |
82
  |---|---|---:|---:|---:|---:|---:|---:|
83
- | **v1 · local CPU (released)** | v1 | 6,000 | 93.4 / 96.8 | **90.4** | 81.6 | 81.8 | 81.8 |
 
 
 
 
 
 
84
 
85
  ## Scaling study
86
 
@@ -97,6 +103,7 @@ the choice and are shown for transparency.
97
  - **Weak receiver (open weakness).** If the receiver is only 55% accurate and nobody lies, RACE-Net (57.1) and RACE-S online
98
  (54.4) stay near the receiver's own accuracy, while majority reaches 81.8 and discrete RACE 82.4.
99
  Score-level anchoring trusts a weak receiver too much.
 
100
 
101
  ## Limitations
102
 
 
80
 
81
  | checkpoint | generator | steps | synthetic acc. (v1 / v2) | A0 dev (selects) | SO-101 | Panda | Panda, no attack |
82
  |---|---|---:|---:|---:|---:|---:|---:|
83
+ | **v1 · local CPU · 6k steps (released)** | v1 | 6,000 | 93.4 / 96.8 | **90.4** | 81.6 | 81.8 | 81.8 |
84
+ | v1 · Kaggle CPU · 15k steps | v1 | 15,000 | 93.9 / 97.2 | **90.2** | 80.9 | 82.3 | 84.5 |
85
+ | v2 · local CPU · 7k steps | v2 | 7,000 | 92.6 / 96.9 | **90.2** | 82.6 | 83.5 | 88.5 |
86
+
87
+ Generator v2 widens agent skill (receivers down to near chance, near-perfect peers) and trains on swarms of up to 12 + 3 agents. It was designed after the v1 results on the test tracks and the scaling study were known, so for v2 checkpoints the SO-101 and Panda columns are **not** untouched test results; the v1 rows are.
88
+
89
+ The 15k-step run was trained in a Kaggle CPU notebook (4 vCPU, AMD EPYC 7B12; 130 min); its log is in `results/kaggle/racenet_training.json`. All three checkpoints are in `models/racenet/candidates/`.
90
 
91
  ## Scaling study
92
 
 
103
  - **Weak receiver (open weakness).** If the receiver is only 55% accurate and nobody lies, RACE-Net (57.1) and RACE-S online
104
  (54.4) stay near the receiver's own accuracy, while majority reaches 81.8 and discrete RACE 82.4.
105
  Score-level anchoring trusts a weak receiver too much.
106
+ - **v2 checkpoint (post hoc).** Training on a wider skill range helps with weak receivers (59.8 at 55% and 82.5 at 70% receiver accuracy, against 57.1 and 77.4 for the released model) but costs accuracy on large swarms (80.8 against 89.1 at 24 agents). It is not uniformly better, which agrees with the development track not selecting it.
107
 
108
  ## Limitations
109
 
analysis.json CHANGED
@@ -19,7 +19,8 @@
19
  "dawid_skene": 0.8105,
20
  "race_online": 0.8424999999999999,
21
  "race_s_online": 0.767,
22
- "race_net": 0.795
 
23
  },
24
  "se": {
25
  "receiver_only": 0.011599568957508724,
@@ -27,15 +28,17 @@
27
  "dawid_skene": 0.011346254888728704,
28
  "race_online": 0.010243900624273938,
29
  "race_s_online": 0.015034959261667456,
30
- "race_net": 0.012816005617976295
 
31
  },
32
  "ms": {
33
- "receiver_only": 0.2772879997792188,
34
- "majority": 0.19835900002362905,
35
- "dawid_skene": 5.101164500047162,
36
- "race_online": 8.66529949962569,
37
- "race_s_online": 3.411491499718977,
38
- "race_net": 7.380093499705254
 
39
  }
40
  },
41
  {
@@ -49,7 +52,8 @@
49
  "dawid_skene": 0.776125,
50
  "race_online": 0.8081249999999999,
51
  "race_s_online": 0.768,
52
- "race_net": 0.788625
 
53
  },
54
  "se": {
55
  "receiver_only": 0.01114618768907109,
@@ -57,15 +61,17 @@
57
  "dawid_skene": 0.013559446660907665,
58
  "race_online": 0.008879180353501102,
59
  "race_s_online": 0.014380542409798039,
60
- "race_net": 0.011726672322104004
 
61
  },
62
  "ms": {
63
- "receiver_only": 0.29290675001902855,
64
- "majority": 0.21077387441437168,
65
- "dawid_skene": 5.563213374898623,
66
- "race_online": 9.696928374978597,
67
- "race_s_online": 3.5514857502221275,
68
- "race_net": 10.230751874814814
 
69
  }
70
  },
71
  {
@@ -79,7 +85,8 @@
79
  "dawid_skene": 0.6375,
80
  "race_online": 0.7892499999999999,
81
  "race_s_online": 0.749,
82
- "race_net": 0.771
 
83
  },
84
  "se": {
85
  "receiver_only": 0.01039891821296812,
@@ -87,15 +94,17 @@
87
  "dawid_skene": 0.019017261895446464,
88
  "race_online": 0.011731501395814602,
89
  "race_s_online": 0.014524117873385634,
90
- "race_net": 0.012330602175076444
 
91
  },
92
  "ms": {
93
- "receiver_only": 0.2747267503764306,
94
- "majority": 0.1942389999385341,
95
- "dawid_skene": 5.083128250362279,
96
- "race_online": 11.62928487519821,
97
- "race_s_online": 3.4307115001865895,
98
- "race_net": 10.828311874774954
 
99
  }
100
  },
101
  {
@@ -109,7 +118,8 @@
109
  "dawid_skene": 0.4035,
110
  "race_online": 0.7675000000000002,
111
  "race_s_online": 0.77625,
112
- "race_net": 0.796125
 
113
  },
114
  "se": {
115
  "receiver_only": 0.010896673804423074,
@@ -117,15 +127,17 @@
117
  "dawid_skene": 0.01997920794225837,
118
  "race_online": 0.012097262086935207,
119
  "race_s_online": 0.01345884931931404,
120
- "race_net": 0.010641127466109965
 
121
  },
122
  "ms": {
123
- "receiver_only": 0.3036801247162657,
124
- "majority": 0.2086862498344999,
125
- "dawid_skene": 5.964542874608014,
126
- "race_online": 11.907079875527415,
127
- "race_s_online": 3.46638487508244,
128
- "race_net": 10.520827875097893
 
129
  }
130
  },
131
  {
@@ -139,7 +151,8 @@
139
  "dawid_skene": 0.3197499999999999,
140
  "race_online": 0.7191249999999999,
141
  "race_s_online": 0.757875,
142
- "race_net": 0.77175
 
143
  },
144
  "se": {
145
  "receiver_only": 0.009515776374001231,
@@ -147,15 +160,17 @@
147
  "dawid_skene": 0.01407002576401337,
148
  "race_online": 0.010241269147425042,
149
  "race_s_online": 0.012261824650108158,
150
- "race_net": 0.010441952643064424
 
151
  },
152
  "ms": {
153
- "receiver_only": 0.2961078755561175,
154
- "majority": 0.2007364996643446,
155
- "dawid_skene": 5.863659874876248,
156
- "race_online": 10.061210499998197,
157
- "race_s_online": 3.046369374715141,
158
- "race_net": 10.14635212527537
 
159
  }
160
  }
161
  ],
@@ -171,7 +186,8 @@
171
  "dawid_skene": 0.9345000000000001,
172
  "race_online": 0.9435,
173
  "race_s_online": 0.9285,
174
- "race_net": 0.8755000000000001
 
175
  },
176
  "se": {
177
  "receiver_only": 0.011624865590620823,
@@ -179,15 +195,17 @@
179
  "dawid_skene": 0.00661343329897565,
180
  "race_online": 0.005927689263110874,
181
  "race_s_online": 0.0063354163241258225,
182
- "race_net": 0.011607648340641614
 
183
  },
184
  "ms": {
185
- "receiver_only": 0.3562029996828642,
186
- "majority": 0.27310499990562676,
187
- "dawid_skene": 6.571876499947393,
188
- "race_online": 14.054156500606041,
189
- "race_s_online": 4.018874000394135,
190
- "race_net": 16.768835001130356
 
191
  }
192
  },
193
  {
@@ -201,7 +219,8 @@
201
  "dawid_skene": 0.903125,
202
  "race_online": 0.9354999999999999,
203
  "race_s_online": 0.9295,
204
- "race_net": 0.8885
 
205
  },
206
  "se": {
207
  "receiver_only": 0.009222255689363641,
@@ -209,15 +228,17 @@
209
  "dawid_skene": 0.007781940230430967,
210
  "race_online": 0.005767906899387335,
211
  "race_s_online": 0.006732848579910288,
212
- "race_net": 0.008958934088383505
 
213
  },
214
  "ms": {
215
- "receiver_only": 0.3890925006544421,
216
- "majority": 0.2764887501598423,
217
- "dawid_skene": 6.777383249755076,
218
- "race_online": 14.151794375038662,
219
- "race_s_online": 3.96030824981608,
220
- "race_net": 17.923576125213003
 
221
  }
222
  },
223
  {
@@ -231,7 +252,8 @@
231
  "dawid_skene": 0.7207500000000001,
232
  "race_online": 0.928125,
233
  "race_s_online": 0.9192500000000001,
234
- "race_net": 0.8952500000000001
 
235
  },
236
  "se": {
237
  "receiver_only": 0.011123735883236351,
@@ -239,15 +261,17 @@
239
  "dawid_skene": 0.006290121222997218,
240
  "race_online": 0.004885805332798266,
241
  "race_s_online": 0.007794429100325434,
242
- "race_net": 0.007611463394118103
 
243
  },
244
  "ms": {
245
- "receiver_only": 0.36924262508364336,
246
- "majority": 0.26562587527223513,
247
- "dawid_skene": 6.6529963744415,
248
- "race_online": 13.870407999775125,
249
- "race_s_online": 3.9161395002338395,
250
- "race_net": 18.62918825008819
 
251
  }
252
  },
253
  {
@@ -261,7 +285,8 @@
261
  "dawid_skene": 0.4763749999999999,
262
  "race_online": 0.9040000000000001,
263
  "race_s_online": 0.8953749999999999,
264
- "race_net": 0.8865000000000002
 
265
  },
266
  "se": {
267
  "receiver_only": 0.010874281585465773,
@@ -269,15 +294,17 @@
269
  "dawid_skene": 0.034240178792611466,
270
  "race_online": 0.004806896087913694,
271
  "race_s_online": 0.01288806982251415,
272
- "race_net": 0.006323072828933728
 
273
  },
274
  "ms": {
275
- "receiver_only": 0.3762671249205596,
276
- "majority": 0.27882962535841216,
277
- "dawid_skene": 6.810715374513165,
278
- "race_online": 15.567067624488118,
279
- "race_s_online": 4.2933539998557535,
280
- "race_net": 19.173047124695586
 
281
  }
282
  },
283
  {
@@ -291,7 +318,8 @@
291
  "dawid_skene": 0.32775,
292
  "race_online": 0.909375,
293
  "race_s_online": 0.882,
294
- "race_net": 0.86175
 
295
  },
296
  "se": {
297
  "receiver_only": 0.013745453793891277,
@@ -299,15 +327,17 @@
299
  "dawid_skene": 0.020215170664627097,
300
  "race_online": 0.006884936001881206,
301
  "race_s_online": 0.01112092622041887,
302
- "race_net": 0.011124438188061449
 
303
  },
304
  "ms": {
305
- "receiver_only": 0.3784703749261098,
306
- "majority": 0.27233262494519295,
307
- "dawid_skene": 6.583474624903829,
308
- "race_online": 15.7015337501889,
309
- "race_s_online": 4.287914249971436,
310
- "race_net": 20.093656749850197
 
311
  }
312
  },
313
  {
@@ -321,7 +351,8 @@
321
  "dawid_skene": 0.260625,
322
  "race_online": 0.8717499999999999,
323
  "race_s_online": 0.85975,
324
- "race_net": 0.85825
 
325
  },
326
  "se": {
327
  "receiver_only": 0.008404611829228047,
@@ -329,15 +360,17 @@
329
  "dawid_skene": 0.006663508366468822,
330
  "race_online": 0.006157058957002116,
331
  "race_s_online": 0.008574854226166181,
332
- "race_net": 0.006235232553802621
 
333
  },
334
  "ms": {
335
- "receiver_only": 0.3712228749463975,
336
- "majority": 0.2772140001070511,
337
- "dawid_skene": 6.748552999624735,
338
- "race_online": 16.421656500369863,
339
- "race_s_online": 4.458211375094834,
340
- "race_net": 19.425959874752152
 
341
  }
342
  },
343
  {
@@ -351,7 +384,8 @@
351
  "dawid_skene": 0.215625,
352
  "race_online": 0.8138750000000001,
353
  "race_s_online": 0.8266249999999999,
354
- "race_net": 0.836125
 
355
  },
356
  "se": {
357
  "receiver_only": 0.012354654993159462,
@@ -359,15 +393,17 @@
359
  "dawid_skene": 0.005433906858789539,
360
  "race_online": 0.008784707949044183,
361
  "race_s_online": 0.015569677059913604,
362
- "race_net": 0.008138402407721065
 
363
  },
364
  "ms": {
365
- "receiver_only": 0.3587508745113155,
366
- "majority": 0.2541757503422559,
367
- "dawid_skene": 6.521433250100017,
368
- "race_online": 18.871999374823645,
369
- "race_s_online": 3.941388875091434,
370
- "race_net": 20.608236500265775
 
371
  }
372
  },
373
  {
@@ -381,7 +417,8 @@
381
  "dawid_skene": 0.20875000000000005,
382
  "race_online": 0.786625,
383
  "race_s_online": 0.834375,
384
- "race_net": 0.8320000000000001
 
385
  },
386
  "se": {
387
  "receiver_only": 0.009964938534682487,
@@ -389,15 +426,17 @@
389
  "dawid_skene": 0.0056527095272975066,
390
  "race_online": 0.008309021828711245,
391
  "race_s_online": 0.012361577316426894,
392
- "race_net": 0.010979526401443729
 
393
  },
394
  "ms": {
395
- "receiver_only": 0.36916712451784406,
396
- "majority": 0.2602773754460941,
397
- "dawid_skene": 6.586847749758817,
398
- "race_online": 17.89988624977923,
399
- "race_s_online": 4.286726999680468,
400
- "race_net": 19.185820249731478
 
401
  }
402
  }
403
  ],
@@ -413,7 +452,8 @@
413
  "dawid_skene": 0.428125,
414
  "race_online": 0.68125,
415
  "race_s_online": 0.728125,
416
- "race_net": 0.675
 
417
  },
418
  "se": {
419
  "receiver_only": 0.03354101966249684,
@@ -421,15 +461,17 @@
421
  "dawid_skene": 0.02915643421699574,
422
  "race_online": 0.03937996000505841,
423
  "race_s_online": 0.03866043075613618,
424
- "race_net": 0.05045689682788668
 
425
  },
426
  "ms": {
427
- "receiver_only": 0.20381725016704877,
428
- "majority": 0.14142587497190107,
429
- "dawid_skene": 3.896357874964451,
430
- "race_online": 2.7888003751286305,
431
- "race_s_online": 1.560288124892395,
432
- "race_net": 2.436062375181791
 
433
  }
434
  },
435
  {
@@ -443,7 +485,8 @@
443
  "dawid_skene": 0.43359375,
444
  "race_online": 0.696875,
445
  "race_s_online": 0.70234375,
446
- "race_net": 0.6921875
 
447
  },
448
  "se": {
449
  "receiver_only": 0.022910747341586216,
@@ -451,15 +494,17 @@
451
  "dawid_skene": 0.029082549367866205,
452
  "race_online": 0.021690066490792508,
453
  "race_s_online": 0.02513331155278538,
454
- "race_net": 0.023466128348909623
 
455
  },
456
  "ms": {
457
- "receiver_only": 0.20208625028317329,
458
- "majority": 0.140530374892478,
459
- "dawid_skene": 4.063699624794026,
460
- "race_online": 3.9346086250588996,
461
- "race_s_online": 1.7068274999019195,
462
- "race_net": 2.7482862499255134
 
463
  }
464
  },
465
  {
@@ -473,7 +518,8 @@
473
  "dawid_skene": 0.377734375,
474
  "race_online": 0.759375,
475
  "race_s_online": 0.7390625,
476
- "race_net": 0.75859375
 
477
  },
478
  "se": {
479
  "receiver_only": 0.0189314140385762,
@@ -481,15 +527,17 @@
481
  "dawid_skene": 0.018890061186271838,
482
  "race_online": 0.021563066115756823,
483
  "race_s_online": 0.016649975365972767,
484
- "race_net": 0.01679142353396147
 
485
  },
486
  "ms": {
487
- "receiver_only": 0.23646812519473315,
488
- "majority": 0.16245062488451367,
489
- "dawid_skene": 4.85215100025016,
490
- "race_online": 5.326368624764655,
491
- "race_s_online": 2.0306320002418943,
492
- "race_net": 3.6502384996310866
 
493
  }
494
  },
495
  {
@@ -503,7 +551,8 @@
503
  "dawid_skene": 0.3736328125,
504
  "race_online": 0.7607421875,
505
  "race_s_online": 0.7474609375,
506
- "race_net": 0.77265625
 
507
  },
508
  "se": {
509
  "receiver_only": 0.01411773008434341,
@@ -511,15 +560,17 @@
511
  "dawid_skene": 0.015500664973607532,
512
  "race_online": 0.014840858579409191,
513
  "race_s_online": 0.014182095073673386,
514
- "race_net": 0.012145862367746885
 
515
  },
516
  "ms": {
517
- "receiver_only": 0.25433687505937996,
518
- "majority": 0.1841116250034247,
519
- "dawid_skene": 4.593158375200801,
520
- "race_online": 8.179479874797835,
521
- "race_s_online": 2.9732773755313247,
522
- "race_net": 5.08545337470423
 
523
  }
524
  },
525
  {
@@ -533,7 +584,8 @@
533
  "dawid_skene": 0.39248046875,
534
  "race_online": 0.76357421875,
535
  "race_s_online": 0.7654296875,
536
- "race_net": 0.78046875
 
537
  },
538
  "se": {
539
  "receiver_only": 0.007519404422631738,
@@ -541,15 +593,17 @@
541
  "dawid_skene": 0.021578042883029624,
542
  "race_online": 0.008270827158069627,
543
  "race_s_online": 0.00839832394545325,
544
- "race_net": 0.008616358754345929
 
545
  },
546
  "ms": {
547
- "receiver_only": 0.3085287501107814,
548
- "majority": 0.21723912550442037,
549
- "dawid_skene": 6.555987749834458,
550
- "race_online": 16.973454124808995,
551
- "race_s_online": 3.800161500066679,
552
- "race_net": 8.68280474992389
 
553
  }
554
  },
555
  {
@@ -563,7 +617,8 @@
563
  "dawid_skene": 0.3998046875,
564
  "race_online": 0.760302734375,
565
  "race_s_online": 0.784521484375,
566
- "race_net": 0.776904296875
 
567
  },
568
  "se": {
569
  "receiver_only": 0.006488962044731711,
@@ -571,15 +626,17 @@
571
  "dawid_skene": 0.016211437539351194,
572
  "race_online": 0.006989239119019292,
573
  "race_s_online": 0.00700620641771385,
574
- "race_net": 0.007469099284029169
 
575
  },
576
  "ms": {
577
- "receiver_only": 0.3781822501878196,
578
- "majority": 0.25851887448880007,
579
- "dawid_skene": 8.029971375208333,
580
- "race_online": 28.371019625410554,
581
- "race_s_online": 5.326530500042281,
582
- "race_net": 16.118368125034976
 
583
  }
584
  }
585
  ],
@@ -595,7 +652,8 @@
595
  "dawid_skene": 0.32825,
596
  "race_online": 0.7200000000000001,
597
  "race_s_online": 0.73275,
598
- "race_net": 0.7358749999999998
 
599
  },
600
  "se": {
601
  "receiver_only": 0.010700467279516348,
@@ -603,15 +661,17 @@
603
  "dawid_skene": 0.012532582535136162,
604
  "race_online": 0.010700467279516348,
605
  "race_s_online": 0.01150258123205396,
606
- "race_net": 0.012435175662209197
 
607
  },
608
  "ms": {
609
- "receiver_only": 0.25277012559854484,
610
- "majority": 0.18364437482887297,
611
- "dawid_skene": 4.991113375126588,
612
- "race_online": 6.835640875124227,
613
- "race_s_online": 2.3170128752099117,
614
- "race_net": 5.439215625074212
 
615
  }
616
  },
617
  {
@@ -625,7 +685,8 @@
625
  "dawid_skene": 0.4035,
626
  "race_online": 0.7675000000000002,
627
  "race_s_online": 0.77625,
628
- "race_net": 0.796125
 
629
  },
630
  "se": {
631
  "receiver_only": 0.010896673804423074,
@@ -633,15 +694,17 @@
633
  "dawid_skene": 0.01997920794225837,
634
  "race_online": 0.012097262086935207,
635
  "race_s_online": 0.01345884931931404,
636
- "race_net": 0.010641127466109965
 
637
  },
638
  "ms": {
639
- "receiver_only": 0.28967925027245656,
640
- "majority": 0.20006049999210518,
641
- "dawid_skene": 5.206961875273919,
642
- "race_online": 11.093725250248099,
643
- "race_s_online": 3.371671624790906,
644
- "race_net": 8.306291375220098
 
645
  }
646
  },
647
  {
@@ -655,7 +718,8 @@
655
  "dawid_skene": 0.42524999999999996,
656
  "race_online": 0.81275,
657
  "race_s_online": 0.781125,
658
- "race_net": 0.795625
 
659
  },
660
  "se": {
661
  "receiver_only": 0.012674284989694687,
@@ -663,15 +727,17 @@
663
  "dawid_skene": 0.022856823379463733,
664
  "race_online": 0.008817418840000741,
665
  "race_s_online": 0.014856054111035003,
666
- "race_net": 0.013010662694497929
 
667
  },
668
  "ms": {
669
- "receiver_only": 0.3264987501552241,
670
- "majority": 0.22650649998467998,
671
- "dawid_skene": 6.216220625219648,
672
- "race_online": 11.287604250128425,
673
- "race_s_online": 3.753791000235651,
674
- "race_net": 11.099079125415301
 
675
  }
676
  },
677
  {
@@ -685,7 +751,8 @@
685
  "dawid_skene": 0.42275,
686
  "race_online": 0.8430000000000002,
687
  "race_s_online": 0.8099999999999999,
688
- "race_net": 0.8172499999999999
 
689
  },
690
  "se": {
691
  "receiver_only": 0.006725882841679595,
@@ -693,15 +760,17 @@
693
  "dawid_skene": 0.025814785007820612,
694
  "race_online": 0.008222682044199448,
695
  "race_s_online": 0.012826973532365296,
696
- "race_net": 0.01108475868027807
 
697
  },
698
  "ms": {
699
- "receiver_only": 0.3311696250420937,
700
- "majority": 0.24703199983378,
701
- "dawid_skene": 6.382177374462117,
702
- "race_online": 11.602530625168583,
703
- "race_s_online": 4.043969624945021,
704
- "race_net": 11.696756999754143
 
705
  }
706
  },
707
  {
@@ -715,7 +784,8 @@
715
  "dawid_skene": 0.39075000000000004,
716
  "race_online": 0.8685,
717
  "race_s_online": 0.83575,
718
- "race_net": 0.8396250000000001
 
719
  },
720
  "se": {
721
  "receiver_only": 0.01198905751091386,
@@ -723,15 +793,17 @@
723
  "dawid_skene": 0.02485149643381662,
724
  "race_online": 0.007612489737267302,
725
  "race_s_online": 0.015323694887330533,
726
- "race_net": 0.011038674909154628
 
727
  },
728
  "ms": {
729
- "receiver_only": 0.34774212485899625,
730
- "majority": 0.2464538752064982,
731
- "dawid_skene": 6.60796962506538,
732
- "race_online": 13.29157337545439,
733
- "race_s_online": 4.303733500137241,
734
- "race_net": 15.045734249724774
 
735
  }
736
  },
737
  {
@@ -745,7 +817,8 @@
745
  "dawid_skene": 0.32775,
746
  "race_online": 0.909375,
747
  "race_s_online": 0.882,
748
- "race_net": 0.86175
 
749
  },
750
  "se": {
751
  "receiver_only": 0.013745453793891277,
@@ -753,15 +826,17 @@
753
  "dawid_skene": 0.020215170664627097,
754
  "race_online": 0.006884936001881206,
755
  "race_s_online": 0.01112092622041887,
756
- "race_net": 0.011124438188061449
 
757
  },
758
  "ms": {
759
- "receiver_only": 0.36458950012274727,
760
- "majority": 0.27975574994343333,
761
- "dawid_skene": 6.839673249714906,
762
- "race_online": 16.037981249837685,
763
- "race_s_online": 4.48347475025912,
764
- "race_net": 13.71562025019557
 
765
  }
766
  },
767
  {
@@ -775,7 +850,8 @@
775
  "dawid_skene": 0.34012500000000007,
776
  "race_online": 0.9337500000000001,
777
  "race_s_online": 0.9324999999999999,
778
- "race_net": 0.89075
 
779
  },
780
  "se": {
781
  "receiver_only": 0.010278010507875537,
@@ -783,15 +859,17 @@
783
  "dawid_skene": 0.017458410401580096,
784
  "race_online": 0.006114787404317504,
785
  "race_s_online": 0.005326818938165631,
786
- "race_net": 0.008338727420895831
 
787
  },
788
  "ms": {
789
- "receiver_only": 0.4244741251113737,
790
- "majority": 0.3165976245327329,
791
- "dawid_skene": 7.446009000432241,
792
- "race_online": 23.352996375251678,
793
- "race_s_online": 4.657189750560065,
794
- "race_net": 21.131173624780786
 
795
  }
796
  }
797
  ],
@@ -807,7 +885,8 @@
807
  "dawid_skene": 0.8470000000000001,
808
  "race_online": 0.8240000000000001,
809
  "race_s_online": 0.544,
810
- "race_net": 0.571
 
811
  },
812
  "se": {
813
  "receiver_only": 0.011235546270653686,
@@ -815,15 +894,17 @@
815
  "dawid_skene": 0.0095420123663722,
816
  "race_online": 0.028225874654295477,
817
  "race_s_online": 0.01600624877977348,
818
- "race_net": 0.026276415280627602
 
819
  },
820
  "ms": {
821
- "receiver_only": 0.28504750025604153,
822
- "majority": 0.20617999962269096,
823
- "dawid_skene": 5.937592499321909,
824
- "race_online": 9.416694499122968,
825
- "race_s_online": 1.4332850005303044,
826
- "race_net": 6.667908000963507
 
827
  }
828
  },
829
  {
@@ -837,7 +918,8 @@
837
  "dawid_skene": 0.8394999999999999,
838
  "race_online": 0.8625,
839
  "race_s_online": 0.5934999999999999,
840
- "race_net": 0.6349999999999999
 
841
  },
842
  "se": {
843
  "receiver_only": 0.008901544809750717,
@@ -845,15 +927,17 @@
845
  "dawid_skene": 0.00826059925187029,
846
  "race_online": 0.006887488656977954,
847
  "race_s_online": 0.019445757892147073,
848
- "race_net": 0.025283393759541063
 
849
  },
850
  "ms": {
851
- "receiver_only": 0.3078800009461702,
852
- "majority": 0.21749450024799444,
853
- "dawid_skene": 6.024277499818709,
854
- "race_online": 9.409731500454654,
855
- "race_s_online": 2.466451500367839,
856
- "race_net": 6.906335000167019
 
857
  }
858
  },
859
  {
@@ -867,7 +951,8 @@
867
  "dawid_skene": 0.8500000000000002,
868
  "race_online": 0.869,
869
  "race_s_online": 0.733,
870
- "race_net": 0.7735000000000001
 
871
  },
872
  "se": {
873
  "receiver_only": 0.008132496541653123,
@@ -875,15 +960,17 @@
875
  "dawid_skene": 0.009460443964212252,
876
  "race_online": 0.0072422372233999655,
877
  "race_s_online": 0.025220031720836512,
878
- "race_net": 0.019522743147416557
 
879
  },
880
  "ms": {
881
- "receiver_only": 0.29712749983445974,
882
- "majority": 0.20899950050079497,
883
- "dawid_skene": 5.803269000352884,
884
- "race_online": 9.292306499446568,
885
- "race_s_online": 3.5193550002077245,
886
- "race_net": 7.701153499510838
 
887
  }
888
  },
889
  {
@@ -897,7 +984,8 @@
897
  "dawid_skene": 0.8629999999999999,
898
  "race_online": 0.8854999999999998,
899
  "race_s_online": 0.8604999999999998,
900
- "race_net": 0.8605
 
901
  },
902
  "se": {
903
  "receiver_only": 0.009051933495115837,
@@ -905,15 +993,17 @@
905
  "dawid_skene": 0.010818964830333816,
906
  "race_online": 0.0066511277239277265,
907
  "race_s_online": 0.0157396791581023,
908
- "race_net": 0.01540251602823383
 
909
  },
910
  "ms": {
911
- "receiver_only": 0.29016900043643545,
912
- "majority": 0.20973649952793494,
913
- "dawid_skene": 5.81656199938152,
914
- "race_online": 8.157389000189141,
915
- "race_s_online": 3.447674499511777,
916
- "race_net": 6.744342000274628
 
917
  }
918
  },
919
  {
@@ -927,7 +1017,8 @@
927
  "dawid_skene": 0.8895000000000002,
928
  "race_online": 0.9075,
929
  "race_s_online": 0.917,
930
- "race_net": 0.908
 
931
  },
932
  "se": {
933
  "receiver_only": 0.010664778478712061,
@@ -935,15 +1026,17 @@
935
  "dawid_skene": 0.009013184786744363,
936
  "race_online": 0.006199798383818621,
937
  "race_s_online": 0.0060868711174132785,
938
- "race_net": 0.007127411872482184
 
939
  },
940
  "ms": {
941
- "receiver_only": 0.30406549922190607,
942
- "majority": 0.2051210003628512,
943
- "dawid_skene": 5.951209000158997,
944
- "race_online": 7.319172499592241,
945
- "race_s_online": 3.24231000013242,
946
- "race_net": 7.515299998885894
 
947
  }
948
  }
949
  ],
@@ -959,7 +1052,8 @@
959
  "dawid_skene": 0.481,
960
  "race_online": 0.6417499999999998,
961
  "race_s_online": 0.5589999999999999,
962
- "race_net": 0.585
 
963
  },
964
  "se": {
965
  "receiver_only": 0.008469799289239383,
@@ -967,15 +1061,17 @@
967
  "dawid_skene": 0.0481285388309265,
968
  "race_online": 0.047997102777146874,
969
  "race_s_online": 0.009959919678390987,
970
- "race_net": 0.015880412463157245
 
971
  },
972
  "ms": {
973
- "receiver_only": 0.30586537468479946,
974
- "majority": 0.2098574996125535,
975
- "dawid_skene": 5.529378874598478,
976
- "race_online": 12.973063125173212,
977
- "race_s_online": 2.466887124683126,
978
- "race_net": 8.118049374843395
 
979
  }
980
  },
981
  {
@@ -989,7 +1085,8 @@
989
  "dawid_skene": 0.5121249999999999,
990
  "race_online": 0.7665,
991
  "race_s_online": 0.6142500000000001,
992
- "race_net": 0.6552499999999999
 
993
  },
994
  "se": {
995
  "receiver_only": 0.007330586606813947,
@@ -997,15 +1094,17 @@
997
  "dawid_skene": 0.04520345499793129,
998
  "race_online": 0.024663611860390604,
999
  "race_s_online": 0.012080588768764541,
1000
- "race_net": 0.01593924010108387
 
1001
  },
1002
  "ms": {
1003
- "receiver_only": 0.2994011251757911,
1004
- "majority": 0.21714787521887047,
1005
- "dawid_skene": 6.0125981249257165,
1006
- "race_online": 11.39390199978152,
1007
- "race_s_online": 2.411334374528451,
1008
- "race_net": 7.713639999792576
 
1009
  }
1010
  },
1011
  {
@@ -1019,7 +1118,8 @@
1019
  "dawid_skene": 0.54925,
1020
  "race_online": 0.813875,
1021
  "race_s_online": 0.75725,
1022
- "race_net": 0.769625
 
1023
  },
1024
  "se": {
1025
  "receiver_only": 0.009068489400115103,
@@ -1027,15 +1127,17 @@
1027
  "dawid_skene": 0.04750858475054798,
1028
  "race_online": 0.017005950245428808,
1029
  "race_s_online": 0.023010120056183973,
1030
- "race_net": 0.018510364495330712
 
1031
  },
1032
  "ms": {
1033
- "receiver_only": 0.30635324992545065,
1034
- "majority": 0.21949550000499585,
1035
- "dawid_skene": 5.978858250500707,
1036
- "race_online": 11.818120249927233,
1037
- "race_s_online": 3.2623380000131874,
1038
- "race_net": 7.954976500286648
 
1039
  }
1040
  },
1041
  {
@@ -1049,7 +1151,8 @@
1049
  "dawid_skene": 0.615,
1050
  "race_online": 0.851125,
1051
  "race_s_online": 0.856625,
1052
- "race_net": 0.8547499999999999
 
1053
  },
1054
  "se": {
1055
  "receiver_only": 0.009996874511566105,
@@ -1057,15 +1160,17 @@
1057
  "dawid_skene": 0.048518359926938995,
1058
  "race_online": 0.012554226131068375,
1059
  "race_s_online": 0.013942151690108665,
1060
- "race_net": 0.01295894961021147
 
1061
  },
1062
  "ms": {
1063
- "receiver_only": 0.28444499980651017,
1064
- "majority": 0.2048301246304618,
1065
- "dawid_skene": 5.572323749674979,
1066
- "race_online": 8.966723625007944,
1067
- "race_s_online": 3.1502536253356084,
1068
- "race_net": 7.966545875206066
 
1069
  }
1070
  },
1071
  {
@@ -1079,7 +1184,8 @@
1079
  "dawid_skene": 0.640125,
1080
  "race_online": 0.881625,
1081
  "race_s_online": 0.907875,
1082
- "race_net": 0.907
 
1083
  },
1084
  "se": {
1085
  "receiver_only": 0.006452712607888246,
@@ -1087,15 +1193,17 @@
1087
  "dawid_skene": 0.045247746836168536,
1088
  "race_online": 0.007728346766935345,
1089
  "race_s_online": 0.00765317540305983,
1090
- "race_net": 0.006725882841679593
 
1091
  },
1092
  "ms": {
1093
- "receiver_only": 0.2880299998651026,
1094
- "majority": 0.20996637522330275,
1095
- "dawid_skene": 5.960517249604891,
1096
- "race_online": 8.65633850003178,
1097
- "race_s_online": 2.988680500038754,
1098
- "race_net": 7.226339874932819
 
1099
  }
1100
  }
1101
  ]
 
19
  "dawid_skene": 0.8105,
20
  "race_online": 0.8424999999999999,
21
  "race_s_online": 0.767,
22
+ "race_net": 0.795,
23
+ "race_net_v2": 0.7945
24
  },
25
  "se": {
26
  "receiver_only": 0.011599568957508724,
 
28
  "dawid_skene": 0.011346254888728704,
29
  "race_online": 0.010243900624273938,
30
  "race_s_online": 0.015034959261667456,
31
+ "race_net": 0.012816005617976295,
32
+ "race_net_v2": 0.015041193436692442
33
  },
34
  "ms": {
35
+ "receiver_only": 0.3061474999412894,
36
+ "majority": 0.19896949925168883,
37
+ "dawid_skene": 5.814404000375362,
38
+ "race_online": 8.176781499969366,
39
+ "race_s_online": 3.4600314993440406,
40
+ "race_net": 10.48869349870074,
41
+ "race_net_v2": 9.215379000124813
42
  }
43
  },
44
  {
 
52
  "dawid_skene": 0.776125,
53
  "race_online": 0.8081249999999999,
54
  "race_s_online": 0.768,
55
+ "race_net": 0.788625,
56
+ "race_net_v2": 0.787625
57
  },
58
  "se": {
59
  "receiver_only": 0.01114618768907109,
 
61
  "dawid_skene": 0.013559446660907665,
62
  "race_online": 0.008879180353501102,
63
  "race_s_online": 0.014380542409798039,
64
+ "race_net": 0.011726672322104004,
65
+ "race_net_v2": 0.010218663990463723
66
  },
67
  "ms": {
68
+ "receiver_only": 0.29731062545579334,
69
+ "majority": 0.19507375009197858,
70
+ "dawid_skene": 5.235779875192748,
71
+ "race_online": 9.054927999613938,
72
+ "race_s_online": 3.391855875179317,
73
+ "race_net": 9.11645887481427,
74
+ "race_net_v2": 8.24997487507062
75
  }
76
  },
77
  {
 
85
  "dawid_skene": 0.6375,
86
  "race_online": 0.7892499999999999,
87
  "race_s_online": 0.749,
88
+ "race_net": 0.771,
89
+ "race_net_v2": 0.7671249999999998
90
  },
91
  "se": {
92
  "receiver_only": 0.01039891821296812,
 
94
  "dawid_skene": 0.019017261895446464,
95
  "race_online": 0.011731501395814602,
96
  "race_s_online": 0.014524117873385634,
97
+ "race_net": 0.012330602175076444,
98
+ "race_net_v2": 0.012045843422110385
99
  },
100
  "ms": {
101
+ "receiver_only": 0.29714937454627943,
102
+ "majority": 0.19901999985449947,
103
+ "dawid_skene": 5.338263125395315,
104
+ "race_online": 12.364271875640043,
105
+ "race_s_online": 3.401136000320548,
106
+ "race_net": 10.021206250485193,
107
+ "race_net_v2": 9.07413887534858
108
  }
109
  },
110
  {
 
118
  "dawid_skene": 0.4035,
119
  "race_online": 0.7675000000000002,
120
  "race_s_online": 0.77625,
121
+ "race_net": 0.796125,
122
+ "race_net_v2": 0.778875
123
  },
124
  "se": {
125
  "receiver_only": 0.010896673804423074,
 
127
  "dawid_skene": 0.01997920794225837,
128
  "race_online": 0.012097262086935207,
129
  "race_s_online": 0.01345884931931404,
130
+ "race_net": 0.010641127466109965,
131
+ "race_net_v2": 0.009411553737295452
132
  },
133
  "ms": {
134
+ "receiver_only": 0.2900992501508881,
135
+ "majority": 0.19295387482998194,
136
+ "dawid_skene": 5.017030500312103,
137
+ "race_online": 10.637302124905545,
138
+ "race_s_online": 2.995525750293382,
139
+ "race_net": 8.675254624904483,
140
+ "race_net_v2": 8.395347499799755
141
  }
142
  },
143
  {
 
151
  "dawid_skene": 0.3197499999999999,
152
  "race_online": 0.7191249999999999,
153
  "race_s_online": 0.757875,
154
+ "race_net": 0.77175,
155
+ "race_net_v2": 0.779375
156
  },
157
  "se": {
158
  "receiver_only": 0.009515776374001231,
 
160
  "dawid_skene": 0.01407002576401337,
161
  "race_online": 0.010241269147425042,
162
  "race_s_online": 0.012261824650108158,
163
+ "race_net": 0.010441952643064424,
164
+ "race_net_v2": 0.00932549428984866
165
  },
166
  "ms": {
167
+ "receiver_only": 0.2997355002207769,
168
+ "majority": 0.2016631249261991,
169
+ "dawid_skene": 5.339377749578489,
170
+ "race_online": 9.776992749948477,
171
+ "race_s_online": 3.001059750204149,
172
+ "race_net": 9.278568875288329,
173
+ "race_net_v2": 8.674088749557995
174
  }
175
  }
176
  ],
 
186
  "dawid_skene": 0.9345000000000001,
187
  "race_online": 0.9435,
188
  "race_s_online": 0.9285,
189
+ "race_net": 0.8755000000000001,
190
+ "race_net_v2": 0.7974999999999999
191
  },
192
  "se": {
193
  "receiver_only": 0.011624865590620823,
 
195
  "dawid_skene": 0.00661343329897565,
196
  "race_online": 0.005927689263110874,
197
  "race_s_online": 0.0063354163241258225,
198
+ "race_net": 0.011607648340641614,
199
+ "race_net_v2": 0.014796536756957693
200
  },
201
  "ms": {
202
+ "receiver_only": 0.3754700001081801,
203
+ "majority": 0.26513550164963817,
204
+ "dawid_skene": 6.373291499585321,
205
+ "race_online": 13.29903199984983,
206
+ "race_s_online": 4.0056480002022,
207
+ "race_net": 20.042500499585003,
208
+ "race_net_v2": 19.248348000473925
209
  }
210
  },
211
  {
 
219
  "dawid_skene": 0.903125,
220
  "race_online": 0.9354999999999999,
221
  "race_s_online": 0.9295,
222
+ "race_net": 0.8885,
223
+ "race_net_v2": 0.8168750000000001
224
  },
225
  "se": {
226
  "receiver_only": 0.009222255689363641,
 
228
  "dawid_skene": 0.007781940230430967,
229
  "race_online": 0.005767906899387335,
230
  "race_s_online": 0.006732848579910288,
231
+ "race_net": 0.008958934088383505,
232
+ "race_net_v2": 0.0124785753894425
233
  },
234
  "ms": {
235
+ "receiver_only": 0.3751716253646009,
236
+ "majority": 0.2645123747697653,
237
+ "dawid_skene": 6.518947875065351,
238
+ "race_online": 13.857671624919021,
239
+ "race_s_online": 3.830718749895823,
240
+ "race_net": 20.30240037515796,
241
+ "race_net_v2": 19.082624750353716
242
  }
243
  },
244
  {
 
252
  "dawid_skene": 0.7207500000000001,
253
  "race_online": 0.928125,
254
  "race_s_online": 0.9192500000000001,
255
+ "race_net": 0.8952500000000001,
256
+ "race_net_v2": 0.82975
257
  },
258
  "se": {
259
  "receiver_only": 0.011123735883236351,
 
261
  "dawid_skene": 0.006290121222997218,
262
  "race_online": 0.004885805332798266,
263
  "race_s_online": 0.007794429100325434,
264
+ "race_net": 0.007611463394118103,
265
+ "race_net_v2": 0.010813619884201585
266
  },
267
  "ms": {
268
+ "receiver_only": 0.37503025055229955,
269
+ "majority": 0.26065224983540247,
270
+ "dawid_skene": 6.40530600026068,
271
+ "race_online": 13.541720375087607,
272
+ "race_s_online": 3.790755125010037,
273
+ "race_net": 20.515281000371033,
274
+ "race_net_v2": 18.779808250201313
275
  }
276
  },
277
  {
 
285
  "dawid_skene": 0.4763749999999999,
286
  "race_online": 0.9040000000000001,
287
  "race_s_online": 0.8953749999999999,
288
+ "race_net": 0.8865000000000002,
289
+ "race_net_v2": 0.819125
290
  },
291
  "se": {
292
  "receiver_only": 0.010874281585465773,
 
294
  "dawid_skene": 0.034240178792611466,
295
  "race_online": 0.004806896087913694,
296
  "race_s_online": 0.01288806982251415,
297
+ "race_net": 0.006323072828933728,
298
+ "race_net_v2": 0.009121189272786743
299
  },
300
  "ms": {
301
+ "receiver_only": 0.398358249640296,
302
+ "majority": 0.2645686249707069,
303
+ "dawid_skene": 6.692075125101837,
304
+ "race_online": 15.21854649990928,
305
+ "race_s_online": 4.127775999677397,
306
+ "race_net": 21.835635375282436,
307
+ "race_net_v2": 20.03746149989638
308
  }
309
  },
310
  {
 
318
  "dawid_skene": 0.32775,
319
  "race_online": 0.909375,
320
  "race_s_online": 0.882,
321
+ "race_net": 0.86175,
322
+ "race_net_v2": 0.8106249999999999
323
  },
324
  "se": {
325
  "receiver_only": 0.013745453793891277,
 
327
  "dawid_skene": 0.020215170664627097,
328
  "race_online": 0.006884936001881206,
329
  "race_s_online": 0.01112092622041887,
330
+ "race_net": 0.011124438188061449,
331
+ "race_net_v2": 0.011499830161789347
332
  },
333
  "ms": {
334
+ "receiver_only": 0.38165412502166873,
335
+ "majority": 0.2747978749084723,
336
+ "dawid_skene": 6.678269125359293,
337
+ "race_online": 15.364066500069384,
338
+ "race_s_online": 4.1182576251230785,
339
+ "race_net": 20.33903862502484,
340
+ "race_net_v2": 19.392319250073342
341
  }
342
  },
343
  {
 
351
  "dawid_skene": 0.260625,
352
  "race_online": 0.8717499999999999,
353
  "race_s_online": 0.85975,
354
+ "race_net": 0.85825,
355
+ "race_net_v2": 0.826375
356
  },
357
  "se": {
358
  "receiver_only": 0.008404611829228047,
 
360
  "dawid_skene": 0.006663508366468822,
361
  "race_online": 0.006157058957002116,
362
  "race_s_online": 0.008574854226166181,
363
+ "race_net": 0.006235232553802621,
364
+ "race_net_v2": 0.008074796824069317
365
  },
366
  "ms": {
367
+ "receiver_only": 0.37751649983874813,
368
+ "majority": 0.2603556247322558,
369
+ "dawid_skene": 6.733221374815912,
370
+ "race_online": 17.367023749784494,
371
+ "race_s_online": 4.412451750113178,
372
+ "race_net": 21.10360424967439,
373
+ "race_net_v2": 19.559220875407846
374
  }
375
  },
376
  {
 
384
  "dawid_skene": 0.215625,
385
  "race_online": 0.8138750000000001,
386
  "race_s_online": 0.8266249999999999,
387
+ "race_net": 0.836125,
388
+ "race_net_v2": 0.849375
389
  },
390
  "se": {
391
  "receiver_only": 0.012354654993159462,
 
393
  "dawid_skene": 0.005433906858789539,
394
  "race_online": 0.008784707949044183,
395
  "race_s_online": 0.015569677059913604,
396
+ "race_net": 0.008138402407721065,
397
+ "race_net_v2": 0.007152086671035246
398
  },
399
  "ms": {
400
+ "receiver_only": 0.3864080001676484,
401
+ "majority": 0.26372462457402435,
402
+ "dawid_skene": 6.632045374999507,
403
+ "race_online": 18.647589499778405,
404
+ "race_s_online": 4.241555624957982,
405
+ "race_net": 21.35055975031719,
406
+ "race_net_v2": 19.55898825008262
407
  }
408
  },
409
  {
 
417
  "dawid_skene": 0.20875000000000005,
418
  "race_online": 0.786625,
419
  "race_s_online": 0.834375,
420
+ "race_net": 0.8320000000000001,
421
+ "race_net_v2": 0.8547500000000001
422
  },
423
  "se": {
424
  "receiver_only": 0.009964938534682487,
 
426
  "dawid_skene": 0.0056527095272975066,
427
  "race_online": 0.008309021828711245,
428
  "race_s_online": 0.012361577316426894,
429
+ "race_net": 0.010979526401443729,
430
+ "race_net_v2": 0.007232522035915269
431
  },
432
  "ms": {
433
+ "receiver_only": 0.38573250026274764,
434
+ "majority": 0.26304799962417746,
435
+ "dawid_skene": 6.581845500249983,
436
+ "race_online": 18.896191375233684,
437
+ "race_s_online": 4.315268875416223,
438
+ "race_net": 21.000340624823366,
439
+ "race_net_v2": 19.36698899999101
440
  }
441
  }
442
  ],
 
452
  "dawid_skene": 0.428125,
453
  "race_online": 0.68125,
454
  "race_s_online": 0.728125,
455
+ "race_net": 0.675,
456
+ "race_net_v2": 0.6546875
457
  },
458
  "se": {
459
  "receiver_only": 0.03354101966249684,
 
461
  "dawid_skene": 0.02915643421699574,
462
  "race_online": 0.03937996000505841,
463
  "race_s_online": 0.03866043075613618,
464
+ "race_net": 0.05045689682788668,
465
+ "race_net_v2": 0.049087125587444005
466
  },
467
  "ms": {
468
+ "receiver_only": 0.2015668744661525,
469
+ "majority": 0.12902774960821262,
470
+ "dawid_skene": 3.5532863748812815,
471
+ "race_online": 2.5358972500271193,
472
+ "race_s_online": 1.4239648755847156,
473
+ "race_net": 2.2430451253967476,
474
+ "race_net_v2": 2.4243708753601823
475
  }
476
  },
477
  {
 
485
  "dawid_skene": 0.43359375,
486
  "race_online": 0.696875,
487
  "race_s_online": 0.70234375,
488
+ "race_net": 0.6921875,
489
+ "race_net_v2": 0.68046875
490
  },
491
  "se": {
492
  "receiver_only": 0.022910747341586216,
 
494
  "dawid_skene": 0.029082549367866205,
495
  "race_online": 0.021690066490792508,
496
  "race_s_online": 0.02513331155278538,
497
+ "race_net": 0.023466128348909623,
498
+ "race_net_v2": 0.02180443417310743
499
  },
500
  "ms": {
501
+ "receiver_only": 0.19757725021918304,
502
+ "majority": 0.13672512500306766,
503
+ "dawid_skene": 3.6808553747960104,
504
+ "race_online": 3.887610749416126,
505
+ "race_s_online": 1.7339892499421694,
506
+ "race_net": 2.554789374698885,
507
+ "race_net_v2": 2.118542999824058
508
  }
509
  },
510
  {
 
518
  "dawid_skene": 0.377734375,
519
  "race_online": 0.759375,
520
  "race_s_online": 0.7390625,
521
+ "race_net": 0.75859375,
522
+ "race_net_v2": 0.7515625
523
  },
524
  "se": {
525
  "receiver_only": 0.0189314140385762,
 
527
  "dawid_skene": 0.018890061186271838,
528
  "race_online": 0.021563066115756823,
529
  "race_s_online": 0.016649975365972767,
530
+ "race_net": 0.01679142353396147,
531
+ "race_net_v2": 0.016111564776000186
532
  },
533
  "ms": {
534
+ "receiver_only": 0.2522683753340971,
535
+ "majority": 0.1849532497999462,
536
+ "dawid_skene": 5.223806250114649,
537
+ "race_online": 5.601473125580014,
538
+ "race_s_online": 1.9886899999619345,
539
+ "race_net": 3.6933076248715224,
540
+ "race_net_v2": 3.429246125051577
541
  }
542
  },
543
  {
 
551
  "dawid_skene": 0.3736328125,
552
  "race_online": 0.7607421875,
553
  "race_s_online": 0.7474609375,
554
+ "race_net": 0.77265625,
555
+ "race_net_v2": 0.765625
556
  },
557
  "se": {
558
  "receiver_only": 0.01411773008434341,
 
560
  "dawid_skene": 0.015500664973607532,
561
  "race_online": 0.014840858579409191,
562
  "race_s_online": 0.014182095073673386,
563
+ "race_net": 0.012145862367746885,
564
+ "race_net_v2": 0.009548364501234622
565
  },
566
  "ms": {
567
+ "receiver_only": 0.261966874859354,
568
+ "majority": 0.17756874967744807,
569
+ "dawid_skene": 4.47720512511296,
570
+ "race_online": 7.934072999660202,
571
+ "race_s_online": 2.817375499944319,
572
+ "race_net": 4.958349249818639,
573
+ "race_net_v2": 4.499349500065364
574
  }
575
  },
576
  {
 
584
  "dawid_skene": 0.39248046875,
585
  "race_online": 0.76357421875,
586
  "race_s_online": 0.7654296875,
587
+ "race_net": 0.78046875,
588
+ "race_net_v2": 0.77744140625
589
  },
590
  "se": {
591
  "receiver_only": 0.007519404422631738,
 
593
  "dawid_skene": 0.021578042883029624,
594
  "race_online": 0.008270827158069627,
595
  "race_s_online": 0.00839832394545325,
596
+ "race_net": 0.008616358754345929,
597
+ "race_net_v2": 0.008351151173438959
598
  },
599
  "ms": {
600
+ "receiver_only": 0.3078841250498954,
601
+ "majority": 0.21446375012601493,
602
+ "dawid_skene": 5.958042250085782,
603
+ "race_online": 14.479490125040684,
604
+ "race_s_online": 3.729335749767415,
605
+ "race_net": 10.034621249815245,
606
+ "race_net_v2": 8.517950625446247
607
  }
608
  },
609
  {
 
617
  "dawid_skene": 0.3998046875,
618
  "race_online": 0.760302734375,
619
  "race_s_online": 0.784521484375,
620
+ "race_net": 0.776904296875,
621
+ "race_net_v2": 0.76953125
622
  },
623
  "se": {
624
  "receiver_only": 0.006488962044731711,
 
626
  "dawid_skene": 0.016211437539351194,
627
  "race_online": 0.006989239119019292,
628
  "race_s_online": 0.00700620641771385,
629
+ "race_net": 0.007469099284029169,
630
+ "race_net_v2": 0.00643420562891763
631
  },
632
  "ms": {
633
+ "receiver_only": 0.36633562558563426,
634
+ "majority": 0.2592798753084935,
635
+ "dawid_skene": 7.683889999952953,
636
+ "race_online": 27.06556174985053,
637
+ "race_s_online": 5.113057624839712,
638
+ "race_net": 14.934778874930998,
639
+ "race_net_v2": 13.464431124930343
640
  }
641
  }
642
  ],
 
652
  "dawid_skene": 0.32825,
653
  "race_online": 0.7200000000000001,
654
  "race_s_online": 0.73275,
655
+ "race_net": 0.7358749999999998,
656
+ "race_net_v2": 0.7418750000000001
657
  },
658
  "se": {
659
  "receiver_only": 0.010700467279516348,
 
661
  "dawid_skene": 0.012532582535136162,
662
  "race_online": 0.010700467279516348,
663
  "race_s_online": 0.01150258123205396,
664
+ "race_net": 0.012435175662209197,
665
+ "race_net_v2": 0.011240099810499907
666
  },
667
  "ms": {
668
+ "receiver_only": 0.26661287529350375,
669
+ "majority": 0.18646775015440653,
670
+ "dawid_skene": 5.166177875253197,
671
+ "race_online": 6.54958387485749,
672
+ "race_s_online": 2.289614875053303,
673
+ "race_net": 5.2806331245847105,
674
+ "race_net_v2": 4.847107499927006
675
  }
676
  },
677
  {
 
685
  "dawid_skene": 0.4035,
686
  "race_online": 0.7675000000000002,
687
  "race_s_online": 0.77625,
688
+ "race_net": 0.796125,
689
+ "race_net_v2": 0.778875
690
  },
691
  "se": {
692
  "receiver_only": 0.010896673804423074,
 
694
  "dawid_skene": 0.01997920794225837,
695
  "race_online": 0.012097262086935207,
696
  "race_s_online": 0.01345884931931404,
697
+ "race_net": 0.010641127466109965,
698
+ "race_net_v2": 0.009411553737295452
699
  },
700
  "ms": {
701
+ "receiver_only": 0.30133337486404344,
702
+ "majority": 0.19793474984908244,
703
+ "dawid_skene": 5.603325125093761,
704
+ "race_online": 11.433071500050573,
705
+ "race_s_online": 3.3942548752747825,
706
+ "race_net": 7.382696999911786,
707
+ "race_net_v2": 6.455946999722073
708
  }
709
  },
710
  {
 
718
  "dawid_skene": 0.42524999999999996,
719
  "race_online": 0.81275,
720
  "race_s_online": 0.781125,
721
+ "race_net": 0.795625,
722
+ "race_net_v2": 0.772375
723
  },
724
  "se": {
725
  "receiver_only": 0.012674284989694687,
 
727
  "dawid_skene": 0.022856823379463733,
728
  "race_online": 0.008817418840000741,
729
  "race_s_online": 0.014856054111035003,
730
+ "race_net": 0.013010662694497929,
731
+ "race_net_v2": 0.011275630525606976
732
  },
733
  "ms": {
734
+ "receiver_only": 0.3027661250598612,
735
+ "majority": 0.213934499925017,
736
+ "dawid_skene": 5.747407124545134,
737
+ "race_online": 10.746808124622476,
738
+ "race_s_online": 3.656911499774651,
739
+ "race_net": 8.427007250020324,
740
+ "race_net_v2": 7.580552000035823
741
  }
742
  },
743
  {
 
751
  "dawid_skene": 0.42275,
752
  "race_online": 0.8430000000000002,
753
  "race_s_online": 0.8099999999999999,
754
+ "race_net": 0.8172499999999999,
755
+ "race_net_v2": 0.782125
756
  },
757
  "se": {
758
  "receiver_only": 0.006725882841679595,
 
760
  "dawid_skene": 0.025814785007820612,
761
  "race_online": 0.008222682044199448,
762
  "race_s_online": 0.012826973532365296,
763
+ "race_net": 0.01108475868027807,
764
+ "race_net_v2": 0.008439629360937598
765
  },
766
  "ms": {
767
+ "receiver_only": 0.32585312465016614,
768
+ "majority": 0.24184687481465517,
769
+ "dawid_skene": 6.358325749715732,
770
+ "race_online": 12.686153499544162,
771
+ "race_s_online": 3.977208625201456,
772
+ "race_net": 10.73325399966052,
773
+ "race_net_v2": 10.479275625129958
774
  }
775
  },
776
  {
 
784
  "dawid_skene": 0.39075000000000004,
785
  "race_online": 0.8685,
786
  "race_s_online": 0.83575,
787
+ "race_net": 0.8396250000000001,
788
+ "race_net_v2": 0.7826250000000001
789
  },
790
  "se": {
791
  "receiver_only": 0.01198905751091386,
 
793
  "dawid_skene": 0.02485149643381662,
794
  "race_online": 0.007612489737267302,
795
  "race_s_online": 0.015323694887330533,
796
+ "race_net": 0.011038674909154628,
797
+ "race_net_v2": 0.011122155535236863
798
  },
799
  "ms": {
800
+ "receiver_only": 0.3402389997972932,
801
+ "majority": 0.24244500036729733,
802
+ "dawid_skene": 6.153978499696677,
803
+ "race_online": 12.800111000387915,
804
+ "race_s_online": 4.269729000498046,
805
+ "race_net": 12.391785249974419,
806
+ "race_net_v2": 11.016863125178133
807
  }
808
  },
809
  {
 
817
  "dawid_skene": 0.32775,
818
  "race_online": 0.909375,
819
  "race_s_online": 0.882,
820
+ "race_net": 0.86175,
821
+ "race_net_v2": 0.8106249999999999
822
  },
823
  "se": {
824
  "receiver_only": 0.013745453793891277,
 
826
  "dawid_skene": 0.020215170664627097,
827
  "race_online": 0.006884936001881206,
828
  "race_s_online": 0.01112092622041887,
829
+ "race_net": 0.011124438188061449,
830
+ "race_net_v2": 0.011499830161789347
831
  },
832
  "ms": {
833
+ "receiver_only": 0.37591437535411387,
834
+ "majority": 0.27939462529502634,
835
+ "dawid_skene": 6.621083124628058,
836
+ "race_online": 15.470638625401989,
837
+ "race_s_online": 4.270621000614483,
838
+ "race_net": 14.900676874958663,
839
+ "race_net_v2": 12.803076999489349
840
  }
841
  },
842
  {
 
850
  "dawid_skene": 0.34012500000000007,
851
  "race_online": 0.9337500000000001,
852
  "race_s_online": 0.9324999999999999,
853
+ "race_net": 0.89075,
854
+ "race_net_v2": 0.807625
855
  },
856
  "se": {
857
  "receiver_only": 0.010278010507875537,
 
859
  "dawid_skene": 0.017458410401580096,
860
  "race_online": 0.006114787404317504,
861
  "race_s_online": 0.005326818938165631,
862
+ "race_net": 0.008338727420895831,
863
+ "race_net_v2": 0.00973632855598043
864
  },
865
  "ms": {
866
+ "receiver_only": 0.42583925028338854,
867
+ "majority": 0.311263999719813,
868
+ "dawid_skene": 7.201720999773897,
869
+ "race_online": 23.04028750040743,
870
+ "race_s_online": 4.8597763752695755,
871
+ "race_net": 20.337359250333975,
872
+ "race_net_v2": 19.45768787504676
873
  }
874
  }
875
  ],
 
885
  "dawid_skene": 0.8470000000000001,
886
  "race_online": 0.8240000000000001,
887
  "race_s_online": 0.544,
888
+ "race_net": 0.571,
889
+ "race_net_v2": 0.5980000000000001
890
  },
891
  "se": {
892
  "receiver_only": 0.011235546270653686,
 
894
  "dawid_skene": 0.0095420123663722,
895
  "race_online": 0.028225874654295477,
896
  "race_s_online": 0.01600624877977348,
897
+ "race_net": 0.026276415280627602,
898
+ "race_net_v2": 0.041319486928082724
899
  },
900
  "ms": {
901
+ "receiver_only": 0.28864449996035546,
902
+ "majority": 0.19890549992851447,
903
+ "dawid_skene": 5.694833998859394,
904
+ "race_online": 9.291724499234988,
905
+ "race_s_online": 1.4365155002451502,
906
+ "race_net": 7.469124499948521,
907
+ "race_net_v2": 6.639774499490159
908
  }
909
  },
910
  {
 
918
  "dawid_skene": 0.8394999999999999,
919
  "race_online": 0.8625,
920
  "race_s_online": 0.5934999999999999,
921
+ "race_net": 0.6349999999999999,
922
+ "race_net_v2": 0.6945000000000001
923
  },
924
  "se": {
925
  "receiver_only": 0.008901544809750717,
 
927
  "dawid_skene": 0.00826059925187029,
928
  "race_online": 0.006887488656977954,
929
  "race_s_online": 0.019445757892147073,
930
+ "race_net": 0.025283393759541063,
931
+ "race_net_v2": 0.03729259309836203
932
  },
933
  "ms": {
934
+ "receiver_only": 0.28297199969529174,
935
+ "majority": 0.19331000021338696,
936
+ "dawid_skene": 5.376616999456019,
937
+ "race_online": 10.090314500303066,
938
+ "race_s_online": 2.352800499465957,
939
+ "race_net": 6.609885999750986,
940
+ "race_net_v2": 5.823647499710205
941
  }
942
  },
943
  {
 
951
  "dawid_skene": 0.8500000000000002,
952
  "race_online": 0.869,
953
  "race_s_online": 0.733,
954
+ "race_net": 0.7735000000000001,
955
+ "race_net_v2": 0.8245000000000001
956
  },
957
  "se": {
958
  "receiver_only": 0.008132496541653123,
 
960
  "dawid_skene": 0.009460443964212252,
961
  "race_online": 0.0072422372233999655,
962
  "race_s_online": 0.025220031720836512,
963
+ "race_net": 0.019522743147416557,
964
+ "race_net_v2": 0.018158675612499942
965
  },
966
  "ms": {
967
+ "receiver_only": 0.2912190002462012,
968
+ "majority": 0.20162950022495352,
969
+ "dawid_skene": 5.991226999867649,
970
+ "race_online": 9.862000000794069,
971
+ "race_s_online": 3.500417999930505,
972
+ "race_net": 7.715689500400913,
973
+ "race_net_v2": 8.173166999768
974
  }
975
  },
976
  {
 
984
  "dawid_skene": 0.8629999999999999,
985
  "race_online": 0.8854999999999998,
986
  "race_s_online": 0.8604999999999998,
987
+ "race_net": 0.8605,
988
+ "race_net_v2": 0.8775000000000001
989
  },
990
  "se": {
991
  "receiver_only": 0.009051933495115837,
 
993
  "dawid_skene": 0.010818964830333816,
994
  "race_online": 0.0066511277239277265,
995
  "race_s_online": 0.0157396791581023,
996
+ "race_net": 0.01540251602823383,
997
+ "race_net_v2": 0.012101136310280947
998
  },
999
  "ms": {
1000
+ "receiver_only": 0.274241499027994,
1001
+ "majority": 0.19889800023520365,
1002
+ "dawid_skene": 5.620339000415697,
1003
+ "race_online": 8.123171999613987,
1004
+ "race_s_online": 3.1436230001418153,
1005
+ "race_net": 6.302750000031665,
1006
+ "race_net_v2": 5.640695499096182
1007
  }
1008
  },
1009
  {
 
1017
  "dawid_skene": 0.8895000000000002,
1018
  "race_online": 0.9075,
1019
  "race_s_online": 0.917,
1020
+ "race_net": 0.908,
1021
+ "race_net_v2": 0.9179999999999999
1022
  },
1023
  "se": {
1024
  "receiver_only": 0.010664778478712061,
 
1026
  "dawid_skene": 0.009013184786744363,
1027
  "race_online": 0.006199798383818621,
1028
  "race_s_online": 0.0060868711174132785,
1029
+ "race_net": 0.007127411872482184,
1030
+ "race_net_v2": 0.005899152481501045
1031
  },
1032
  "ms": {
1033
+ "receiver_only": 0.27915350074181333,
1034
+ "majority": 0.21367349927459145,
1035
+ "dawid_skene": 5.878158000996336,
1036
+ "race_online": 7.460996999725467,
1037
+ "race_s_online": 3.09438299973408,
1038
+ "race_net": 6.904907999341958,
1039
+ "race_net_v2": 6.400322500667244
1040
  }
1041
  }
1042
  ],
 
1052
  "dawid_skene": 0.481,
1053
  "race_online": 0.6417499999999998,
1054
  "race_s_online": 0.5589999999999999,
1055
+ "race_net": 0.585,
1056
+ "race_net_v2": 0.6377499999999999
1057
  },
1058
  "se": {
1059
  "receiver_only": 0.008469799289239383,
 
1061
  "dawid_skene": 0.0481285388309265,
1062
  "race_online": 0.047997102777146874,
1063
  "race_s_online": 0.009959919678390987,
1064
+ "race_net": 0.015880412463157245,
1065
+ "race_net_v2": 0.027365523473889545
1066
  },
1067
  "ms": {
1068
+ "receiver_only": 0.301404374795311,
1069
+ "majority": 0.20358787537588796,
1070
+ "dawid_skene": 5.614235500161158,
1071
+ "race_online": 12.894517499944413,
1072
+ "race_s_online": 2.462718750393833,
1073
+ "race_net": 7.945240750132143,
1074
+ "race_net_v2": 6.660816374505885
1075
  }
1076
  },
1077
  {
 
1085
  "dawid_skene": 0.5121249999999999,
1086
  "race_online": 0.7665,
1087
  "race_s_online": 0.6142500000000001,
1088
+ "race_net": 0.6552499999999999,
1089
+ "race_net_v2": 0.713
1090
  },
1091
  "se": {
1092
  "receiver_only": 0.007330586606813947,
 
1094
  "dawid_skene": 0.04520345499793129,
1095
  "race_online": 0.024663611860390604,
1096
  "race_s_online": 0.012080588768764541,
1097
+ "race_net": 0.01593924010108387,
1098
+ "race_net_v2": 0.023493749168661864
1099
  },
1100
  "ms": {
1101
+ "receiver_only": 0.2900674999182229,
1102
+ "majority": 0.20731800009343715,
1103
+ "dawid_skene": 5.664082875227905,
1104
+ "race_online": 10.911093124832405,
1105
+ "race_s_online": 2.3391426248053904,
1106
+ "race_net": 7.407797250152726,
1107
+ "race_net_v2": 6.672377999848322
1108
  }
1109
  },
1110
  {
 
1118
  "dawid_skene": 0.54925,
1119
  "race_online": 0.813875,
1120
  "race_s_online": 0.75725,
1121
+ "race_net": 0.769625,
1122
+ "race_net_v2": 0.784625
1123
  },
1124
  "se": {
1125
  "receiver_only": 0.009068489400115103,
 
1127
  "dawid_skene": 0.04750858475054798,
1128
  "race_online": 0.017005950245428808,
1129
  "race_s_online": 0.023010120056183973,
1130
+ "race_net": 0.018510364495330712,
1131
+ "race_net_v2": 0.019293194233977944
1132
  },
1133
  "ms": {
1134
+ "receiver_only": 0.3033424998193368,
1135
+ "majority": 0.20387800054777472,
1136
+ "dawid_skene": 5.6705407500885485,
1137
+ "race_online": 12.159222749687615,
1138
+ "race_s_online": 3.3824668746547104,
1139
+ "race_net": 7.640688625315306,
1140
+ "race_net_v2": 6.587755375448978
1141
  }
1142
  },
1143
  {
 
1151
  "dawid_skene": 0.615,
1152
  "race_online": 0.851125,
1153
  "race_s_online": 0.856625,
1154
+ "race_net": 0.8547499999999999,
1155
+ "race_net_v2": 0.848625
1156
  },
1157
  "se": {
1158
  "receiver_only": 0.009996874511566105,
 
1160
  "dawid_skene": 0.048518359926938995,
1161
  "race_online": 0.012554226131068375,
1162
  "race_s_online": 0.013942151690108665,
1163
+ "race_net": 0.01295894961021147,
1164
+ "race_net_v2": 0.01345998119426621
1165
  },
1166
  "ms": {
1167
+ "receiver_only": 0.2874411252378195,
1168
+ "majority": 0.20604112523869844,
1169
+ "dawid_skene": 5.584663249919686,
1170
+ "race_online": 10.130999999773849,
1171
+ "race_s_online": 3.365901125334858,
1172
+ "race_net": 7.443539124551535,
1173
+ "race_net_v2": 6.192349875391301
1174
  }
1175
  },
1176
  {
 
1184
  "dawid_skene": 0.640125,
1185
  "race_online": 0.881625,
1186
  "race_s_online": 0.907875,
1187
+ "race_net": 0.907,
1188
+ "race_net_v2": 0.8953749999999999
1189
  },
1190
  "se": {
1191
  "receiver_only": 0.006452712607888246,
 
1193
  "dawid_skene": 0.045247746836168536,
1194
  "race_online": 0.007728346766935345,
1195
  "race_s_online": 0.00765317540305983,
1196
+ "race_net": 0.006725882841679593,
1197
+ "race_net_v2": 0.006678498614958304
1198
  },
1199
  "ms": {
1200
+ "receiver_only": 0.2965048745409149,
1201
+ "majority": 0.20842812500632135,
1202
+ "dawid_skene": 5.533301250125078,
1203
+ "race_online": 8.693224875287342,
1204
+ "race_s_online": 3.225045999897702,
1205
+ "race_net": 7.62897875051749,
1206
+ "race_net_v2": 6.871246500168127
1207
  }
1208
  }
1209
  ]
racenet_analysis.py CHANGED
@@ -28,6 +28,9 @@ METHODS = {
28
  "race_s_online": lambda P: fusion.race_s(P, P, 0)[:2],
29
  "race_net": lambda P: racenet.race_net(P, P, 0),
30
  }
 
 
 
31
  ATTACKS = ("invert", "collude", "mix", "noise")
32
  SEEDS = 20
33
 
 
28
  "race_s_online": lambda P: fusion.race_s(P, P, 0)[:2],
29
  "race_net": lambda P: racenet.race_net(P, P, 0),
30
  }
31
+ V2 = Path(__file__).resolve().parents[1] / "models/racenet/candidates/v2_local"
32
+ if racenet.available(V2): # the post-hoc v2 checkpoint (wider training distribution), shown next to the released model
33
+ METHODS["race_net_v2"] = lambda P: racenet.race_net(P, P, 0, path=V2)
34
  ATTACKS = ("invert", "collude", "mix", "noise")
35
  SEEDS = 20
36
 
racenet_scaling.png CHANGED

Git LFS Details

  • SHA256: 8c905bb05a1c10e53674167644ac71d157905ec3419e5ae8a6a3131ad74e6e8e
  • Pointer size: 131 Bytes
  • Size of remote file: 373 kB

Git LFS Details

  • SHA256: 750c0709facb1a1df64e6f93cf825fcca41e3280d0f36b4957e7a830e4bde45c
  • Pointer size: 131 Bytes
  • Size of remote file: 400 kB
selection.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
  "rule": "highest mean accuracy on Track A0 (development); ties within 0.1 point go to the higher synthetic accuracy (v2 generator)",
3
- "selected": "v1 \u00b7 local CPU",
4
  "candidates": [
5
  {
6
- "name": "v1 \u00b7 local CPU",
7
  "generator": "v1",
8
  "steps": 6000,
9
  "synthetic_v1": 0.9339709523211542,
@@ -27,6 +27,58 @@
27
  "liar_majority": 0.804509090909091,
28
  "worst": 0.5545454545454546
29
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  }
31
  ]
32
  }
 
1
  {
2
  "rule": "highest mean accuracy on Track A0 (development); ties within 0.1 point go to the higher synthetic accuracy (v2 generator)",
3
+ "selected": "v1 \u00b7 local CPU \u00b7 6k steps",
4
  "candidates": [
5
  {
6
+ "name": "v1 \u00b7 local CPU \u00b7 6k steps",
7
  "generator": "v1",
8
  "steps": 6000,
9
  "synthetic_v1": 0.9339709523211542,
 
27
  "liar_majority": 0.804509090909091,
28
  "worst": 0.5545454545454546
29
  }
30
+ },
31
+ {
32
+ "name": "v1 \u00b7 Kaggle CPU \u00b7 15k steps",
33
+ "generator": "v1",
34
+ "steps": 15000,
35
+ "synthetic_v1": 0.9388268508897906,
36
+ "synthetic_v2": 0.9720712709958418,
37
+ "dev_A0": {
38
+ "mean": 0.9016338028169015,
39
+ "clean": 0.914,
40
+ "liar_majority": 0.8838400000000002,
41
+ "worst": 0.64
42
+ },
43
+ "selected": false,
44
+ "test_SO101": {
45
+ "mean": 0.8088860435339309,
46
+ "clean": 0.8400000000000001,
47
+ "liar_majority": 0.7827636363636363,
48
+ "worst": 0.509090909090909
49
+ },
50
+ "test_Panda": {
51
+ "mean": 0.8225608194622279,
52
+ "clean": 0.8454545454545453,
53
+ "liar_majority": 0.8055999999999999,
54
+ "worst": 0.5545454545454546
55
+ }
56
+ },
57
+ {
58
+ "name": "v2 \u00b7 local CPU \u00b7 7k steps",
59
+ "generator": "v2",
60
+ "steps": 7000,
61
+ "synthetic_v1": 0.9257052827695175,
62
+ "synthetic_v2": 0.9694320895095003,
63
+ "dev_A0": {
64
+ "mean": 0.9017464788732394,
65
+ "clean": 0.9179999999999999,
66
+ "liar_majority": 0.8808000000000001,
67
+ "worst": 0.64
68
+ },
69
+ "selected": false,
70
+ "test_SO101": {
71
+ "mean": 0.8259667093469911,
72
+ "clean": 0.8690909090909091,
73
+ "liar_majority": 0.7957818181818183,
74
+ "worst": 0.5181818181818182
75
+ },
76
+ "test_Panda": {
77
+ "mean": 0.8353393085787453,
78
+ "clean": 0.8854545454545455,
79
+ "liar_majority": 0.8220363636363636,
80
+ "worst": 0.5454545454545454
81
+ }
82
  }
83
  ]
84
  }