rafmacalaba commited on
Commit
8ddf81e
·
verified ·
1 Parent(s): d7cd397

Upload holdout_metrics.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. holdout_metrics.json +368 -440
holdout_metrics.json CHANGED
@@ -2,259 +2,227 @@
2
  "real": {
3
  "n_eval": 5464,
4
  "overall": {
5
- "tp": 4815,
6
- "fp": 1151,
7
- "fn": 1427,
8
- "precision": 0.8070734160241367,
9
- "recall": 0.7713873758410765,
10
- "f05": 0.7996744834916627,
11
- "f1": 0.788826998689384
12
  },
13
  "per_key": {
14
  "producer": {
15
- "tp": 1062,
16
- "fp": 372,
17
- "fn": 387,
18
- "precision": 0.7405857740585774,
19
- "recall": 0.7329192546583851,
20
- "f05": 0.7390396659707724,
21
- "f1": 0.7367325702393341
22
  },
23
  "year": {
24
- "tp": 1008,
25
- "fp": 214,
26
- "fn": 342,
27
- "precision": 0.8248772504091653,
28
- "recall": 0.7466666666666667,
29
- "f05": 0.8079512664315486,
30
- "f1": 0.7838258164852255
31
  },
32
  "geography": {
33
- "tp": 1548,
34
- "fp": 399,
35
- "fn": 479,
36
- "precision": 0.7950693374422187,
37
- "recall": 0.7636901825357671,
38
- "f05": 0.7885888945491595,
39
- "f1": 0.7790639154504279
40
  },
41
  "acronym": {
42
- "tp": 1197,
43
- "fp": 166,
44
- "fn": 219,
45
- "precision": 0.8782098312545855,
46
- "recall": 0.8453389830508474,
47
- "f05": 0.871432731508445,
48
- "f1": 0.8614609571788414
49
  }
50
  },
51
  "verbatim": {
52
- "emitted": 5966,
53
- "verbatim": 5950,
54
- "rate": 0.9973181361045927,
55
  "per_key": {
56
  "producer": {
57
- "emitted": 1434,
58
- "verbatim": 1427
59
  },
60
  "year": {
61
- "emitted": 1222,
62
- "verbatim": 1222
63
  },
64
  "geography": {
65
- "emitted": 1947,
66
- "verbatim": 1941
67
  },
68
  "acronym": {
69
- "emitted": 1363,
70
- "verbatim": 1360
71
  }
72
  }
73
  },
74
  "usage_impact": {
75
- "data_type_macro_f1": 0.7162670036046718,
76
- "usage_action_macro_f1": 0.5258974005820316,
77
- "impact_label_macro_f1": 0.43798544045970644,
78
  "per_label": {
79
  "data_type": {
80
  "administrative": {
81
- "tp": 197,
82
- "fp": 50,
83
- "fn": 76,
84
- "precision": 0.7975708502024291,
85
- "recall": 0.7216117216117216,
86
- "f1": 0.7576923076923077
87
  },
88
  "database": {
89
- "tp": 268,
90
- "fp": 84,
91
- "fn": 101,
92
- "precision": 0.7613636363636364,
93
- "recall": 0.7262872628726287,
94
- "f1": 0.7434119278779473
95
  },
96
  "geospatial": {
97
- "tp": 102,
98
- "fp": 30,
99
- "fn": 20,
100
- "precision": 0.7727272727272727,
101
- "recall": 0.8360655737704918,
102
- "f1": 0.8031496062992125
103
  },
104
  "indicator": {
105
- "tp": 118,
106
- "fp": 57,
107
- "fn": 33,
108
- "precision": 0.6742857142857143,
109
- "recall": 0.7814569536423841,
110
- "f1": 0.7239263803680981
111
  },
112
  "other": {
113
- "tp": 129,
114
- "fp": 97,
115
- "fn": 79,
116
- "precision": 0.5707964601769911,
117
- "recall": 0.6201923076923077,
118
- "f1": 0.5944700460829494
 
 
 
 
 
 
 
 
119
  },
120
  "report": {
121
- "tp": 13,
122
- "fp": 12,
123
- "fn": 17,
124
- "precision": 0.52,
125
- "recall": 0.43333333333333335,
126
- "f1": 0.4727272727272728
127
  },
128
  "survey": {
129
- "tp": 755,
130
- "fp": 65,
131
- "fn": 69,
132
- "precision": 0.9207317073170732,
133
- "recall": 0.9162621359223301,
134
- "f1": 0.9184914841849148
135
  }
136
  },
137
  "usage_action": {
138
- "causal": {
139
- "tp": 76,
140
- "fp": 77,
141
- "fn": 64,
142
- "precision": 0.49673202614379086,
143
- "recall": 0.5428571428571428,
144
- "f1": 0.5187713310580205
 
 
 
 
 
 
 
 
145
  },
146
  "inform": {
147
- "tp": 86,
148
- "fp": 55,
149
- "fn": 66,
150
- "precision": 0.6099290780141844,
151
- "recall": 0.5657894736842105,
152
- "f1": 0.5870307167235496
153
- },
154
- "measure": {
155
- "tp": 188,
156
- "fp": 114,
157
- "fn": 208,
158
- "precision": 0.6225165562913907,
159
- "recall": 0.47474747474747475,
160
- "f1": 0.5386819484240687
161
- },
162
- "model": {
163
- "tp": 182,
164
- "fp": 166,
165
- "fn": 71,
166
- "precision": 0.5229885057471264,
167
- "recall": 0.7193675889328063,
168
- "f1": 0.605657237936772
169
- },
170
- "other": {
171
- "tp": 17,
172
- "fp": 19,
173
- "fn": 40,
174
- "precision": 0.4722222222222222,
175
- "recall": 0.2982456140350877,
176
- "f1": 0.3655913978494624
177
- },
178
- "produce": {
179
- "tp": 61,
180
- "fp": 93,
181
- "fn": 69,
182
- "precision": 0.3961038961038961,
183
- "recall": 0.46923076923076923,
184
- "f1": 0.4295774647887323
185
  },
186
  "source": {
187
- "tp": 346,
188
- "fp": 220,
189
- "fn": 211,
190
- "precision": 0.6113074204946997,
191
- "recall": 0.6211849192100538,
192
- "f1": 0.616206589492431
193
  },
194
  "validate": {
195
- "tp": 131,
196
- "fp": 72,
197
- "fn": 80,
198
- "precision": 0.645320197044335,
199
- "recall": 0.6208530805687204,
200
- "f1": 0.6328502415458936
201
- },
202
- "wrangle": {
203
- "tp": 34,
204
- "fp": 40,
205
- "fn": 47,
206
- "precision": 0.4594594594594595,
207
- "recall": 0.41975308641975306,
208
- "f1": 0.43870967741935485
209
  }
210
  },
211
  "impact_label": {
212
  "evidence-base": {
213
- "tp": 1259,
214
- "fp": 309,
215
- "fn": 172,
216
- "precision": 0.8029336734693877,
217
- "recall": 0.8798043326345213,
218
- "f1": 0.8396132044014671
219
- },
220
- "market-use": {
221
- "tp": 2,
222
- "fp": 3,
223
- "fn": 37,
224
- "precision": 0.4,
225
- "recall": 0.05128205128205128,
226
- "f1": 0.09090909090909091
227
  },
228
  "none": {
229
- "tp": 41,
230
- "fp": 12,
231
- "fn": 63,
232
- "precision": 0.7735849056603774,
233
- "recall": 0.3942307692307692,
234
- "f1": 0.5222929936305732
235
- },
236
- "operational-use": {
237
- "tp": 34,
238
- "fp": 59,
239
- "fn": 73,
240
- "precision": 0.3655913978494624,
241
- "recall": 0.3177570093457944,
242
- "f1": 0.34
243
  },
244
  "policy": {
245
- "tp": 110,
246
- "fp": 148,
247
- "fn": 186,
248
- "precision": 0.4263565891472868,
249
- "recall": 0.3716216216216216,
250
- "f1": 0.3971119133574007
 
 
 
 
 
 
 
 
251
  }
252
  }
253
  },
254
  "usage_summary": {
255
  "n": 1977,
256
- "mean_sim": 0.6563892960548401,
257
- "grounded_rate": 0.9170460293373799
258
  }
259
  },
260
  "examples": [
@@ -338,352 +306,312 @@
338
  {
339
  "mention": "COVID-19 Enterprise Survey",
340
  "gold": "{\"geography\": \"Chad\", \"year\": \"2020\"}",
341
- "pred": "{\"year\": \"2020\"}",
342
- "correct": "year",
343
- "verbatim": "year"
344
  }
345
  ]
346
  },
347
  "synthetic": {
348
- "n_eval": 1730,
349
  "overall": {
350
- "tp": 463,
351
- "fp": 95,
352
- "fn": 121,
353
- "precision": 0.8297491039426523,
354
- "recall": 0.7928082191780822,
355
- "f05": 0.8220880681818183,
356
- "f1": 0.8108581436077057
357
  },
358
  "per_key": {
359
  "producer": {
360
- "tp": 116,
361
- "fp": 19,
362
- "fn": 22,
363
- "precision": 0.8592592592592593,
364
- "recall": 0.8405797101449275,
365
- "f05": 0.8554572271386431,
366
- "f1": 0.8498168498168497
367
  },
368
  "year": {
369
- "tp": 122,
370
- "fp": 26,
371
- "fn": 29,
372
- "precision": 0.8243243243243243,
373
- "recall": 0.8079470198675497,
374
- "f05": 0.8209959623149395,
375
- "f1": 0.8160535117056857
376
  },
377
  "geography": {
378
  "tp": 163,
379
- "fp": 36,
380
- "fn": 40,
381
- "precision": 0.8190954773869347,
382
- "recall": 0.8029556650246306,
383
- "f05": 0.815815815815816,
384
- "f1": 0.8109452736318409
385
  },
386
  "acronym": {
387
- "tp": 62,
388
- "fp": 14,
389
- "fn": 30,
390
- "precision": 0.8157894736842105,
391
- "recall": 0.6739130434782609,
392
- "f05": 0.7828282828282827,
393
- "f1": 0.7380952380952381
394
  }
395
  },
396
  "verbatim": {
397
- "emitted": 558,
398
- "verbatim": 556,
399
- "rate": 0.996415770609319,
400
  "per_key": {
401
  "producer": {
402
- "emitted": 135,
403
- "verbatim": 135
404
  },
405
  "year": {
406
- "emitted": 148,
407
- "verbatim": 148
408
  },
409
  "geography": {
410
- "emitted": 199,
411
- "verbatim": 197
412
  },
413
  "acronym": {
414
- "emitted": 76,
415
- "verbatim": 76
416
  }
417
  }
418
  },
419
  "usage_impact": {
420
- "data_type_macro_f1": 0.8357854014319261,
421
- "usage_action_macro_f1": 0.6240329583409329,
422
- "impact_label_macro_f1": 0.5103286663565687,
423
  "per_label": {
424
  "data_type": {
425
  "administrative": {
426
- "tp": 174,
427
  "fp": 23,
428
- "fn": 29,
429
- "precision": 0.883248730964467,
430
- "recall": 0.8571428571428571,
431
- "f1": 0.87
432
  },
433
  "database": {
434
- "tp": 195,
435
- "fp": 28,
436
- "fn": 47,
437
- "precision": 0.874439461883408,
438
- "recall": 0.8057851239669421,
439
- "f1": 0.8387096774193548
440
  },
441
  "geospatial": {
442
- "tp": 99,
443
  "fp": 22,
444
- "fn": 9,
445
- "precision": 0.8181818181818182,
446
- "recall": 0.9166666666666666,
447
- "f1": 0.8646288209606987
448
  },
449
  "indicator": {
450
- "tp": 117,
451
- "fp": 21,
452
- "fn": 22,
453
- "precision": 0.8478260869565217,
454
- "recall": 0.841726618705036,
455
- "f1": 0.8447653429602888
456
  },
457
  "other": {
458
- "tp": 96,
459
- "fp": 44,
460
- "fn": 39,
461
- "precision": 0.6857142857142857,
462
- "recall": 0.7111111111111111,
463
- "f1": 0.6981818181818182
464
  },
465
  "report": {
466
- "tp": 31,
467
- "fp": 7,
468
- "fn": 10,
469
- "precision": 0.8157894736842105,
470
- "recall": 0.7560975609756098,
471
- "f1": 0.7848101265822786
472
  },
473
  "survey": {
474
- "tp": 516,
475
- "fp": 33,
476
- "fn": 22,
477
- "precision": 0.9398907103825137,
478
- "recall": 0.9591078066914498,
479
- "f1": 0.9494020239190434
480
  }
481
  },
482
  "usage_action": {
483
- "causal": {
484
- "tp": 74,
485
- "fp": 34,
486
- "fn": 34,
487
- "precision": 0.6851851851851852,
488
- "recall": 0.6851851851851852,
489
- "f1": 0.6851851851851852
490
- },
491
- "inform": {
492
- "tp": 58,
493
- "fp": 39,
494
- "fn": 39,
495
- "precision": 0.5979381443298969,
496
- "recall": 0.5979381443298969,
497
- "f1": 0.5979381443298969
498
- },
499
- "measure": {
500
- "tp": 171,
501
- "fp": 82,
502
- "fn": 99,
503
- "precision": 0.6758893280632411,
504
- "recall": 0.6333333333333333,
505
- "f1": 0.6539196940726577
506
- },
507
- "model": {
508
- "tp": 108,
509
- "fp": 69,
510
- "fn": 50,
511
- "precision": 0.6101694915254238,
512
- "recall": 0.6835443037974683,
513
- "f1": 0.6447761194029851
514
  },
515
- "other": {
516
- "tp": 26,
517
- "fp": 18,
518
- "fn": 35,
519
- "precision": 0.5909090909090909,
520
- "recall": 0.4262295081967213,
521
- "f1": 0.49523809523809526
522
  },
523
- "produce": {
524
- "tp": 81,
525
- "fp": 75,
526
- "fn": 37,
527
- "precision": 0.5192307692307693,
528
- "recall": 0.6864406779661016,
529
- "f1": 0.5912408759124088
530
  },
531
  "source": {
532
- "tp": 271,
533
- "fp": 108,
534
- "fn": 124,
535
- "precision": 0.7150395778364116,
536
- "recall": 0.6860759493670886,
537
- "f1": 0.7002583979328165
538
  },
539
  "validate": {
540
- "tp": 98,
541
- "fp": 50,
542
- "fn": 49,
543
- "precision": 0.6621621621621622,
544
- "recall": 0.6666666666666666,
545
- "f1": 0.6644067796610169
546
- },
547
- "wrangle": {
548
- "tp": 28,
549
- "fp": 16,
550
- "fn": 24,
551
- "precision": 0.6363636363636364,
552
- "recall": 0.5384615384615384,
553
- "f1": 0.5833333333333334
554
  }
555
  },
556
  "impact_label": {
557
  "evidence-base": {
558
- "tp": 948,
559
- "fp": 187,
560
  "fn": 80,
561
- "precision": 0.8352422907488987,
562
- "recall": 0.9221789883268483,
563
- "f1": 0.8765603328710125
564
- },
565
- "market-use": {
566
- "tp": 1,
567
- "fp": 3,
568
- "fn": 7,
569
- "precision": 0.25,
570
- "recall": 0.125,
571
- "f1": 0.16666666666666666
572
  },
573
  "none": {
574
- "tp": 42,
575
- "fp": 15,
576
- "fn": 54,
577
- "precision": 0.7368421052631579,
578
- "recall": 0.4375,
579
- "f1": 0.5490196078431372
580
- },
581
- "operational-use": {
582
- "tp": 24,
583
- "fp": 25,
584
- "fn": 42,
585
- "precision": 0.4897959183673469,
586
- "recall": 0.36363636363636365,
587
- "f1": 0.41739130434782606
588
  },
589
  "policy": {
590
- "tp": 100,
591
- "fp": 61,
592
- "fn": 108,
593
- "precision": 0.6211180124223602,
594
- "recall": 0.4807692307692308,
595
- "f1": 0.5420054200542006
 
 
 
 
 
 
 
 
596
  }
597
  }
598
  },
599
  "usage_summary": {
600
- "n": 1406,
601
- "mean_sim": 0.5191029906272888,
602
- "grounded_rate": 0.5597439544807966
603
  }
604
  },
605
  "examples": [
606
  {
607
- "mention": "ICRISAT surveys",
608
- "gold": "{\"producer\": \"International Crops Institute for Semi Arid Tropics\"}",
609
- "pred": "{\"producer\": \"International Crops Institute for Semi Arid Tropics\"}",
610
- "correct": "producer",
611
- "verbatim": "producer"
612
- },
613
- {
614
- "mention": "Southeast Asia Digital ( SEAD ) data set",
615
- "gold": "{\"acronym\": \"SEAD\", \"geography\": \"four Southeast Asian countries\"}",
616
- "pred": "{\"acronym\": \"SEAD\", \"geography\": \"Southeast Asia\"}",
617
- "correct": "acronym",
618
- "verbatim": "geography,acronym"
619
- },
620
- {
621
- "mention": "Household Expenditure Survey",
622
- "gold": "{\"acronym\": \"HES\", \"geography\": \"Bangladesh\", \"producer\": \"Bangladesh Bureau of Statistics\", \"year\": \"1991 / 92\"}",
623
- "pred": "{\"acronym\": \"HES\", \"geography\": \"Bangladesh\", \"producer\": \"Bangladesh Bureau of Statistics\", \"year\": \"1991\"}",
624
- "correct": "producer,geography,acronym",
625
- "verbatim": "producer,year,geography,acronym"
626
  },
627
  {
628
- "mention": "2018 / 19 NLSS",
629
- "gold": "{\"acronym\": \"NLSS\", \"geography\": \"Nigeria\", \"year\": \"2018 / 19\"}",
630
- "pred": "{\"acronym\": \"NLSS\", \"geography\": \"Nigeria\", \"year\": \"2018\"}",
631
- "correct": "geography,acronym",
632
- "verbatim": "year,geography,acronym"
633
  },
634
  {
635
- "mention": "IE-LFS 2019",
636
- "gold": "{\"producer\": \"World Bank\", \"year\": \"2019\"}",
637
- "pred": "{\"year\": \"2019\"}",
638
- "correct": "year",
639
- "verbatim": "year"
640
  },
641
  {
642
- "mention": "World Bank ' s main database",
643
  "gold": "{\"producer\": \"World Bank\"}",
644
  "pred": "{\"producer\": \"World Bank\"}",
645
  "correct": "producer",
646
  "verbatim": "producer"
647
  },
648
  {
649
- "mention": "LSMSISA surveys",
650
- "gold": "{\"geography\": \"Nigeria and Tanzania\"}",
651
- "pred": "{\"acronym\": \"LSMSISA\"}",
652
  "correct": "-",
653
- "verbatim": "acronym"
654
  },
655
  {
656
- "mention": "survey data",
657
- "gold": "{\"producer\": \"Center for Global Development ( CGD )\", \"year\": \"2022\"}",
658
- "pred": "{\"year\": \"2022\"}",
659
- "correct": "year",
660
- "verbatim": "year"
661
  },
662
  {
663
- "mention": "administrative data",
664
- "gold": "{\"year\": \"2015\"}",
665
- "pred": "{\"year\": \"2015 / 2016\"}",
666
- "correct": "-",
667
- "verbatim": "year"
668
  },
669
  {
670
- "mention": "Domah 2001 survey",
671
- "gold": "{\"year\": \"2001\"}",
672
- "pred": "{\"year\": \"2001\"}",
673
- "correct": "year",
674
- "verbatim": "year"
675
  },
676
  {
677
- "mention": "historical data in Malawi",
678
- "gold": "{\"geography\": \"Malawi\"}",
679
- "pred": "{\"geography\": \"Malawi\"}",
680
  "correct": "geography",
681
  "verbatim": "geography"
682
  },
683
  {
684
- "mention": "Access to Finance Survey",
685
- "gold": "{\"acronym\": \"Access to Finance Survey\", \"geography\": \"Nigeria\", \"year\": \"2008\"}",
686
- "pred": "{\"geography\": \"Nigeria\", \"year\": \"2008\"}",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
687
  "correct": "year,geography",
688
  "verbatim": "year,geography"
689
  }
 
2
  "real": {
3
  "n_eval": 5464,
4
  "overall": {
5
+ "tp": 4973,
6
+ "fp": 1296,
7
+ "fn": 1269,
8
+ "precision": 0.7932684638698357,
9
+ "recall": 0.7966997757129125,
10
+ "f05": 0.7939523596653683,
11
+ "f1": 0.7949804172328352
12
  },
13
  "per_key": {
14
  "producer": {
15
+ "tp": 1081,
16
+ "fp": 374,
17
+ "fn": 368,
18
+ "precision": 0.7429553264604811,
19
+ "recall": 0.746031746031746,
20
+ "f05": 0.7435685788966846,
21
+ "f1": 0.7444903581267217
22
  },
23
  "year": {
24
+ "tp": 1002,
25
+ "fp": 230,
26
+ "fn": 348,
27
+ "precision": 0.8133116883116883,
28
+ "recall": 0.7422222222222222,
29
+ "f05": 0.798024848677923,
30
+ "f1": 0.7761425251742835
31
  },
32
  "geography": {
33
+ "tp": 1620,
34
+ "fp": 446,
35
+ "fn": 407,
36
+ "precision": 0.7841239109390126,
37
+ "recall": 0.7992106561420819,
38
+ "f05": 0.7870955203575939,
39
+ "f1": 0.7915954067920842
40
  },
41
  "acronym": {
42
+ "tp": 1270,
43
+ "fp": 246,
44
+ "fn": 146,
45
+ "precision": 0.837730870712401,
46
+ "recall": 0.8968926553672316,
47
+ "f05": 0.8489304812834224,
48
+ "f1": 0.8663028649386084
49
  }
50
  },
51
  "verbatim": {
52
+ "emitted": 6269,
53
+ "verbatim": 6249,
54
+ "rate": 0.9968096985165098,
55
  "per_key": {
56
  "producer": {
57
+ "emitted": 1455,
58
+ "verbatim": 1450
59
  },
60
  "year": {
61
+ "emitted": 1232,
62
+ "verbatim": 1232
63
  },
64
  "geography": {
65
+ "emitted": 2066,
66
+ "verbatim": 2053
67
  },
68
  "acronym": {
69
+ "emitted": 1516,
70
+ "verbatim": 1514
71
  }
72
  }
73
  },
74
  "usage_impact": {
75
+ "data_type_macro_f1": 0.6005128473597353,
76
+ "usage_action_macro_f1": 0.5623718118758655,
77
+ "impact_label_macro_f1": 0.49325217399161364,
78
  "per_label": {
79
  "data_type": {
80
  "administrative": {
81
+ "tp": 205,
82
+ "fp": 79,
83
+ "fn": 68,
84
+ "precision": 0.721830985915493,
85
+ "recall": 0.7509157509157509,
86
+ "f1": 0.7360861759425493
87
  },
88
  "database": {
89
+ "tp": 240,
90
+ "fp": 67,
91
+ "fn": 129,
92
+ "precision": 0.7817589576547231,
93
+ "recall": 0.6504065040650406,
94
+ "f1": 0.7100591715976332
95
  },
96
  "geospatial": {
97
+ "tp": 101,
98
+ "fp": 38,
99
+ "fn": 21,
100
+ "precision": 0.7266187050359713,
101
+ "recall": 0.8278688524590164,
102
+ "f1": 0.7739463601532568
103
  },
104
  "indicator": {
105
+ "tp": 114,
106
+ "fp": 46,
107
+ "fn": 37,
108
+ "precision": 0.7125,
109
+ "recall": 0.7549668874172185,
110
+ "f1": 0.7331189710610932
111
  },
112
  "other": {
113
+ "tp": 109,
114
+ "fp": 84,
115
+ "fn": 99,
116
+ "precision": 0.5647668393782384,
117
+ "recall": 0.5240384615384616,
118
+ "f1": 0.5436408977556111
119
+ },
120
+ "program": {
121
+ "tp": 0,
122
+ "fp": 1,
123
+ "fn": 0,
124
+ "precision": 0.0,
125
+ "recall": 0.0,
126
+ "f1": 0.0
127
  },
128
  "report": {
129
+ "tp": 10,
130
+ "fp": 11,
131
+ "fn": 20,
132
+ "precision": 0.47619047619047616,
133
+ "recall": 0.3333333333333333,
134
+ "f1": 0.39215686274509803
135
  },
136
  "survey": {
137
+ "tp": 776,
138
+ "fp": 96,
139
+ "fn": 48,
140
+ "precision": 0.8899082568807339,
141
+ "recall": 0.941747572815534,
142
+ "f1": 0.9150943396226414
143
  }
144
  },
145
  "usage_action": {
146
+ "analyze": {
147
+ "tp": 553,
148
+ "fp": 299,
149
+ "fn": 236,
150
+ "precision": 0.6490610328638498,
151
+ "recall": 0.7008871989860583,
152
+ "f1": 0.6739792809262645
153
+ },
154
+ "curate": {
155
+ "tp": 80,
156
+ "fp": 59,
157
+ "fn": 131,
158
+ "precision": 0.5755395683453237,
159
+ "recall": 0.3791469194312796,
160
+ "f1": 0.4571428571428572
161
  },
162
  "inform": {
163
+ "tp": 98,
164
+ "fp": 72,
165
+ "fn": 111,
166
+ "precision": 0.5764705882352941,
167
+ "recall": 0.4688995215311005,
168
+ "f1": 0.5171503957783641
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
169
  },
170
  "source": {
171
+ "tp": 366,
172
+ "fp": 273,
173
+ "fn": 191,
174
+ "precision": 0.5727699530516432,
175
+ "recall": 0.6570915619389587,
176
+ "f1": 0.6120401337792644
177
  },
178
  "validate": {
179
+ "tp": 107,
180
+ "fp": 70,
181
+ "fn": 104,
182
+ "precision": 0.6045197740112994,
183
+ "recall": 0.5071090047393365,
184
+ "f1": 0.5515463917525774
 
 
 
 
 
 
 
 
185
  }
186
  },
187
  "impact_label": {
188
  "evidence-base": {
189
+ "tp": 1134,
190
+ "fp": 261,
191
+ "fn": 297,
192
+ "precision": 0.8129032258064516,
193
+ "recall": 0.7924528301886793,
194
+ "f1": 0.802547770700637
 
 
 
 
 
 
 
 
195
  },
196
  "none": {
197
+ "tp": 51,
198
+ "fp": 47,
199
+ "fn": 53,
200
+ "precision": 0.5204081632653061,
201
+ "recall": 0.49038461538461536,
202
+ "f1": 0.504950495049505
 
 
 
 
 
 
 
 
203
  },
204
  "policy": {
205
+ "tp": 114,
206
+ "fp": 219,
207
+ "fn": 182,
208
+ "precision": 0.34234234234234234,
209
+ "recall": 0.38513513513513514,
210
+ "f1": 0.3624801271860096
211
+ },
212
+ "uptake": {
213
+ "tp": 45,
214
+ "fp": 106,
215
+ "fn": 101,
216
+ "precision": 0.2980132450331126,
217
+ "recall": 0.3082191780821918,
218
+ "f1": 0.30303030303030304
219
  }
220
  }
221
  },
222
  "usage_summary": {
223
  "n": 1977,
224
+ "mean_sim": 0.6581464409828186,
225
+ "grounded_rate": 0.9165402124430956
226
  }
227
  },
228
  "examples": [
 
306
  {
307
  "mention": "COVID-19 Enterprise Survey",
308
  "gold": "{\"geography\": \"Chad\", \"year\": \"2020\"}",
309
+ "pred": "{\"acronym\": \"ES\", \"geography\": \"Chad\", \"producer\": \"World Bank\", \"year\": \"2020\"}",
310
+ "correct": "year,geography",
311
+ "verbatim": "producer,year,geography,acronym"
312
  }
313
  ]
314
  },
315
  "synthetic": {
316
+ "n_eval": 1564,
317
  "overall": {
318
+ "tp": 490,
319
+ "fp": 102,
320
+ "fn": 96,
321
+ "precision": 0.8277027027027027,
322
+ "recall": 0.8361774744027304,
323
+ "f05": 0.8293838862559243,
324
+ "f1": 0.831918505942275
325
  },
326
  "per_key": {
327
  "producer": {
328
+ "tp": 126,
329
+ "fp": 23,
330
+ "fn": 26,
331
+ "precision": 0.8456375838926175,
332
+ "recall": 0.8289473684210527,
333
+ "f05": 0.8422459893048128,
334
+ "f1": 0.8372093023255816
335
  },
336
  "year": {
337
+ "tp": 123,
338
+ "fp": 24,
339
+ "fn": 21,
340
+ "precision": 0.8367346938775511,
341
+ "recall": 0.8541666666666666,
342
+ "f05": 0.8401639344262294,
343
+ "f1": 0.845360824742268
344
  },
345
  "geography": {
346
  "tp": 163,
347
+ "fp": 38,
348
+ "fn": 36,
349
+ "precision": 0.8109452736318408,
350
+ "recall": 0.8190954773869347,
351
+ "f05": 0.8125623130608176,
352
+ "f1": 0.815
353
  },
354
  "acronym": {
355
+ "tp": 78,
356
+ "fp": 17,
357
+ "fn": 13,
358
+ "precision": 0.8210526315789474,
359
+ "recall": 0.8571428571428571,
360
+ "f05": 0.8280254777070063,
361
+ "f1": 0.8387096774193549
362
  }
363
  },
364
  "verbatim": {
365
+ "emitted": 592,
366
+ "verbatim": 592,
367
+ "rate": 1.0,
368
  "per_key": {
369
  "producer": {
370
+ "emitted": 149,
371
+ "verbatim": 149
372
  },
373
  "year": {
374
+ "emitted": 147,
375
+ "verbatim": 147
376
  },
377
  "geography": {
378
+ "emitted": 201,
379
+ "verbatim": 201
380
  },
381
  "acronym": {
382
+ "emitted": 95,
383
+ "verbatim": 95
384
  }
385
  }
386
  },
387
  "usage_impact": {
388
+ "data_type_macro_f1": 0.8559476982147013,
389
+ "usage_action_macro_f1": 0.7706078022508355,
390
+ "impact_label_macro_f1": 0.798462757547076,
391
  "per_label": {
392
  "data_type": {
393
  "administrative": {
394
+ "tp": 188,
395
  "fp": 23,
396
+ "fn": 20,
397
+ "precision": 0.8909952606635071,
398
+ "recall": 0.9038461538461539,
399
+ "f1": 0.8973747016706445
400
  },
401
  "database": {
402
+ "tp": 169,
403
+ "fp": 19,
404
+ "fn": 37,
405
+ "precision": 0.898936170212766,
406
+ "recall": 0.8203883495145631,
407
+ "f1": 0.8578680203045685
408
  },
409
  "geospatial": {
410
+ "tp": 111,
411
  "fp": 22,
412
+ "fn": 6,
413
+ "precision": 0.8345864661654135,
414
+ "recall": 0.9487179487179487,
415
+ "f1": 0.888
416
  },
417
  "indicator": {
418
+ "tp": 78,
419
+ "fp": 13,
420
+ "fn": 25,
421
+ "precision": 0.8571428571428571,
422
+ "recall": 0.7572815533980582,
423
+ "f1": 0.8041237113402061
424
  },
425
  "other": {
426
+ "tp": 97,
427
+ "fp": 25,
428
+ "fn": 29,
429
+ "precision": 0.7950819672131147,
430
+ "recall": 0.7698412698412699,
431
+ "f1": 0.782258064516129
432
  },
433
  "report": {
434
+ "tp": 51,
435
+ "fp": 5,
436
+ "fn": 18,
437
+ "precision": 0.9107142857142857,
438
+ "recall": 0.7391304347826086,
439
+ "f1": 0.8159999999999998
440
  },
441
  "survey": {
442
+ "tp": 403,
443
+ "fp": 37,
444
+ "fn": 9,
445
+ "precision": 0.9159090909090909,
446
+ "recall": 0.9781553398058253,
447
+ "f1": 0.9460093896713615
448
  }
449
  },
450
  "usage_action": {
451
+ "analyze": {
452
+ "tp": 302,
453
+ "fp": 99,
454
+ "fn": 79,
455
+ "precision": 0.7531172069825436,
456
+ "recall": 0.7926509186351706,
457
+ "f1": 0.7723785166240409
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
458
  },
459
+ "curate": {
460
+ "tp": 104,
461
+ "fp": 21,
462
+ "fn": 40,
463
+ "precision": 0.832,
464
+ "recall": 0.7222222222222222,
465
+ "f1": 0.7732342007434944
466
  },
467
+ "inform": {
468
+ "tp": 206,
469
+ "fp": 40,
470
+ "fn": 65,
471
+ "precision": 0.8373983739837398,
472
+ "recall": 0.7601476014760148,
473
+ "f1": 0.7969052224371374
474
  },
475
  "source": {
476
+ "tp": 278,
477
+ "fp": 101,
478
+ "fn": 69,
479
+ "precision": 0.7335092348284961,
480
+ "recall": 0.8011527377521613,
481
+ "f1": 0.7658402203856749
482
  },
483
  "validate": {
484
+ "tp": 70,
485
+ "fp": 20,
486
+ "fn": 28,
487
+ "precision": 0.7777777777777778,
488
+ "recall": 0.7142857142857143,
489
+ "f1": 0.7446808510638298
 
 
 
 
 
 
 
 
490
  }
491
  },
492
  "impact_label": {
493
  "evidence-base": {
494
+ "tp": 304,
495
+ "fp": 140,
496
  "fn": 80,
497
+ "precision": 0.6846846846846847,
498
+ "recall": 0.7916666666666666,
499
+ "f1": 0.7342995169082125
 
 
 
 
 
 
 
 
500
  },
501
  "none": {
502
+ "tp": 244,
503
+ "fp": 21,
504
+ "fn": 52,
505
+ "precision": 0.9207547169811321,
506
+ "recall": 0.8243243243243243,
507
+ "f1": 0.8698752228163993
 
 
 
 
 
 
 
 
508
  },
509
  "policy": {
510
+ "tp": 178,
511
+ "fp": 66,
512
+ "fn": 86,
513
+ "precision": 0.7295081967213115,
514
+ "recall": 0.6742424242424242,
515
+ "f1": 0.7007874015748032
516
+ },
517
+ "uptake": {
518
+ "tp": 260,
519
+ "fp": 28,
520
+ "fn": 37,
521
+ "precision": 0.9027777777777778,
522
+ "recall": 0.8754208754208754,
523
+ "f1": 0.888888888888889
524
  }
525
  }
526
  },
527
  "usage_summary": {
528
+ "n": 1241,
529
+ "mean_sim": 0.5276422500610352,
530
+ "grounded_rate": 0.5898468976631749
531
  }
532
  },
533
  "examples": [
534
  {
535
+ "mention": "Gallup World Poll data",
536
+ "gold": "{\"geography\": \"Iraq\", \"producer\": \"Gallup World Poll\", \"year\": \"2010\"}",
537
+ "pred": "{\"geography\": \"Iraq\", \"producer\": \"Gallup World Poll\"}",
538
+ "correct": "producer,geography",
539
+ "verbatim": "producer,geography"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
540
  },
541
  {
542
+ "mention": "Philippines data",
543
+ "gold": "{\"geography\": \"Philippines\"}",
544
+ "pred": "{\"geography\": \"Philippines\"}",
545
+ "correct": "geography",
546
+ "verbatim": "geography"
547
  },
548
  {
549
+ "mention": "World Bank data",
550
+ "gold": "{\"producer\": \"World Bank\"}",
551
+ "pred": "{\"producer\": \"World Bank\"}",
552
+ "correct": "producer",
553
+ "verbatim": "producer"
554
  },
555
  {
556
+ "mention": "COVID-19 Business Pulse Surveys",
557
  "gold": "{\"producer\": \"World Bank\"}",
558
  "pred": "{\"producer\": \"World Bank\"}",
559
  "correct": "producer",
560
  "verbatim": "producer"
561
  },
562
  {
563
+ "mention": "FAOSTAT",
564
+ "gold": "{\"acronym\": \"FAOSTAT\", \"producer\": \"Food and Agriculture Organization of the United Nations\"}",
565
+ "pred": "{\"producer\": \"FAOSTAT\"}",
566
  "correct": "-",
567
+ "verbatim": "producer"
568
  },
569
  {
570
+ "mention": "WBES",
571
+ "gold": "{\"acronym\": \"WBES\", \"producer\": \"World Bank\"}",
572
+ "pred": "{\"acronym\": \"WBES\", \"producer\": \"World Bank\"}",
573
+ "correct": "producer,acronym",
574
+ "verbatim": "producer,acronym"
575
  },
576
  {
577
+ "mention": "International Income Distribution Database",
578
+ "gold": "{\"acronym\": \"I2D2\"}",
579
+ "pred": "{\"acronym\": \"I2D2\", \"geography\": \"Western Balkans\"}",
580
+ "correct": "acronym",
581
+ "verbatim": "geography,acronym"
582
  },
583
  {
584
+ "mention": "GEOSTAT Statistics Survey of Enterprises",
585
+ "gold": "{\"geography\": \"Georgia\", \"year\": \"2006\"}",
586
+ "pred": "{\"geography\": \"Georgia\", \"producer\": \"GEOSTAT\", \"year\": \"2006-2017\"}",
587
+ "correct": "geography",
588
+ "verbatim": "producer,year,geography"
589
  },
590
  {
591
+ "mention": "household surveys",
592
+ "gold": "{\"geography\": \"Brazil, Costa Rica, and Ecuador\"}",
593
+ "pred": "{\"geography\": \"Brazil, Costa Rica, and Ecuador\"}",
594
  "correct": "geography",
595
  "verbatim": "geography"
596
  },
597
  {
598
+ "mention": "geolocated household data",
599
+ "gold": "{\"geography\": \"Afghanistan\", \"producer\": \"World Bank\"}",
600
+ "pred": "{\"geography\": \"Afghanistan\", \"producer\": \"World Bank\"}",
601
+ "correct": "producer,geography",
602
+ "verbatim": "producer,geography"
603
+ },
604
+ {
605
+ "mention": "Mexican National Employment Survey ( ENE )",
606
+ "gold": "{\"acronym\": \"ENE\", \"producer\": \"Mexican government\", \"year\": \"2000\"}",
607
+ "pred": "{\"acronym\": \"ENE\", \"geography\": \"Mexican\", \"producer\": \"Mexican government\", \"year\": \"2000\"}",
608
+ "correct": "producer,year,acronym",
609
+ "verbatim": "producer,year,geography,acronym"
610
+ },
611
+ {
612
+ "mention": "2011 Census",
613
+ "gold": "{\"geography\": \"Armenia\", \"producer\": \"Committee of the Republic of Armenia ( ArmStat )\", \"year\": \"2011\"}",
614
+ "pred": "{\"geography\": \"Armenia\", \"year\": \"2011\"}",
615
  "correct": "year,geography",
616
  "verbatim": "year,geography"
617
  }