Zap / eval /eval_test_float32.json
ProCreations's picture
Zap v1: 20.7M-param web form & field classifier for autofill (bf16, MIT)
e4805d0 verified
Raw
History Blame Contribute Delete
22.3 kB
{
"split": "test",
"dtype": "float32",
"forms_per_s_gpu": 3416.2872663427547,
"model": {
"n_forms": 6922,
"n_fields": 20130,
"field_acc": 0.9719324391455539,
"field_macro_f1": 0.9597248737921183,
"form_acc": 0.9569488587113552,
"form_macro_f1": 0.932616790369714,
"password_precision": 0.9942263279445728,
"password_recall": 0.9976825028968713,
"password_f1": 0.9959514170040485,
"cur_vs_new_acc": 0.9915025106218617,
"identifier_f1": 0.9939004103360319,
"per_field": {
"username": {
"p": 0.9782101167315175,
"r": 0.9736638264910922,
"f1": 0.9759316770186335,
"n": 1291
},
"email": {
"p": 0.9870049504950495,
"r": 0.9940791523839202,
"f1": 0.9905294208973761,
"n": 3209
},
"current-password": {
"p": 0.9866238630283574,
"r": 0.9956803455723542,
"f1": 0.9911314162859447,
"n": 1852
},
"new-password": {
"p": 0.9917695473251029,
"r": 0.9810040705563093,
"f1": 0.9863574351978172,
"n": 737
},
"one-time-code": {
"p": 0.9318181818181818,
"r": 0.9425287356321839,
"f1": 0.9371428571428572,
"n": 174
},
"tel": {
"p": 0.9809802487198245,
"r": 0.9933333333333333,
"f1": 0.9871181450128819,
"n": 1350
},
"name": {
"p": 0.9501329787234043,
"r": 0.9740967961826857,
"f1": 0.9619656681252104,
"n": 1467
},
"given-name": {
"p": 0.9737302977232924,
"r": 0.9129720853858785,
"f1": 0.9423728813559322,
"n": 609
},
"family-name": {
"p": 0.9881422924901185,
"r": 0.9765625,
"f1": 0.9823182711198427,
"n": 512
},
"organization": {
"p": 0.953307392996109,
"r": 0.9423076923076923,
"f1": 0.9477756286266925,
"n": 260
},
"street-address": {
"p": 0.9396551724137931,
"r": 0.9775784753363229,
"f1": 0.9582417582417583,
"n": 223
},
"address-line2": {
"p": 0.975609756097561,
"r": 0.7547169811320755,
"f1": 0.851063829787234,
"n": 53
},
"address-level2": {
"p": 0.9149659863945578,
"r": 0.8848684210526315,
"f1": 0.8996655518394647,
"n": 304
},
"address-level1": {
"p": 0.9005524861878453,
"r": 0.8858695652173914,
"f1": 0.8931506849315068,
"n": 184
},
"postal-code": {
"p": 0.9731343283582089,
"r": 0.9673590504451038,
"f1": 0.9702380952380952,
"n": 337
},
"country": {
"p": 0.8743169398907104,
"r": 0.9580838323353293,
"f1": 0.9142857142857143,
"n": 167
},
"cc-name": {
"p": 0.9583333333333334,
"r": 1.0,
"f1": 0.9787234042553191,
"n": 23
},
"cc-number": {
"p": 0.9696969696969697,
"r": 1.0,
"f1": 0.9846153846153847,
"n": 32
},
"cc-exp": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 18
},
"cc-exp-month": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 14
},
"cc-exp-year": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 14
},
"cc-csc": {
"p": 1.0,
"r": 0.967741935483871,
"f1": 0.9836065573770492,
"n": 31
},
"bday": {
"p": 0.96,
"r": 0.9056603773584906,
"f1": 0.9320388349514563,
"n": 53
},
"search": {
"p": 0.9670212765957447,
"r": 0.9588607594936709,
"f1": 0.9629237288135594,
"n": 948
},
"captcha": {
"p": 0.9632107023411371,
"r": 0.9442622950819672,
"f1": 0.9536423841059603,
"n": 305
},
"other": {
"p": 0.9693911873528422,
"r": 0.9666275364749287,
"f1": 0.9680073893693845,
"n": 5963
}
},
"per_form": {
"login": {
"p": 0.9886914378029079,
"r": 0.9956616052060737,
"f1": 0.9921642799243447,
"n": 1844
},
"signup": {
"p": 0.9777227722772277,
"r": 0.947242206235012,
"f1": 0.9622411693057246,
"n": 417
},
"password-change": {
"p": 0.9154929577464789,
"r": 0.9848484848484849,
"f1": 0.948905109489051,
"n": 66
},
"password-recovery": {
"p": 0.9555555555555556,
"r": 0.9347826086956522,
"f1": 0.945054945054945,
"n": 46
},
"otp": {
"p": 0.9464285714285714,
"r": 0.9137931034482759,
"f1": 0.9298245614035087,
"n": 58
},
"payment": {
"p": 0.9393939393939394,
"r": 0.8857142857142857,
"f1": 0.9117647058823529,
"n": 35
},
"address": {
"p": 0.8048780487804879,
"r": 0.7857142857142857,
"f1": 0.7951807228915663,
"n": 42
},
"newsletter": {
"p": 0.9725609756097561,
"r": 0.9565217391304348,
"f1": 0.9644746787603932,
"n": 667
},
"contact": {
"p": 0.9453262786596119,
"r": 0.9804878048780488,
"f1": 0.9625860520802155,
"n": 1640
},
"search": {
"p": 0.967487684729064,
"r": 0.9415148609779482,
"f1": 0.9543245869776482,
"n": 1043
},
"other": {
"p": 0.9012464046021093,
"r": 0.8834586466165414,
"f1": 0.8922638822971048,
"n": 1064
}
},
"top_confusions": [
[
"given-name",
"name",
48
],
[
"search",
"other",
39
],
[
"address-level2",
"other",
28
],
[
"username",
"email",
26
],
[
"other",
"search",
25
],
[
"other",
"address-level2",
22
],
[
"other",
"country",
19
],
[
"address-level1",
"other",
18
],
[
"name",
"other",
15
],
[
"other",
"address-level1",
15
],
[
"other",
"tel",
15
],
[
"new-password",
"current-password",
14
],
[
"captcha",
"other",
14
],
[
"other",
"username",
13
],
[
"other",
"name",
12
],
[
"other",
"email",
11
],
[
"name",
"given-name",
11
],
[
"address-line2",
"other",
11
],
[
"other",
"street-address",
11
],
[
"other",
"current-password",
10
],
[
"other",
"captcha",
9
],
[
"other",
"postal-code",
9
],
[
"email",
"username",
8
],
[
"email",
"other",
8
],
[
"organization",
"other",
8
]
]
},
"model_real": {
"n_forms": 6838,
"n_fields": 19811,
"field_acc": 0.971833829690576,
"field_macro_f1": 0.9447407167557139,
"form_acc": 0.9568587306229892,
"form_macro_f1": 0.9047413119700765,
"password_precision": 0.9941383352872216,
"password_recall": 0.9976470588235294,
"password_f1": 0.9958896065766295,
"cur_vs_new_acc": 0.991764705882353,
"identifier_f1": 0.993869133875822,
"per_field": {
"username": {
"p": 0.9780907668231612,
"r": 0.9735202492211839,
"f1": 0.975800156128025,
"n": 1284
},
"email": {
"p": 0.9869402985074627,
"r": 0.9940494832445975,
"f1": 0.9904821344983618,
"n": 3193
},
"current-password": {
"p": 0.9870550161812298,
"r": 0.9956474428726877,
"f1": 0.991332611050921,
"n": 1838
},
"new-password": {
"p": 0.9914893617021276,
"r": 0.9817415730337079,
"f1": 0.9865913902611151,
"n": 712
},
"one-time-code": {
"p": 0.9328358208955224,
"r": 0.9259259259259259,
"f1": 0.929368029739777,
"n": 135
},
"tel": {
"p": 0.9822485207100592,
"r": 0.9932685115931189,
"f1": 0.9877277798438081,
"n": 1337
},
"name": {
"p": 0.9499332443257676,
"r": 0.973990417522245,
"f1": 0.9618114227779655,
"n": 1461
},
"given-name": {
"p": 0.9734513274336283,
"r": 0.912106135986733,
"f1": 0.9417808219178082,
"n": 603
},
"family-name": {
"p": 0.988,
"r": 0.9762845849802372,
"f1": 0.9821073558648111,
"n": 506
},
"organization": {
"p": 0.9525691699604744,
"r": 0.94140625,
"f1": 0.9469548133595285,
"n": 256
},
"street-address": {
"p": 0.9357798165137615,
"r": 0.9807692307692307,
"f1": 0.9577464788732395,
"n": 208
},
"address-line2": {
"p": 0.972972972972973,
"r": 0.75,
"f1": 0.8470588235294119,
"n": 48
},
"address-level2": {
"p": 0.911660777385159,
"r": 0.8835616438356164,
"f1": 0.8973913043478261,
"n": 292
},
"address-level1": {
"p": 0.8959537572254336,
"r": 0.8857142857142857,
"f1": 0.8908045977011495,
"n": 175
},
"postal-code": {
"p": 0.9719626168224299,
"r": 0.968944099378882,
"f1": 0.9704510108864696,
"n": 322
},
"country": {
"p": 0.8685714285714285,
"r": 0.9620253164556962,
"f1": 0.9129129129129129,
"n": 158
},
"cc-name": {
"p": 0.6666666666666666,
"r": 1.0,
"f1": 0.8,
"n": 2
},
"cc-number": {
"p": 0.8571428571428571,
"r": 1.0,
"f1": 0.923076923076923,
"n": 6
},
"cc-exp-month": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 6
},
"cc-exp-year": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 6
},
"cc-csc": {
"p": 1.0,
"r": 0.8333333333333334,
"f1": 0.9090909090909091,
"n": 6
},
"bday": {
"p": 0.96,
"r": 0.9056603773584906,
"f1": 0.9320388349514563,
"n": 53
},
"search": {
"p": 0.9670212765957447,
"r": 0.9588607594936709,
"f1": 0.9629237288135594,
"n": 948
},
"captcha": {
"p": 0.9627118644067797,
"r": 0.9435215946843853,
"f1": 0.953020134228188,
"n": 301
},
"other": {
"p": 0.9695132221660772,
"r": 0.9665827036104114,
"f1": 0.9680457450386813,
"n": 5955
}
},
"per_form": {
"login": {
"p": 0.9891774891774892,
"r": 0.9956427015250545,
"f1": 0.992399565689468,
"n": 1836
},
"signup": {
"p": 0.9777227722772277,
"r": 0.947242206235012,
"f1": 0.9622411693057246,
"n": 417
},
"password-change": {
"p": 0.8928571428571429,
"r": 1.0,
"f1": 0.9433962264150945,
"n": 50
},
"password-recovery": {
"p": 0.9459459459459459,
"r": 0.9210526315789473,
"f1": 0.9333333333333332,
"n": 38
},
"otp": {
"p": 0.9210526315789473,
"r": 0.875,
"f1": 0.8974358974358975,
"n": 40
},
"payment": {
"p": 0.7142857142857143,
"r": 0.625,
"f1": 0.6666666666666666,
"n": 8
},
"address": {
"p": 0.7941176470588235,
"r": 0.7714285714285715,
"f1": 0.782608695652174,
"n": 35
},
"newsletter": {
"p": 0.9725609756097561,
"r": 0.9565217391304348,
"f1": 0.9644746787603932,
"n": 667
},
"contact": {
"p": 0.9453262786596119,
"r": 0.9804878048780488,
"f1": 0.9625860520802155,
"n": 1640
},
"search": {
"p": 0.967487684729064,
"r": 0.9415148609779482,
"f1": 0.9543245869776482,
"n": 1043
},
"other": {
"p": 0.9021113243761996,
"r": 0.8834586466165414,
"f1": 0.892687559354226,
"n": 1064
}
},
"top_confusions": [
[
"given-name",
"name",
48
],
[
"search",
"other",
39
],
[
"address-level2",
"other",
27
],
[
"username",
"email",
26
],
[
"other",
"search",
25
],
[
"other",
"address-level2",
22
],
[
"other",
"country",
19
],
[
"address-level1",
"other",
18
],
[
"name",
"other",
15
],
[
"other",
"address-level1",
15
],
[
"other",
"tel",
15
],
[
"captcha",
"other",
14
],
[
"new-password",
"current-password",
13
],
[
"other",
"username",
13
],
[
"other",
"name",
12
],
[
"other",
"email",
11
],
[
"name",
"given-name",
11
],
[
"address-line2",
"other",
11
],
[
"other",
"street-address",
11
],
[
"other",
"current-password",
10
],
[
"other",
"captcha",
9
],
[
"other",
"postal-code",
9
],
[
"email",
"username",
8
],
[
"email",
"other",
8
],
[
"organization",
"other",
8
]
]
},
"model_synthetic": {
"n_forms": 84,
"n_fields": 319,
"field_acc": 0.9780564263322884,
"field_macro_f1": 0.9765257496746269,
"form_acc": 0.9642857142857143,
"form_macro_f1": 0.9578655564477768,
"password_precision": 1.0,
"password_recall": 1.0,
"password_f1": 1.0,
"cur_vs_new_acc": 0.9743589743589743,
"identifier_f1": 1.0,
"per_field": {
"username": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 7
},
"email": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 16
},
"current-password": {
"p": 0.9333333333333333,
"r": 1.0,
"f1": 0.9655172413793104,
"n": 14
},
"new-password": {
"p": 1.0,
"r": 0.96,
"f1": 0.9795918367346939,
"n": 25
},
"one-time-code": {
"p": 0.9285714285714286,
"r": 1.0,
"f1": 0.962962962962963,
"n": 39
},
"tel": {
"p": 0.8666666666666667,
"r": 1.0,
"f1": 0.9285714285714286,
"n": 13
},
"name": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 6
},
"given-name": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 6
},
"family-name": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 6
},
"organization": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 4
},
"street-address": {
"p": 1.0,
"r": 0.9333333333333333,
"f1": 0.9655172413793104,
"n": 15
},
"address-line2": {
"p": 1.0,
"r": 0.8,
"f1": 0.888888888888889,
"n": 5
},
"address-level2": {
"p": 1.0,
"r": 0.9166666666666666,
"f1": 0.9565217391304348,
"n": 12
},
"address-level1": {
"p": 1.0,
"r": 0.8888888888888888,
"f1": 0.9411764705882353,
"n": 9
},
"postal-code": {
"p": 1.0,
"r": 0.9333333333333333,
"f1": 0.9655172413793104,
"n": 15
},
"country": {
"p": 1.0,
"r": 0.8888888888888888,
"f1": 0.9411764705882353,
"n": 9
},
"cc-name": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 21
},
"cc-number": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 26
},
"cc-exp": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 18
},
"cc-exp-month": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 8
},
"cc-exp-year": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 8
},
"cc-csc": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 25
},
"captcha": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 4
},
"other": {
"p": 0.8888888888888888,
"r": 1.0,
"f1": 0.9411764705882353,
"n": 8
}
},
"per_form": {
"login": {
"p": 0.8888888888888888,
"r": 1.0,
"f1": 0.9411764705882353,
"n": 8
},
"password-change": {
"p": 1.0,
"r": 0.9375,
"f1": 0.967741935483871,
"n": 16
},
"password-recovery": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 8
},
"otp": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 18
},
"payment": {
"p": 1.0,
"r": 0.9629629629629629,
"f1": 0.9811320754716981,
"n": 27
},
"address": {
"p": 0.8571428571428571,
"r": 0.8571428571428571,
"f1": 0.8571428571428571,
"n": 7
}
},
"top_confusions": [
[
"new-password",
"current-password",
1
],
[
"street-address",
"tel",
1
],
[
"address-line2",
"one-time-code",
1
],
[
"address-level2",
"other",
1
],
[
"address-level1",
"one-time-code",
1
],
[
"postal-code",
"one-time-code",
1
],
[
"country",
"tel",
1
]
]
},
"heuristic_baseline_real": {
"n_forms": 6838,
"n_fields": 19811,
"field_acc": 0.827368633587401,
"field_macro_f1": 0.7233335250918939,
"form_acc": 0.7157063468850541,
"form_macro_f1": 0.48529320010648663,
"password_precision": 0.9391143911439115,
"password_recall": 0.9980392156862745,
"password_f1": 0.967680608365019,
"cur_vs_new_acc": 0.8545098039215686,
"identifier_f1": 0.9453604914322663,
"per_field": {
"username": {
"p": 0.7456213511259383,
"r": 0.6962616822429907,
"f1": 0.7200966572694322,
"n": 1284
},
"email": {
"p": 0.8662225922842076,
"r": 0.9774506733479487,
"f1": 0.918481459682166,
"n": 3193
},
"current-password": {
"p": 0.8279176201372997,
"r": 0.984221980413493,
"f1": 0.8993288590604027,
"n": 1838
},
"new-password": {
"p": 0.7047619047619048,
"r": 0.5196629213483146,
"f1": 0.5982215036378336,
"n": 712
},
"one-time-code": {
"p": 0.6486486486486487,
"r": 0.35555555555555557,
"f1": 0.45933014354066987,
"n": 135
},
"tel": {
"p": 0.8867791842475387,
"r": 0.9431563201196709,
"f1": 0.91409931134469,
"n": 1337
},
"name": {
"p": 0.8157695223654283,
"r": 0.7364818617385352,
"f1": 0.7741007194244605,
"n": 1461
},
"given-name": {
"p": 0.6842105263157895,
"r": 0.7114427860696517,
"f1": 0.697560975609756,
"n": 603
},
"family-name": {
"p": 0.9160671462829736,
"r": 0.7549407114624506,
"f1": 0.8277356446370531,
"n": 506
},
"organization": {
"p": 0.8041666666666667,
"r": 0.75390625,
"f1": 0.778225806451613,
"n": 256
},
"street-address": {
"p": 0.6370370370370371,
"r": 0.8269230769230769,
"f1": 0.7196652719665272,
"n": 208
},
"address-line2": {
"p": 0.453125,
"r": 0.6041666666666666,
"f1": 0.5178571428571429,
"n": 48
},
"address-level2": {
"p": 0.6318537859007833,
"r": 0.8287671232876712,
"f1": 0.717037037037037,
"n": 292
},
"address-level1": {
"p": 0.5258964143426295,
"r": 0.7542857142857143,
"f1": 0.6197183098591549,
"n": 175
},
"postal-code": {
"p": 0.925,
"r": 0.8043478260869565,
"f1": 0.8604651162790697,
"n": 322
},
"country": {
"p": 0.8226950354609929,
"r": 0.7341772151898734,
"f1": 0.7759197324414716,
"n": 158
},
"cc-name": {
"p": 0.5,
"r": 1.0,
"f1": 0.6666666666666666,
"n": 2
},
"cc-number": {
"p": 0.5,
"r": 1.0,
"f1": 0.6666666666666666,
"n": 6
},
"cc-exp-month": {
"p": 1.0,
"r": 0.5,
"f1": 0.6666666666666666,
"n": 6
},
"cc-exp-year": {
"p": 0.6,
"r": 0.5,
"f1": 0.5454545454545454,
"n": 6
},
"cc-csc": {
"p": 0.4166666666666667,
"r": 0.8333333333333334,
"f1": 0.5555555555555556,
"n": 6
},
"bday": {
"p": 0.9318181818181818,
"r": 0.7735849056603774,
"f1": 0.8453608247422681,
"n": 53
},
"search": {
"p": 0.8284714119019837,
"r": 0.7489451476793249,
"f1": 0.7867036011080333,
"n": 948
},
"captcha": {
"p": 0.9297297297297298,
"r": 0.5714285714285714,
"f1": 0.7078189300411523,
"n": 301
},
"other": {
"p": 0.8645796693633486,
"r": 0.8255247691015953,
"f1": 0.8446009792973114,
"n": 5955
}
},
"per_form": {
"login": {
"p": 0.94069640914037,
"r": 0.9417211328976035,
"f1": 0.9412084921066958,
"n": 1836
},
"signup": {
"p": 0.5643224699828473,
"r": 0.7889688249400479,
"f1": 0.6579999999999999,
"n": 417
},
"password-change": {
"p": 0.0,
"r": 0.0,
"f1": 0.0,
"n": 50
},
"password-recovery": {
"p": 0.26666666666666666,
"r": 0.21052631578947367,
"f1": 0.23529411764705882,
"n": 38
},
"otp": {
"p": 0.3,
"r": 0.375,
"f1": 0.33333333333333326,
"n": 40
},
"payment": {
"p": 0.21052631578947367,
"r": 0.5,
"f1": 0.2962962962962963,
"n": 8
},
"address": {
"p": 0.03722504230118443,
"r": 0.6285714285714286,
"f1": 0.07028753993610225,
"n": 35
},
"newsletter": {
"p": 0.8864774624373957,
"r": 0.7961019490254873,
"f1": 0.8388625592417062,
"n": 667
},
"contact": {
"p": 0.9479371316306483,
"r": 0.5884146341463414,
"f1": 0.7261098570353649,
"n": 1640
},
"search": {
"p": 0.984313725490196,
"r": 0.4813039309683605,
"f1": 0.6464906632324533,
"n": 1043
},
"other": {
"p": 0.493125,
"r": 0.7415413533834586,
"f1": 0.5923423423423424,
"n": 1064
}
},
"top_confusions": [
[
"new-password",
"current-password",
339
],
[
"username",
"email",
309
],
[
"name",
"given-name",
148
],
[
"name",
"other",
135
],
[
"other",
"search",
134
],
[
"search",
"other",
128
],
[
"other",
"email",
118
],
[
"other",
"new-password",
117
],
[
"other",
"address-level2",
108
],
[
"captcha",
"other",
100
],
[
"given-name",
"name",
91
],
[
"other",
"name",
89
],
[
"other",
"username",
84
],
[
"other",
"tel",
84
],
[
"other",
"address-level1",
82
],
[
"family-name",
"other",
63
],
[
"email",
"username",
62
],
[
"tel",
"other",
54
],
[
"name",
"username",
52
],
[
"username",
"other",
49
],
[
"given-name",
"other",
39
],
[
"other",
"street-address",
39
],
[
"one-time-code",
"other",
39
],
[
"other",
"current-password",
32
],
[
"other",
"address-line2",
31
]
]
},
"autocomplete_masked": {
"n_fields": 1622,
"acc": 0.9784217016029593,
"per_class": {
"address-level1": "4/4",
"address-level2": "12/14",
"address-line2": "0/3",
"bday": "5/5",
"cc-csc": "1/1",
"cc-exp-month": "1/1",
"cc-exp-year": "1/1",
"cc-number": "1/1",
"country": "6/6",
"current-password": "323/324",
"email": "417/418",
"family-name": "34/37",
"given-name": "39/42",
"name": "182/187",
"new-password": "180/180",
"one-time-code": "7/7",
"organization": "16/16",
"postal-code": "26/30",
"street-address": "11/13",
"tel": "111/111",
"username": "210/221"
},
"heuristic_acc": 0.7866831072749692
}
}