Zap / eval /eval_val_bfloat16.json
ProCreations's picture
Zap v1: 20.7M-param web form & field classifier for autofill (bf16, MIT)
e4805d0 verified
Raw
History Blame Contribute Delete
21.3 kB
{
"split": "val",
"dtype": "bfloat16",
"forms_per_s_gpu": 2872.840982485025,
"model": {
"n_forms": 3408,
"n_fields": 9819,
"field_acc": 0.9747428455036155,
"field_macro_f1": 0.9662059517142381,
"form_acc": 0.9568661971830986,
"form_macro_f1": 0.9375185465621217,
"password_precision": 0.9898358092259578,
"password_recall": 0.998422712933754,
"password_f1": 0.994110718492344,
"cur_vs_new_acc": 0.9944794952681388,
"identifier_f1": 0.9945380063723259,
"per_field": {
"username": {
"p": 0.9921752738654147,
"r": 0.9768875192604006,
"f1": 0.984472049689441,
"n": 649
},
"email": {
"p": 0.9884541372674791,
"r": 0.9961215255332903,
"f1": 0.9922730199613651,
"n": 1547
},
"current-password": {
"p": 0.9814814814814815,
"r": 0.9977851605758582,
"f1": 0.9895661724327293,
"n": 903
},
"new-password": {
"p": 0.997229916897507,
"r": 0.9863013698630136,
"f1": 0.9917355371900827,
"n": 365
},
"one-time-code": {
"p": 0.9901960784313726,
"r": 0.9805825242718447,
"f1": 0.9853658536585367,
"n": 103
},
"tel": {
"p": 0.9815950920245399,
"r": 0.9937888198757764,
"f1": 0.9876543209876544,
"n": 644
},
"name": {
"p": 0.9592123769338959,
"r": 0.9855491329479769,
"f1": 0.9722024233784746,
"n": 692
},
"given-name": {
"p": 0.978494623655914,
"r": 0.9161073825503355,
"f1": 0.9462738301559791,
"n": 298
},
"family-name": {
"p": 0.9961089494163424,
"r": 0.9808429118773946,
"f1": 0.9884169884169886,
"n": 261
},
"organization": {
"p": 0.9761904761904762,
"r": 0.9318181818181818,
"f1": 0.9534883720930233,
"n": 132
},
"street-address": {
"p": 0.9369369369369369,
"r": 0.9122807017543859,
"f1": 0.9244444444444444,
"n": 114
},
"address-line2": {
"p": 1.0,
"r": 0.9090909090909091,
"f1": 0.9523809523809523,
"n": 22
},
"address-level2": {
"p": 0.8843537414965986,
"r": 0.8904109589041096,
"f1": 0.887372013651877,
"n": 146
},
"address-level1": {
"p": 0.898989898989899,
"r": 0.9368421052631579,
"f1": 0.9175257731958762,
"n": 95
},
"postal-code": {
"p": 0.9724137931034482,
"r": 0.9724137931034482,
"f1": 0.9724137931034482,
"n": 145
},
"country": {
"p": 0.8863636363636364,
"r": 0.9873417721518988,
"f1": 0.9341317365269461,
"n": 79
},
"cc-name": {
"p": 1.0,
"r": 0.95,
"f1": 0.9743589743589743,
"n": 20
},
"cc-number": {
"p": 0.9565217391304348,
"r": 1.0,
"f1": 0.9777777777777777,
"n": 22
},
"cc-exp": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 13
},
"cc-exp-month": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 9
},
"cc-exp-year": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 8
},
"cc-csc": {
"p": 0.9523809523809523,
"r": 1.0,
"f1": 0.975609756097561,
"n": 20
},
"bday": {
"p": 1.0,
"r": 0.8461538461538461,
"f1": 0.9166666666666666,
"n": 39
},
"search": {
"p": 0.9624217118997912,
"r": 0.9808510638297873,
"f1": 0.9715489989462592,
"n": 470
},
"captcha": {
"p": 0.9814814814814815,
"r": 0.9352941176470588,
"f1": 0.9578313253012049,
"n": 170
},
"other": {
"p": 0.9704016913319239,
"r": 0.9652996845425867,
"f1": 0.9678439641539274,
"n": 2853
}
},
"per_form": {
"login": {
"p": 0.9900552486187846,
"r": 0.9933481152993349,
"f1": 0.991698948533481,
"n": 902
},
"signup": {
"p": 0.9951456310679612,
"r": 0.9855769230769231,
"f1": 0.9903381642512078,
"n": 208
},
"password-change": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 22
},
"password-recovery": {
"p": 0.8461538461538461,
"r": 0.9565217391304348,
"f1": 0.8979591836734695,
"n": 23
},
"otp": {
"p": 0.8888888888888888,
"r": 0.8571428571428571,
"f1": 0.8727272727272727,
"n": 28
},
"payment": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 22
},
"address": {
"p": 0.8666666666666667,
"r": 0.7222222222222222,
"f1": 0.7878787878787877,
"n": 36
},
"newsletter": {
"p": 0.9774193548387097,
"r": 0.9439252336448598,
"f1": 0.9603803486529319,
"n": 321
},
"contact": {
"p": 0.9369482976040353,
"r": 0.9750656167979003,
"f1": 0.9556270096463023,
"n": 762
},
"search": {
"p": 0.9554730983302412,
"r": 0.9662288930581614,
"f1": 0.9608208955223881,
"n": 533
},
"other": {
"p": 0.9147727272727273,
"r": 0.8765880217785844,
"f1": 0.8952734012974978,
"n": 551
}
},
"top_confusions": [
[
"given-name",
"name",
22
],
[
"other",
"address-level2",
16
],
[
"other",
"search",
16
],
[
"address-level2",
"other",
15
],
[
"other",
"current-password",
11
],
[
"other",
"address-level1",
10
],
[
"captcha",
"other",
9
],
[
"other",
"country",
9
],
[
"search",
"other",
9
],
[
"other",
"email",
9
],
[
"username",
"email",
8
],
[
"street-address",
"other",
7
],
[
"organization",
"other",
7
],
[
"address-level1",
"other",
6
],
[
"name",
"given-name",
5
],
[
"username",
"other",
5
],
[
"other",
"tel",
5
],
[
"other",
"street-address",
5
],
[
"bday",
"other",
5
],
[
"new-password",
"current-password",
4
],
[
"name",
"other",
4
],
[
"other",
"postal-code",
4
],
[
"postal-code",
"other",
4
],
[
"other",
"name",
3
],
[
"family-name",
"name",
3
]
]
},
"model_real": {
"n_forms": 3362,
"n_fields": 9619,
"field_acc": 0.9747374987004886,
"field_macro_f1": 0.9605143456394969,
"form_acc": 0.9562760261748959,
"form_macro_f1": 0.9290284576847291,
"password_precision": 0.9912490055688147,
"password_recall": 0.9983974358974359,
"password_f1": 0.9948103792415169,
"cur_vs_new_acc": 0.9951923076923077,
"identifier_f1": 0.9945054945054944,
"per_field": {
"username": {
"p": 0.9921135646687698,
"r": 0.9767080745341615,
"f1": 0.9843505477308295,
"n": 644
},
"email": {
"p": 0.988394584139265,
"r": 0.9961013645224172,
"f1": 0.992233009708738,
"n": 1539
},
"current-password": {
"p": 0.9835706462212487,
"r": 0.9988876529477196,
"f1": 0.9911699779249449,
"n": 899
},
"new-password": {
"p": 1.0,
"r": 0.9856733524355301,
"f1": 0.9927849927849928,
"n": 349
},
"one-time-code": {
"p": 0.9868421052631579,
"r": 0.974025974025974,
"f1": 0.9803921568627451,
"n": 77
},
"tel": {
"p": 0.9813953488372092,
"r": 0.9937205651491365,
"f1": 0.987519500780031,
"n": 637
},
"name": {
"p": 0.9588652482269504,
"r": 0.9854227405247813,
"f1": 0.9719626168224299,
"n": 686
},
"given-name": {
"p": 0.9783393501805054,
"r": 0.9155405405405406,
"f1": 0.9458987783595114,
"n": 296
},
"family-name": {
"p": 0.996078431372549,
"r": 0.9806949806949807,
"f1": 0.9883268482490273,
"n": 259
},
"organization": {
"p": 0.9758064516129032,
"r": 0.9307692307692308,
"f1": 0.952755905511811,
"n": 130
},
"street-address": {
"p": 0.9326923076923077,
"r": 0.9065420560747663,
"f1": 0.919431279620853,
"n": 107
},
"address-line2": {
"p": 1.0,
"r": 0.8823529411764706,
"f1": 0.9375,
"n": 17
},
"address-level2": {
"p": 0.8794326241134752,
"r": 0.8857142857142857,
"f1": 0.8825622775800712,
"n": 140
},
"address-level1": {
"p": 0.8947368421052632,
"r": 0.9340659340659341,
"f1": 0.913978494623656,
"n": 91
},
"postal-code": {
"p": 0.9708029197080292,
"r": 0.9708029197080292,
"f1": 0.9708029197080292,
"n": 137
},
"country": {
"p": 0.8809523809523809,
"r": 0.9866666666666667,
"f1": 0.930817610062893,
"n": 75
},
"cc-name": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 3
},
"cc-number": {
"p": 0.75,
"r": 1.0,
"f1": 0.8571428571428571,
"n": 3
},
"cc-exp-month": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 3
},
"cc-exp-year": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 3
},
"cc-csc": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 3
},
"bday": {
"p": 1.0,
"r": 0.8461538461538461,
"f1": 0.9166666666666666,
"n": 39
},
"search": {
"p": 0.9624217118997912,
"r": 0.9808510638297873,
"f1": 0.9715489989462592,
"n": 470
},
"captcha": {
"p": 0.9808917197452229,
"r": 0.9333333333333333,
"f1": 0.9565217391304348,
"n": 165
},
"other": {
"p": 0.970712773465067,
"r": 0.9662802950474183,
"f1": 0.968491462770639,
"n": 2847
}
},
"per_form": {
"login": {
"p": 0.9900110987791343,
"r": 0.9933184855233853,
"f1": 0.9916620344635908,
"n": 898
},
"signup": {
"p": 0.9951219512195122,
"r": 0.9855072463768116,
"f1": 0.9902912621359223,
"n": 207
},
"password-change": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 15
},
"password-recovery": {
"p": 0.8181818181818182,
"r": 0.9473684210526315,
"f1": 0.8780487804878049,
"n": 19
},
"otp": {
"p": 0.8571428571428571,
"r": 0.8181818181818182,
"f1": 0.8372093023255814,
"n": 22
},
"payment": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 3
},
"address": {
"p": 0.84,
"r": 0.6774193548387096,
"f1": 0.75,
"n": 31
},
"newsletter": {
"p": 0.9774193548387097,
"r": 0.9439252336448598,
"f1": 0.9603803486529319,
"n": 321
},
"contact": {
"p": 0.9369482976040353,
"r": 0.9750656167979003,
"f1": 0.9556270096463023,
"n": 762
},
"search": {
"p": 0.9554730983302412,
"r": 0.9662288930581614,
"f1": 0.9608208955223881,
"n": 533
},
"other": {
"p": 0.9147727272727273,
"r": 0.8765880217785844,
"f1": 0.8952734012974978,
"n": 551
}
},
"top_confusions": [
[
"given-name",
"name",
22
],
[
"other",
"address-level2",
16
],
[
"other",
"search",
16
],
[
"address-level2",
"other",
15
],
[
"other",
"address-level1",
10
],
[
"captcha",
"other",
9
],
[
"other",
"country",
9
],
[
"search",
"other",
9
],
[
"other",
"email",
9
],
[
"other",
"current-password",
9
],
[
"username",
"email",
8
],
[
"street-address",
"other",
7
],
[
"organization",
"other",
7
],
[
"address-level1",
"other",
6
],
[
"name",
"given-name",
5
],
[
"username",
"other",
5
],
[
"other",
"tel",
5
],
[
"other",
"street-address",
5
],
[
"bday",
"other",
5
],
[
"new-password",
"current-password",
4
],
[
"name",
"other",
4
],
[
"other",
"postal-code",
4
],
[
"postal-code",
"other",
4
],
[
"other",
"name",
3
],
[
"family-name",
"name",
3
]
]
},
"model_synthetic": {
"n_forms": 46,
"n_fields": 200,
"field_acc": 0.975,
"field_macro_f1": 0.9657287157287158,
"form_acc": 1.0,
"form_macro_f1": 1.0,
"password_precision": 0.9090909090909091,
"password_recall": 1.0,
"password_f1": 0.9523809523809523,
"cur_vs_new_acc": 0.95,
"identifier_f1": 1.0,
"per_field": {
"username": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 5
},
"email": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 8
},
"current-password": {
"p": 0.6,
"r": 0.75,
"f1": 0.6666666666666665,
"n": 4
},
"new-password": {
"p": 0.9411764705882353,
"r": 1.0,
"f1": 0.9696969696969697,
"n": 16
},
"one-time-code": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 26
},
"tel": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 7
},
"name": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 6
},
"given-name": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 2
},
"family-name": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 2
},
"organization": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 2
},
"street-address": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 7
},
"address-line2": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 5
},
"address-level2": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 6
},
"address-level1": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 4
},
"postal-code": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 8
},
"country": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 4
},
"cc-name": {
"p": 1.0,
"r": 0.9411764705882353,
"f1": 0.9696969696969697,
"n": 17
},
"cc-number": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 19
},
"cc-exp": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 13
},
"cc-exp-month": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 6
},
"cc-exp-year": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 5
},
"cc-csc": {
"p": 0.9444444444444444,
"r": 1.0,
"f1": 0.9714285714285714,
"n": 17
},
"captcha": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 5
},
"other": {
"p": 0.75,
"r": 0.5,
"f1": 0.6,
"n": 6
}
},
"per_form": {
"login": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 4
},
"signup": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 1
},
"password-change": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 7
},
"password-recovery": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 4
},
"otp": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 6
},
"payment": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 19
},
"address": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 5
}
},
"top_confusions": [
[
"other",
"current-password",
2
],
[
"cc-name",
"other",
1
],
[
"current-password",
"new-password",
1
],
[
"other",
"cc-csc",
1
]
]
},
"heuristic_baseline_real": {
"n_forms": 3362,
"n_fields": 9619,
"field_acc": 0.819523859029005,
"field_macro_f1": 0.7496204195913461,
"form_acc": 0.7245687091017252,
"form_macro_f1": 0.5030916170134985,
"password_precision": 0.9431818181818182,
"password_recall": 0.9975961538461539,
"password_f1": 0.9696261682242991,
"cur_vs_new_acc": 0.8333333333333334,
"identifier_f1": 0.9389296024872307,
"per_field": {
"username": {
"p": 0.7634782608695653,
"r": 0.6816770186335404,
"f1": 0.7202625102543068,
"n": 644
},
"email": {
"p": 0.8578796561604585,
"r": 0.9727095516569201,
"f1": 0.9116930572472595,
"n": 1539
},
"current-password": {
"p": 0.8096980786825252,
"r": 0.9844271412680756,
"f1": 0.8885542168674699,
"n": 899
},
"new-password": {
"p": 0.6828193832599119,
"r": 0.44412607449856734,
"f1": 0.5381944444444444,
"n": 349
},
"one-time-code": {
"p": 0.7538461538461538,
"r": 0.6363636363636364,
"f1": 0.6901408450704226,
"n": 77
},
"tel": {
"p": 0.8990963855421686,
"r": 0.9372056514913658,
"f1": 0.9177555726364336,
"n": 637
},
"name": {
"p": 0.7942583732057417,
"r": 0.7259475218658892,
"f1": 0.7585681645087586,
"n": 686
},
"given-name": {
"p": 0.6656050955414012,
"r": 0.706081081081081,
"f1": 0.6852459016393442,
"n": 296
},
"family-name": {
"p": 0.9074074074074074,
"r": 0.7567567567567568,
"f1": 0.8252631578947369,
"n": 259
},
"organization": {
"p": 0.744,
"r": 0.7153846153846154,
"f1": 0.7294117647058823,
"n": 130
},
"street-address": {
"p": 0.7105263157894737,
"r": 0.7570093457943925,
"f1": 0.7330316742081447,
"n": 107
},
"address-line2": {
"p": 0.4782608695652174,
"r": 0.6470588235294118,
"f1": 0.55,
"n": 17
},
"address-level2": {
"p": 0.5699481865284974,
"r": 0.7857142857142857,
"f1": 0.6606606606606606,
"n": 140
},
"address-level1": {
"p": 0.5511811023622047,
"r": 0.7692307692307693,
"f1": 0.6422018348623854,
"n": 91
},
"postal-code": {
"p": 0.957983193277311,
"r": 0.8321167883211679,
"f1": 0.890625,
"n": 137
},
"country": {
"p": 0.8028169014084507,
"r": 0.76,
"f1": 0.7808219178082192,
"n": 75
},
"cc-name": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 3
},
"cc-number": {
"p": 0.6,
"r": 1.0,
"f1": 0.7499999999999999,
"n": 3
},
"cc-exp-month": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 3
},
"cc-exp-year": {
"p": 1.0,
"r": 1.0,
"f1": 1.0,
"n": 3
},
"cc-csc": {
"p": 0.0,
"r": 0.0,
"f1": 0.0,
"n": 3
},
"bday": {
"p": 0.8125,
"r": 0.6666666666666666,
"f1": 0.7323943661971831,
"n": 39
},
"search": {
"p": 0.8271889400921659,
"r": 0.7638297872340426,
"f1": 0.7942477876106194,
"n": 470
},
"captcha": {
"p": 0.9393939393939394,
"r": 0.5636363636363636,
"f1": 0.7045454545454546,
"n": 165
},
"other": {
"p": 0.855465884079237,
"r": 0.8191078328064629,
"f1": 0.8368921586219272,
"n": 2847
}
},
"per_form": {
"login": {
"p": 0.9395973154362416,
"r": 0.9354120267260579,
"f1": 0.9375,
"n": 898
},
"signup": {
"p": 0.5955882352941176,
"r": 0.782608695652174,
"f1": 0.6764091858037579,
"n": 207
},
"password-change": {
"p": 0.0,
"r": 0.0,
"f1": 0.0,
"n": 15
},
"password-recovery": {
"p": 0.13333333333333333,
"r": 0.10526315789473684,
"f1": 0.11764705882352941,
"n": 19
},
"otp": {
"p": 0.4482758620689655,
"r": 0.5909090909090909,
"f1": 0.5098039215686274,
"n": 22
},
"payment": {
"p": 0.1875,
"r": 1.0,
"f1": 0.3157894736842105,
"n": 3
},
"address": {
"p": 0.06713780918727916,
"r": 0.6129032258064516,
"f1": 0.12101910828025478,
"n": 31
},
"newsletter": {
"p": 0.8866666666666667,
"r": 0.8286604361370716,
"f1": 0.856682769726248,
"n": 321
},
"contact": {
"p": 0.9390756302521008,
"r": 0.5866141732283464,
"f1": 0.7221324717285944,
"n": 762
},
"search": {
"p": 0.9743589743589743,
"r": 0.49906191369606,
"f1": 0.6600496277915633,
"n": 533
},
"other": {
"p": 0.5199004975124378,
"r": 0.7586206896551724,
"f1": 0.6169741697416974,
"n": 551
}
},
"top_confusions": [
[
"new-password",
"current-password",
192
],
[
"username",
"email",
155
],
[
"name",
"given-name",
78
],
[
"other",
"email",
71
],
[
"other",
"search",
71
],
[
"search",
"other",
69
],
[
"other",
"address-level2",
62
],
[
"name",
"other",
62
],
[
"given-name",
"name",
51
],
[
"captcha",
"other",
50
],
[
"other",
"username",
48
],
[
"other",
"address-level1",
47
],
[
"other",
"new-password",
45
],
[
"other",
"name",
28
],
[
"other",
"tel",
28
],
[
"tel",
"other",
28
],
[
"name",
"username",
25
],
[
"one-time-code",
"other",
24
],
[
"email",
"username",
23
],
[
"other",
"organization",
23
],
[
"username",
"other",
23
],
[
"family-name",
"other",
22
],
[
"family-name",
"name",
19
],
[
"organization",
"other",
17
],
[
"street-address",
"other",
16
]
]
},
"autocomplete_masked": {
"n_fields": 641,
"acc": 0.9859594383775351,
"per_class": {
"address-level2": "3/4",
"address-line2": "2/2",
"country": "1/1",
"current-password": "157/157",
"email": "161/161",
"family-name": "11/11",
"given-name": "14/15",
"name": "74/75",
"new-password": "55/58",
"one-time-code": "13/13",
"organization": "5/5",
"postal-code": "6/6",
"street-address": "4/4",
"tel": "26/26",
"username": "100/103"
},
"heuristic_acc": 0.7550702028081123
}
}