{ "split": "test", "dtype": "bfloat16", "forms_per_s_gpu": 2022.5467036804282, "model": { "n_forms": 6922, "n_fields": 20130, "field_acc": 0.9719821162444113, "field_macro_f1": 0.9597690495523868, "form_acc": 0.9572377925455071, "form_macro_f1": 0.9327853059996515, "password_precision": 0.9942263279445728, "password_recall": 0.9976825028968713, "password_f1": 0.9959514170040485, "cur_vs_new_acc": 0.9915025106218617, "identifier_f1": 0.9939004103360319, "per_field": { "username": { "p": 0.9782101167315175, "r": 0.9736638264910922, "f1": 0.9759316770186335, "n": 1291 }, "email": { "p": 0.9870049504950495, "r": 0.9940791523839202, "f1": 0.9905294208973761, "n": 3209 }, "current-password": { "p": 0.9866238630283574, "r": 0.9956803455723542, "f1": 0.9911314162859447, "n": 1852 }, "new-password": { "p": 0.9917695473251029, "r": 0.9810040705563093, "f1": 0.9863574351978172, "n": 737 }, "one-time-code": { "p": 0.9318181818181818, "r": 0.9425287356321839, "f1": 0.9371428571428572, "n": 174 }, "tel": { "p": 0.9809802487198245, "r": 0.9933333333333333, "f1": 0.9871181450128819, "n": 1350 }, "name": { "p": 0.9501661129568106, "r": 0.9747784594410361, "f1": 0.9623149394347241, "n": 1467 }, "given-name": { "p": 0.9754385964912281, "r": 0.9129720853858785, "f1": 0.9431721798134012, "n": 609 }, "family-name": { "p": 0.9881422924901185, "r": 0.9765625, "f1": 0.9823182711198427, "n": 512 }, "organization": { "p": 0.953307392996109, "r": 0.9423076923076923, "f1": 0.9477756286266925, "n": 260 }, "street-address": { "p": 0.9396551724137931, "r": 0.9775784753363229, "f1": 0.9582417582417583, "n": 223 }, "address-line2": { "p": 0.975609756097561, "r": 0.7547169811320755, "f1": 0.851063829787234, "n": 53 }, "address-level2": { "p": 0.9149659863945578, "r": 0.8848684210526315, "f1": 0.8996655518394647, "n": 304 }, "address-level1": { "p": 0.9005524861878453, "r": 0.8858695652173914, "f1": 0.8931506849315068, "n": 184 }, "postal-code": { "p": 0.9731343283582089, "r": 0.9673590504451038, "f1": 0.9702380952380952, "n": 337 }, "country": { "p": 0.8743169398907104, "r": 0.9580838323353293, "f1": 0.9142857142857143, "n": 167 }, "cc-name": { "p": 0.9583333333333334, "r": 1.0, "f1": 0.9787234042553191, "n": 23 }, "cc-number": { "p": 0.9696969696969697, "r": 1.0, "f1": 0.9846153846153847, "n": 32 }, "cc-exp": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 18 }, "cc-exp-month": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 14 }, "cc-exp-year": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 14 }, "cc-csc": { "p": 1.0, "r": 0.967741935483871, "f1": 0.9836065573770492, "n": 31 }, "bday": { "p": 0.96, "r": 0.9056603773584906, "f1": 0.9320388349514563, "n": 53 }, "search": { "p": 0.9670212765957447, "r": 0.9588607594936709, "f1": 0.9629237288135594, "n": 948 }, "captcha": { "p": 0.9632107023411371, "r": 0.9442622950819672, "f1": 0.9536423841059603, "n": 305 }, "other": { "p": 0.9693911873528422, "r": 0.9666275364749287, "f1": 0.9680073893693845, "n": 5963 } }, "per_form": { "login": { "p": 0.9886914378029079, "r": 0.9956616052060737, "f1": 0.9921642799243447, "n": 1844 }, "signup": { "p": 0.9777227722772277, "r": 0.947242206235012, "f1": 0.9622411693057246, "n": 417 }, "password-change": { "p": 0.9154929577464789, "r": 0.9848484848484849, "f1": 0.948905109489051, "n": 66 }, "password-recovery": { "p": 0.9555555555555556, "r": 0.9347826086956522, "f1": 0.945054945054945, "n": 46 }, "otp": { "p": 0.9464285714285714, "r": 0.9137931034482759, "f1": 0.9298245614035087, "n": 58 }, "payment": { "p": 0.9393939393939394, "r": 0.8857142857142857, "f1": 0.9117647058823529, "n": 35 }, "address": { "p": 0.8048780487804879, "r": 0.7857142857142857, "f1": 0.7951807228915663, "n": 42 }, "newsletter": { "p": 0.9740458015267176, "r": 0.9565217391304348, "f1": 0.9652042360060515, "n": 667 }, "contact": { "p": 0.9459141681363904, "r": 0.9810975609756097, "f1": 0.9631846752469321, "n": 1640 }, "search": { "p": 0.967487684729064, "r": 0.9415148609779482, "f1": 0.9543245869776482, "n": 1043 }, "other": { "p": 0.9013409961685823, "r": 0.8843984962406015, "f1": 0.8927893738140418, "n": 1064 } }, "top_confusions": [ [ "given-name", "name", 48 ], [ "search", "other", 39 ], [ "address-level2", "other", 28 ], [ "username", "email", 26 ], [ "other", "search", 25 ], [ "other", "address-level2", 22 ], [ "other", "country", 19 ], [ "address-level1", "other", 18 ], [ "name", "other", 15 ], [ "other", "address-level1", 15 ], [ "other", "tel", 15 ], [ "new-password", "current-password", 14 ], [ "captcha", "other", 14 ], [ "other", "username", 13 ], [ "other", "name", 12 ], [ "other", "email", 11 ], [ "address-line2", "other", 11 ], [ "other", "street-address", 11 ], [ "other", "current-password", 10 ], [ "name", "given-name", 10 ], [ "other", "captcha", 9 ], [ "other", "postal-code", 9 ], [ "email", "username", 8 ], [ "email", "other", 8 ], [ "organization", "other", 8 ] ] }, "model_real": { "n_forms": 6838, "n_fields": 19811, "field_acc": 0.971884306698299, "field_macro_f1": 0.9447870267362889, "form_acc": 0.9571512138052062, "form_macro_f1": 0.9049098319929233, "password_precision": 0.9941383352872216, "password_recall": 0.9976470588235294, "password_f1": 0.9958896065766295, "cur_vs_new_acc": 0.991764705882353, "identifier_f1": 0.993869133875822, "per_field": { "username": { "p": 0.9780907668231612, "r": 0.9735202492211839, "f1": 0.975800156128025, "n": 1284 }, "email": { "p": 0.9869402985074627, "r": 0.9940494832445975, "f1": 0.9904821344983618, "n": 3193 }, "current-password": { "p": 0.9870550161812298, "r": 0.9956474428726877, "f1": 0.991332611050921, "n": 1838 }, "new-password": { "p": 0.9914893617021276, "r": 0.9817415730337079, "f1": 0.9865913902611151, "n": 712 }, "one-time-code": { "p": 0.9328358208955224, "r": 0.9259259259259259, "f1": 0.929368029739777, "n": 135 }, "tel": { "p": 0.9822485207100592, "r": 0.9932685115931189, "f1": 0.9877277798438081, "n": 1337 }, "name": { "p": 0.9499666444296198, "r": 0.974674880219028, "f1": 0.9621621621621621, "n": 1461 }, "given-name": { "p": 0.975177304964539, "r": 0.912106135986733, "f1": 0.9425878320479862, "n": 603 }, "family-name": { "p": 0.988, "r": 0.9762845849802372, "f1": 0.9821073558648111, "n": 506 }, "organization": { "p": 0.9525691699604744, "r": 0.94140625, "f1": 0.9469548133595285, "n": 256 }, "street-address": { "p": 0.9357798165137615, "r": 0.9807692307692307, "f1": 0.9577464788732395, "n": 208 }, "address-line2": { "p": 0.972972972972973, "r": 0.75, "f1": 0.8470588235294119, "n": 48 }, "address-level2": { "p": 0.911660777385159, "r": 0.8835616438356164, "f1": 0.8973913043478261, "n": 292 }, "address-level1": { "p": 0.8959537572254336, "r": 0.8857142857142857, "f1": 0.8908045977011495, "n": 175 }, "postal-code": { "p": 0.9719626168224299, "r": 0.968944099378882, "f1": 0.9704510108864696, "n": 322 }, "country": { "p": 0.8685714285714285, "r": 0.9620253164556962, "f1": 0.9129129129129129, "n": 158 }, "cc-name": { "p": 0.6666666666666666, "r": 1.0, "f1": 0.8, "n": 2 }, "cc-number": { "p": 0.8571428571428571, "r": 1.0, "f1": 0.923076923076923, "n": 6 }, "cc-exp-month": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 6 }, "cc-exp-year": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 6 }, "cc-csc": { "p": 1.0, "r": 0.8333333333333334, "f1": 0.9090909090909091, "n": 6 }, "bday": { "p": 0.96, "r": 0.9056603773584906, "f1": 0.9320388349514563, "n": 53 }, "search": { "p": 0.9670212765957447, "r": 0.9588607594936709, "f1": 0.9629237288135594, "n": 948 }, "captcha": { "p": 0.9627118644067797, "r": 0.9435215946843853, "f1": 0.953020134228188, "n": 301 }, "other": { "p": 0.9695132221660772, "r": 0.9665827036104114, "f1": 0.9680457450386813, "n": 5955 } }, "per_form": { "login": { "p": 0.9891774891774892, "r": 0.9956427015250545, "f1": 0.992399565689468, "n": 1836 }, "signup": { "p": 0.9777227722772277, "r": 0.947242206235012, "f1": 0.9622411693057246, "n": 417 }, "password-change": { "p": 0.8928571428571429, "r": 1.0, "f1": 0.9433962264150945, "n": 50 }, "password-recovery": { "p": 0.9459459459459459, "r": 0.9210526315789473, "f1": 0.9333333333333332, "n": 38 }, "otp": { "p": 0.9210526315789473, "r": 0.875, "f1": 0.8974358974358975, "n": 40 }, "payment": { "p": 0.7142857142857143, "r": 0.625, "f1": 0.6666666666666666, "n": 8 }, "address": { "p": 0.7941176470588235, "r": 0.7714285714285715, "f1": 0.782608695652174, "n": 35 }, "newsletter": { "p": 0.9740458015267176, "r": 0.9565217391304348, "f1": 0.9652042360060515, "n": 667 }, "contact": { "p": 0.9459141681363904, "r": 0.9810975609756097, "f1": 0.9631846752469321, "n": 1640 }, "search": { "p": 0.967487684729064, "r": 0.9415148609779482, "f1": 0.9543245869776482, "n": 1043 }, "other": { "p": 0.9022051773729626, "r": 0.8843984962406015, "f1": 0.8932130991931656, "n": 1064 } }, "top_confusions": [ [ "given-name", "name", 48 ], [ "search", "other", 39 ], [ "address-level2", "other", 27 ], [ "username", "email", 26 ], [ "other", "search", 25 ], [ "other", "address-level2", 22 ], [ "other", "country", 19 ], [ "address-level1", "other", 18 ], [ "name", "other", 15 ], [ "other", "address-level1", 15 ], [ "other", "tel", 15 ], [ "captcha", "other", 14 ], [ "new-password", "current-password", 13 ], [ "other", "username", 13 ], [ "other", "name", 12 ], [ "other", "email", 11 ], [ "address-line2", "other", 11 ], [ "other", "street-address", 11 ], [ "other", "current-password", 10 ], [ "name", "given-name", 10 ], [ "other", "captcha", 9 ], [ "other", "postal-code", 9 ], [ "email", "username", 8 ], [ "email", "other", 8 ], [ "organization", "other", 8 ] ] }, "model_synthetic": { "n_forms": 84, "n_fields": 319, "field_acc": 0.9780564263322884, "field_macro_f1": 0.9765257496746269, "form_acc": 0.9642857142857143, "form_macro_f1": 0.9578655564477768, "password_precision": 1.0, "password_recall": 1.0, "password_f1": 1.0, "cur_vs_new_acc": 0.9743589743589743, "identifier_f1": 1.0, "per_field": { "username": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 7 }, "email": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 16 }, "current-password": { "p": 0.9333333333333333, "r": 1.0, "f1": 0.9655172413793104, "n": 14 }, "new-password": { "p": 1.0, "r": 0.96, "f1": 0.9795918367346939, "n": 25 }, "one-time-code": { "p": 0.9285714285714286, "r": 1.0, "f1": 0.962962962962963, "n": 39 }, "tel": { "p": 0.8666666666666667, "r": 1.0, "f1": 0.9285714285714286, "n": 13 }, "name": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 6 }, "given-name": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 6 }, "family-name": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 6 }, "organization": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 4 }, "street-address": { "p": 1.0, "r": 0.9333333333333333, "f1": 0.9655172413793104, "n": 15 }, "address-line2": { "p": 1.0, "r": 0.8, "f1": 0.888888888888889, "n": 5 }, "address-level2": { "p": 1.0, "r": 0.9166666666666666, "f1": 0.9565217391304348, "n": 12 }, "address-level1": { "p": 1.0, "r": 0.8888888888888888, "f1": 0.9411764705882353, "n": 9 }, "postal-code": { "p": 1.0, "r": 0.9333333333333333, "f1": 0.9655172413793104, "n": 15 }, "country": { "p": 1.0, "r": 0.8888888888888888, "f1": 0.9411764705882353, "n": 9 }, "cc-name": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 21 }, "cc-number": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 26 }, "cc-exp": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 18 }, "cc-exp-month": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 8 }, "cc-exp-year": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 8 }, "cc-csc": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 25 }, "captcha": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 4 }, "other": { "p": 0.8888888888888888, "r": 1.0, "f1": 0.9411764705882353, "n": 8 } }, "per_form": { "login": { "p": 0.8888888888888888, "r": 1.0, "f1": 0.9411764705882353, "n": 8 }, "password-change": { "p": 1.0, "r": 0.9375, "f1": 0.967741935483871, "n": 16 }, "password-recovery": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 8 }, "otp": { "p": 1.0, "r": 1.0, "f1": 1.0, "n": 18 }, "payment": { "p": 1.0, "r": 0.9629629629629629, "f1": 0.9811320754716981, "n": 27 }, "address": { "p": 0.8571428571428571, "r": 0.8571428571428571, "f1": 0.8571428571428571, "n": 7 } }, "top_confusions": [ [ "new-password", "current-password", 1 ], [ "street-address", "tel", 1 ], [ "address-line2", "one-time-code", 1 ], [ "address-level2", "other", 1 ], [ "address-level1", "one-time-code", 1 ], [ "postal-code", "one-time-code", 1 ], [ "country", "tel", 1 ] ] }, "heuristic_baseline_real": { "n_forms": 6838, "n_fields": 19811, "field_acc": 0.827368633587401, "field_macro_f1": 0.7233335250918939, "form_acc": 0.7157063468850541, "form_macro_f1": 0.48529320010648663, "password_precision": 0.9391143911439115, "password_recall": 0.9980392156862745, "password_f1": 0.967680608365019, "cur_vs_new_acc": 0.8545098039215686, "identifier_f1": 0.9453604914322663, "per_field": { "username": { "p": 0.7456213511259383, "r": 0.6962616822429907, "f1": 0.7200966572694322, "n": 1284 }, "email": { "p": 0.8662225922842076, "r": 0.9774506733479487, "f1": 0.918481459682166, "n": 3193 }, "current-password": { "p": 0.8279176201372997, "r": 0.984221980413493, "f1": 0.8993288590604027, "n": 1838 }, "new-password": { "p": 0.7047619047619048, "r": 0.5196629213483146, "f1": 0.5982215036378336, "n": 712 }, "one-time-code": { "p": 0.6486486486486487, "r": 0.35555555555555557, "f1": 0.45933014354066987, "n": 135 }, "tel": { "p": 0.8867791842475387, "r": 0.9431563201196709, "f1": 0.91409931134469, "n": 1337 }, "name": { "p": 0.8157695223654283, "r": 0.7364818617385352, "f1": 0.7741007194244605, "n": 1461 }, "given-name": { "p": 0.6842105263157895, "r": 0.7114427860696517, "f1": 0.697560975609756, "n": 603 }, "family-name": { "p": 0.9160671462829736, "r": 0.7549407114624506, "f1": 0.8277356446370531, "n": 506 }, "organization": { "p": 0.8041666666666667, "r": 0.75390625, "f1": 0.778225806451613, "n": 256 }, "street-address": { "p": 0.6370370370370371, "r": 0.8269230769230769, "f1": 0.7196652719665272, "n": 208 }, "address-line2": { "p": 0.453125, "r": 0.6041666666666666, "f1": 0.5178571428571429, "n": 48 }, "address-level2": { "p": 0.6318537859007833, "r": 0.8287671232876712, "f1": 0.717037037037037, "n": 292 }, "address-level1": { "p": 0.5258964143426295, "r": 0.7542857142857143, "f1": 0.6197183098591549, "n": 175 }, "postal-code": { "p": 0.925, "r": 0.8043478260869565, "f1": 0.8604651162790697, "n": 322 }, "country": { "p": 0.8226950354609929, "r": 0.7341772151898734, "f1": 0.7759197324414716, "n": 158 }, "cc-name": { "p": 0.5, "r": 1.0, "f1": 0.6666666666666666, "n": 2 }, "cc-number": { "p": 0.5, "r": 1.0, "f1": 0.6666666666666666, "n": 6 }, "cc-exp-month": { "p": 1.0, "r": 0.5, "f1": 0.6666666666666666, "n": 6 }, "cc-exp-year": { "p": 0.6, "r": 0.5, "f1": 0.5454545454545454, "n": 6 }, "cc-csc": { "p": 0.4166666666666667, "r": 0.8333333333333334, "f1": 0.5555555555555556, "n": 6 }, "bday": { "p": 0.9318181818181818, "r": 0.7735849056603774, "f1": 0.8453608247422681, "n": 53 }, "search": { "p": 0.8284714119019837, "r": 0.7489451476793249, "f1": 0.7867036011080333, "n": 948 }, "captcha": { "p": 0.9297297297297298, "r": 0.5714285714285714, "f1": 0.7078189300411523, "n": 301 }, "other": { "p": 0.8645796693633486, "r": 0.8255247691015953, "f1": 0.8446009792973114, "n": 5955 } }, "per_form": { "login": { "p": 0.94069640914037, "r": 0.9417211328976035, "f1": 0.9412084921066958, "n": 1836 }, "signup": { "p": 0.5643224699828473, "r": 0.7889688249400479, "f1": 0.6579999999999999, "n": 417 }, "password-change": { "p": 0.0, "r": 0.0, "f1": 0.0, "n": 50 }, "password-recovery": { "p": 0.26666666666666666, "r": 0.21052631578947367, "f1": 0.23529411764705882, "n": 38 }, "otp": { "p": 0.3, "r": 0.375, "f1": 0.33333333333333326, "n": 40 }, "payment": { "p": 0.21052631578947367, "r": 0.5, "f1": 0.2962962962962963, "n": 8 }, "address": { "p": 0.03722504230118443, "r": 0.6285714285714286, "f1": 0.07028753993610225, "n": 35 }, "newsletter": { "p": 0.8864774624373957, "r": 0.7961019490254873, "f1": 0.8388625592417062, "n": 667 }, "contact": { "p": 0.9479371316306483, "r": 0.5884146341463414, "f1": 0.7261098570353649, "n": 1640 }, "search": { "p": 0.984313725490196, "r": 0.4813039309683605, "f1": 0.6464906632324533, "n": 1043 }, "other": { "p": 0.493125, "r": 0.7415413533834586, "f1": 0.5923423423423424, "n": 1064 } }, "top_confusions": [ [ "new-password", "current-password", 339 ], [ "username", "email", 309 ], [ "name", "given-name", 148 ], [ "name", "other", 135 ], [ "other", "search", 134 ], [ "search", "other", 128 ], [ "other", "email", 118 ], [ "other", "new-password", 117 ], [ "other", "address-level2", 108 ], [ "captcha", "other", 100 ], [ "given-name", "name", 91 ], [ "other", "name", 89 ], [ "other", "username", 84 ], [ "other", "tel", 84 ], [ "other", "address-level1", 82 ], [ "family-name", "other", 63 ], [ "email", "username", 62 ], [ "tel", "other", 54 ], [ "name", "username", 52 ], [ "username", "other", 49 ], [ "given-name", "other", 39 ], [ "other", "street-address", 39 ], [ "one-time-code", "other", 39 ], [ "other", "current-password", 32 ], [ "other", "address-line2", 31 ] ] }, "autocomplete_masked": { "n_fields": 1622, "acc": 0.9784217016029593, "per_class": { "address-level1": "4/4", "address-level2": "12/14", "address-line2": "0/3", "bday": "5/5", "cc-csc": "1/1", "cc-exp-month": "1/1", "cc-exp-year": "1/1", "cc-number": "1/1", "country": "6/6", "current-password": "323/324", "email": "417/418", "family-name": "34/37", "given-name": "39/42", "name": "182/187", "new-password": "180/180", "one-time-code": "7/7", "organization": "16/16", "postal-code": "26/30", "street-address": "11/13", "tel": "111/111", "username": "210/221" }, "heuristic_acc": 0.7866831072749692 } }