{ "dataset": { "repo": "fastino/fast-decisions", "revision": "35afb79a0f611a1db44380e855ac2a821b0a4da1", "split": "development", "selection": "zero-based rows 0,10,...,90 per domain; no shuffling or adaptive selection", "files": { "agent_handoff.jsonl": "7c086a3a6a0014fc53d76df3912b93e1b5e4c293d92ab36f1620e5346c3269f8", "banking_intent.jsonl": "43af4aafdf6aeca74400505acbe528d1605aacb1097e1e1c0a169e9dfb6b5a77", "benefits_request.jsonl": "a37c96197152006fbb46e8ea0099dc6ab89feafbca424d21bd57b3f7ae3b235b", "clinic_request.jsonl": "84392b5b6de98c6cf7b474b45b6cffd19ab581770b89fab5d3eb5c77e9397652", "document_type.jsonl": "87798b1b7c57a553fa82b1c15495b4bcefcf9552e80185673f00e733d59552d7", "email_triage.jsonl": "779fdef3ad22d1f8fc0b8b81a15b7a591546ff2f5df3d4a850884e08033b11e0", "news_topic.jsonl": "c31f7710592c2f31f3634930da4840b535ceccd14a3092521547bc9c0f0a6b17", "paper_field.jsonl": "bb4513faf7616d2ee21694ee62fdc222b4e0e86a2ac0601298b23c538d23e5d0", "product_feedback.jsonl": "0b44b04d361cf29fc95728a5c1cae4506f42545fe1cb39af53b835287cba001f", "restaurant_review.jsonl": "64bf7cd3e498045f337956bfec1e002a456cb9aac02c23cac2911fd902914a67", "review_sentiment.jsonl": "413e8f6f81ce048d3e17f39ecd11cd28c1443253712a0930cfe8063bd1d82616", "screen_tags.jsonl": "5433a6cb7d67e88de2c9b9443bf3291d91ce89be4a7ea631b93d71d8b56e227b", "sports_recap.jsonl": "b2e1e6e174650d8b6a114f16dbacbd8b62f93ee77c3967c23d322e9848dac1b3", "support_intent.jsonl": "a293e95915be740dab18e0bba4d9ca1e31ce9ce2f86f127080ad172cdf89ce72", "support_topic.jsonl": "e7f41083889f229bf7eae58c48b42e0587d397dbfc959966bbfc379197102ed6", "ticket_route.jsonl": "8660f5b7bfc6f2475d5aaf2d3202a62d2c647ed49a950dc2dd04d76d14cffe08", "travel_request.jsonl": "6e596454de299d765d1d104e90d93543218f06e42b8b55bfc614cd2c8981ebfb" } }, "scope": "CPU FP32 inference with exact GGUF weights dequantized; separate head calls; max_len=1024 source words; multi-label threshold=0.5; no tuning; not upstream held-out test results or direct quantized kernel proof", "summary": { "examples": 170, "heads": 290, "domains": 17, "fp32_correct": 184, "q8_0_correct": 184, "head_agreement": 289, "max_logit_delta": 0.07910013198852539, "max_probability_delta": 0.018700537670787054, "head_agreement_rate": 0.996551724137931, "accuracy_delta": 0.0, "gate_passed": true }, "outputs": { "fp32": [ { "domain": "agent_handoff", "row": 0, "task": "should_handoff", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.583782970905304, "no": 0.6435691118240356 }, "probabilities": { "yes": 0.2266452091781303, "no": 0.7733547908218698 } }, { "domain": "agent_handoff", "row": 10, "task": "should_handoff", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.26927247643470764, "no": 0.3264096975326538 }, "probabilities": { "yes": 0.3553321644553713, "no": 0.6446678355446287 } }, { "domain": "agent_handoff", "row": 20, "task": "should_handoff", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 0.6657646298408508, "no": -0.7012766599655151 }, "probabilities": { "yes": 0.7969017088092811, "no": 0.2030982911907189 } }, { "domain": "agent_handoff", "row": 30, "task": "should_handoff", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 2.0951828956604004, "no": -1.9927005767822266 }, "probabilities": { "yes": 0.9835020480689189, "no": 0.016497951931081084 } }, { "domain": "agent_handoff", "row": 40, "task": "should_handoff", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 2.0783536434173584, "no": -2.043698787689209 }, "probabilities": { "yes": 0.9840474029800266, "no": 0.01595259701997331 } }, { "domain": "agent_handoff", "row": 50, "task": "should_handoff", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 1.5331108570098877, "no": -1.5852382183074951 }, "probabilities": { "yes": 0.9576433129354969, "no": 0.04235668706450313 } }, { "domain": "agent_handoff", "row": 60, "task": "should_handoff", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 1.9798635244369507, "no": -2.1049439907073975 }, "probabilities": { "yes": 0.9834520639962802, "no": 0.01654793600371987 } }, { "domain": "agent_handoff", "row": 70, "task": "should_handoff", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 1.413184642791748, "no": -1.4588661193847656 }, "probabilities": { "yes": 0.9464473855399121, "no": 0.05355261446008775 } }, { "domain": "agent_handoff", "row": 80, "task": "should_handoff", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.3019620478153229, "no": 0.3061848282814026 }, "probabilities": { "yes": 0.3524820363354785, "no": 0.6475179636645216 } }, { "domain": "agent_handoff", "row": 90, "task": "should_handoff", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.029221773147583, "no": 1.0818341970443726 }, "probabilities": { "yes": 0.10802687444592489, "no": 0.891973125554075 } }, { "domain": "banking_intent", "row": 0, "task": "intent", "multi_label": false, "gold": [ "other" ], "selected": [ "wire" ], "correct": false, "logits": { "balance": -5.706590175628662, "transfer": -4.188775539398193, "card_lost": -8.722769737243652, "dispute_charge": -6.790064811706543, "loan_payment": -1.8599499464035034, "mortgage": -5.697818756103516, "wire": 1.3769760131835938, "statement": -6.81149959564209, "fraud_alert": -6.929731369018555, "branch_appointment": -5.07404899597168, "fee_refund": -9.091879844665527, "direct_deposit": -7.2523932456970215, "close_account": -6.738659858703613, "new_account": -4.040200233459473, "limit_increase": -7.994538307189941, "other": -3.485309362411499 }, "probabilities": { "balance": 0.0007913074277904258, "transfer": 0.0036101376674274404, "card_lost": 3.876458173197204e-05, "dispute_charge": 0.00026779236246786414, "loan_payment": 0.03706123333198243, "mortgage": 0.0007982788471377488, "wire": 0.9434066871926621, "statement": 0.00026211337243360226, "fraud_alert": 0.00023288513658916186, "branch_appointment": 0.0014895476005548378, "fee_refund": 2.679986580837056e-05, "direct_deposit": 0.0001686597681223323, "close_account": 0.0002819181741702093, "new_account": 0.00418841010723571, "limit_increase": 8.02974944282975e-05, "other": 0.007295167069457597 } }, { "domain": "banking_intent", "row": 10, "task": "intent", "multi_label": false, "gold": [ "branch_appointment" ], "selected": [ "branch_appointment" ], "correct": true, "logits": { "balance": -3.5117037296295166, "transfer": -5.933591365814209, "card_lost": -7.479129314422607, "dispute_charge": -6.118645668029785, "loan_payment": -7.406546592712402, "mortgage": -8.9417142868042, "wire": -9.09257698059082, "statement": -7.120903015136719, "fraud_alert": -8.1964693069458, "branch_appointment": 5.0936479568481445, "fee_refund": -8.72911548614502, "direct_deposit": -6.331189155578613, "close_account": -7.157350540161133, "new_account": -6.103314399719238, "limit_increase": -8.678101539611816, "other": -4.6107611656188965 }, "probabilities": { "balance": 0.0001830644430313216, "transfer": 1.6247687685592087e-05, "card_lost": 3.463960729099955e-06, "dispute_charge": 1.3502789586887578e-05, "loan_payment": 3.7247337601101704e-06, "mortgage": 8.023805209524465e-07, "wire": 6.900197818891709e-07, "statement": 4.956198190884017e-06, "fraud_alert": 1.6905816836910384e-06, "branch_appointment": 0.9996794184747368, "fee_refund": 9.92455088872032e-07, "direct_deposit": 1.0917345040696895e-05, "close_account": 4.77880936202034e-06, "new_account": 1.3711399518307864e-05, "limit_increase": 1.044397776202149e-06, "other": 6.0994323506622274e-05 } }, { "domain": "banking_intent", "row": 20, "task": "intent", "multi_label": false, "gold": [ "other" ], "selected": [ "fraud_alert" ], "correct": false, "logits": { "balance": -3.258868455886841, "transfer": -3.4608685970306396, "card_lost": -3.1090939044952393, "dispute_charge": -2.7395334243774414, "loan_payment": -3.513746738433838, "mortgage": -4.423190116882324, "wire": -4.101810455322266, "statement": -3.6322453022003174, "fraud_alert": -0.8724449276924133, "branch_appointment": -3.6813480854034424, "fee_refund": -3.973205327987671, "direct_deposit": -1.383711814880371, "close_account": -3.5613653659820557, "new_account": -2.8370516300201416, "limit_increase": -4.2202301025390625, "other": -1.6408064365386963 }, "probabilities": { "balance": 0.030214896698252558, "transfer": 0.024688435355821, "card_lost": 0.03509678821046568, "dispute_charge": 0.05078850769055846, "loan_payment": 0.023416872004228286, "mortgage": 0.009431106329097105, "wire": 0.013005769562198453, "statement": 0.020800108948551024, "fraud_alert": 0.3285728566283128, "branch_appointment": 0.01980343567132295, "fee_refund": 0.014790693994060826, "direct_deposit": 0.1970567408575783, "close_account": 0.022327925527065202, "new_account": 0.046069535517092614, "limit_increase": 0.011553326732298542, "other": 0.15238300027309623 } }, { "domain": "banking_intent", "row": 30, "task": "intent", "multi_label": false, "gold": [ "limit_increase" ], "selected": [ "mortgage" ], "correct": false, "logits": { "balance": -5.6510725021362305, "transfer": -8.837601661682129, "card_lost": -11.165624618530273, "dispute_charge": -8.378677368164062, "loan_payment": -5.721194267272949, "mortgage": 2.6398823261260986, "wire": -9.96948528289795, "statement": -7.708950042724609, "fraud_alert": -8.894164085388184, "branch_appointment": -6.358283042907715, "fee_refund": -10.002017974853516, "direct_deposit": -5.656504154205322, "close_account": -9.030923843383789, "new_account": -5.844428062438965, "limit_increase": -2.440220594406128, "other": -4.760560989379883 }, "probabilities": { "balance": 0.00024878974406438293, "transfer": 1.0278753305238938e-05, "card_lost": 1.002058137077723e-06, "dispute_charge": 1.6264800144325782e-05, "loan_payment": 0.00023194177568004574, "mortgage": 0.9920839073097028, "wire": 3.3141307022406513e-06, "statement": 3.177661359915195e-05, "fraud_alert": 9.713498879650427e-06, "branch_appointment": 0.0001226577059654292, "fee_refund": 3.2080480423657446e-06, "direct_deposit": 0.0002474420681030832, "close_account": 8.471916880451263e-06, "new_account": 0.00020504973882150382, "limit_increase": 0.006170036583518526, "other": 0.0006061452544538411 } }, { "domain": "banking_intent", "row": 40, "task": "intent", "multi_label": false, "gold": [ "statement" ], "selected": [ "statement" ], "correct": true, "logits": { "balance": -6.8536882400512695, "transfer": -6.048800468444824, "card_lost": -6.092396259307861, "dispute_charge": -7.452272891998291, "loan_payment": -5.25129508972168, "mortgage": -5.829697608947754, "wire": -6.6030731201171875, "statement": 3.3296892642974854, "fraud_alert": -6.857679843902588, "branch_appointment": -6.998213768005371, "fee_refund": -7.750064849853516, "direct_deposit": -6.510276794433594, "close_account": -8.307944297790527, "new_account": -6.058424472808838, "limit_increase": -8.25226879119873, "other": -3.9024462699890137 }, "probabilities": { "balance": 3.773563573245197e-05, "transfer": 8.439369242349676e-05, "card_lost": 8.079352883365311e-05, "dispute_charge": 2.0739088250972677e-05, "loan_payment": 0.0001873536567280871, "mortgage": 0.0001050667144375343, "wire": 4.848332928715401e-05, "statement": 0.9984730063114984, "fraud_alert": 3.758531024302502e-05, "branch_appointment": 3.265765762347869e-05, "fee_refund": 1.5397856063646462e-05, "direct_deposit": 5.3197762569645795e-05, "close_account": 8.814065829587152e-06, "new_account": 8.358538298499003e-05, "limit_increase": 9.318711254633858e-06, "other": 0.0007218712962392799 } }, { "domain": "banking_intent", "row": 50, "task": "intent", "multi_label": false, "gold": [ "loan_payment" ], "selected": [ "loan_payment" ], "correct": true, "logits": { "balance": -4.796566009521484, "transfer": -6.1569671630859375, "card_lost": -5.479534149169922, "dispute_charge": -4.292156219482422, "loan_payment": 2.1639585494995117, "mortgage": -5.218538284301758, "wire": -5.838971138000488, "statement": -4.669269561767578, "fraud_alert": -4.2588934898376465, "branch_appointment": -5.8811798095703125, "fee_refund": -6.027160167694092, "direct_deposit": -5.094327449798584, "close_account": -5.88275671005249, "new_account": -5.056911468505859, "limit_increase": -3.9650096893310547, "other": -3.112818479537964 }, "probabilities": { "balance": 0.0009331633751540781, "transfer": 0.00023941037729939178, "card_lost": 0.00047135529582306067, "dispute_charge": 0.0015453258649279177, "loan_payment": 0.9837281346003323, "mortgage": 0.0006119239548947196, "wire": 0.00032903863134877494, "statement": 0.00105984371824451, "fraud_alert": 0.0015975920606608213, "branch_appointment": 0.0003154393701445221, "fee_refund": 0.0002725947158171835, "direct_deposit": 0.0006928536908510862, "close_account": 0.0003149423456317327, "new_account": 0.0007192685795090044, "limit_increase": 0.0021433742381493552, "other": 0.005025739181211552 } }, { "domain": "banking_intent", "row": 60, "task": "intent", "multi_label": false, "gold": [ "card_lost" ], "selected": [ "mortgage" ], "correct": false, "logits": { "balance": -3.536895513534546, "transfer": -6.466686248779297, "card_lost": -1.9465081691741943, "dispute_charge": -1.0361719131469727, "loan_payment": -5.154088973999023, "mortgage": 1.2098476886749268, "wire": -6.683015823364258, "statement": -5.64157772064209, "fraud_alert": -3.409682512283325, "branch_appointment": -5.761788368225098, "fee_refund": -4.401362419128418, "direct_deposit": -6.284785270690918, "close_account": -5.875317096710205, "new_account": -5.463790416717529, "limit_increase": -4.495704650878906, "other": -4.3624467849731445 }, "probabilities": { "balance": 0.007325219077226527, "transfer": 0.0003912268639168601, "card_lost": 0.03593495170517586, "dispute_charge": 0.0893040342288746, "loan_payment": 0.0014537255441975404, "mortgage": 0.843927225206102, "wire": 0.0003151214221711104, "statement": 0.0008928299475651955, "fraud_alert": 0.00831895015735291, "branch_appointment": 0.000791702339258352, "fee_refund": 0.0030859395244815313, "direct_deposit": 0.0004692748281904347, "close_account": 0.0007067356968268949, "new_account": 0.0010665489540040087, "limit_increase": 0.002808116363408718, "other": 0.003208398141247533 } }, { "domain": "banking_intent", "row": 70, "task": "intent", "multi_label": false, "gold": [ "wire" ], "selected": [ "mortgage" ], "correct": false, "logits": { "balance": -5.431288719177246, "transfer": -2.195232629776001, "card_lost": -4.087642192840576, "dispute_charge": -7.328832626342773, "loan_payment": -1.4523470401763916, "mortgage": 0.07285527884960175, "wire": -3.1050918102264404, "statement": -5.8828125, "fraud_alert": -6.445073127746582, "branch_appointment": -4.536067008972168, "fee_refund": -6.696316719055176, "direct_deposit": -4.002618789672852, "close_account": -6.562131404876709, "new_account": -3.428008794784546, "limit_increase": -7.417348861694336, "other": -3.25258731842041 }, "probabilities": { "balance": 0.00274397768440211, "transfer": 0.06978818085636951, "card_lost": 0.010517653211412445, "dispute_charge": 0.00041142220464783806, "loan_payment": 0.14669421590554535, "mortgage": 0.674217376077754, "wire": 0.028095389447461146, "statement": 0.0017469733852680945, "fraud_alert": 0.0009956337287548587, "branch_appointment": 0.0067169238380104315, "fee_refund": 0.0007744366458358275, "direct_deposit": 0.011451016539804765, "close_account": 0.0008856494083046227, "new_account": 0.020342016019602786, "limit_increase": 0.00037656990994552295, "other": 0.024242565136880678 } }, { "domain": "banking_intent", "row": 80, "task": "intent", "multi_label": false, "gold": [ "other" ], "selected": [ "wire" ], "correct": false, "logits": { "balance": -4.9221510887146, "transfer": -4.523012638092041, "card_lost": -6.727715492248535, "dispute_charge": -6.503632545471191, "loan_payment": -5.705953598022461, "mortgage": -6.495846748352051, "wire": 2.291991710662842, "statement": -4.1957902908325195, "fraud_alert": -6.53499174118042, "branch_appointment": -6.399553298950195, "fee_refund": -8.243063926696777, "direct_deposit": -6.895543098449707, "close_account": -6.674509048461914, "new_account": -5.333415985107422, "limit_increase": -6.299404144287109, "other": -2.9747772216796875 }, "probabilities": { "balance": 0.0007284371205937546, "transfer": 0.001085764644247595, "card_lost": 0.00011974169784733244, "dispute_charge": 0.00014981779438180633, "loan_payment": 0.0003326526322315324, "mortgage": 0.0001509887980166826, "wire": 0.9895881324886572, "statement": 0.0015060748469014188, "fraud_alert": 0.00014519253016383627, "branch_appointment": 0.00016625106709716535, "fee_refund": 2.631103598575314e-05, "direct_deposit": 0.00010124155535887789, "close_account": 0.00012628526408437092, "new_account": 0.00048281637818841234, "limit_increase": 0.0001837632515567045, "other": 0.005106528894687482 } }, { "domain": "banking_intent", "row": 90, "task": "intent", "multi_label": false, "gold": [ "statement" ], "selected": [ "statement" ], "correct": true, "logits": { "balance": -6.510343074798584, "transfer": -6.308371543884277, "card_lost": -5.109225749969482, "dispute_charge": -6.940816879272461, "loan_payment": -5.581257343292236, "mortgage": -1.5998530387878418, "wire": -0.998862624168396, "statement": 0.37191063165664673, "fraud_alert": -5.977160453796387, "branch_appointment": -6.053424835205078, "fee_refund": -6.8867506980896, "direct_deposit": -5.862928867340088, "close_account": -7.40351676940918, "new_account": -5.630578994750977, "limit_increase": -7.465423583984375, "other": -4.5558366775512695 }, "probabilities": { "balance": 0.00072272333227003, "transfer": 0.0008844783299664764, "card_lost": 0.0029340641049513715, "dispute_charge": 0.0004699153998374642, "loan_payment": 0.0018300749735266526, "mortgage": 0.09807781945778131, "wire": 0.17888652285496082, "statement": 0.7045257702109176, "fraud_alert": 0.0012317720512424297, "branch_appointment": 0.0011413245150039295, "fee_refund": 0.000496021295033771, "direct_deposit": 0.0013808308885093817, "close_account": 0.0002958500647606417, "new_account": 0.0017420024479738957, "limit_increase": 0.00027809032568233086, "other": 0.005102739747581922 } }, { "domain": "benefits_request", "row": 0, "task": "program", "multi_label": false, "gold": [ "unemployment" ], "selected": [ "unemployment" ], "correct": true, "logits": { "unemployment": 6.181461334228516, "health_coverage": -6.080854415893555, "food_assistance": -7.038285732269287, "housing_aid": -6.652946949005127, "childcare": -8.20592975616455, "pension": -7.384925842285156, "disability": -5.990830421447754, "tax_credit": -7.700870990753174, "other": -5.661540985107422 }, "probabilities": { "unemployment": 0.9999756490362517, "health_coverage": 4.726431398652587e-06, "food_assistance": 1.8143715244037723e-06, "housing_aid": 2.667329870345944e-06, "childcare": 5.644492038921257e-07, "pension": 1.282869069547654e-06, "disability": 5.171663779820036e-06, "tax_credit": 9.353391218581737e-07, "other": 7.188509779871683e-06 } }, { "domain": "benefits_request", "row": 0, "task": "asking_status", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.013839732855558395, "no": 0.2416926473379135 }, "probabilities": { "yes": 0.43646226350522216, "no": 0.5635377364947778 } }, { "domain": "benefits_request", "row": 10, "task": "program", "multi_label": false, "gold": [ "unemployment" ], "selected": [ "unemployment" ], "correct": true, "logits": { "unemployment": 5.406867504119873, "health_coverage": -6.736975193023682, "food_assistance": -6.027935981750488, "housing_aid": -3.69118070602417, "childcare": -7.2092156410217285, "pension": -6.145174026489258, "disability": -1.702327013015747, "tax_credit": -6.895175457000732, "other": -2.4784095287323 }, "probabilities": { "unemployment": 0.9986625010692517, "health_coverage": 5.313918267600982e-06, "food_assistance": 1.079808364289555e-05, "housing_aid": 0.00011173432482576959, "childcare": 3.313778299614693e-06, "pension": 9.603528981597215e-06, "disability": 0.0008164597754956401, "tax_credit": 4.536379389838092e-06, "other": 0.0003757391418454452 } }, { "domain": "benefits_request", "row": 10, "task": "asking_status", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 0.41876381635665894, "no": -0.3496713638305664 }, "probabilities": { "yes": 0.6831822946179558, "no": 0.31681770538204435 } }, { "domain": "benefits_request", "row": 20, "task": "program", "multi_label": false, "gold": [ "pension" ], "selected": [ "pension" ], "correct": true, "logits": { "unemployment": -5.47337532043457, "health_coverage": -6.4366350173950195, "food_assistance": -6.330082416534424, "housing_aid": -5.918905258178711, "childcare": -8.208484649658203, "pension": 1.2804992198944092, "disability": -0.024712886661291122, "tax_credit": -6.932394981384277, "other": -3.62312388420105 }, "probabilities": { "unemployment": 0.0009099787420487602, "health_coverage": 0.00034729047790830976, "food_assistance": 0.000386338582897457, "housing_aid": 0.0005828275244213274, "childcare": 5.904571117622453e-05, "pension": 0.7801923565166164, "disability": 0.21152186920624336, "tax_credit": 0.00021153734984399801, "other": 0.005788755888844064 } }, { "domain": "benefits_request", "row": 20, "task": "asking_status", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.6233512163162231, "no": 1.685690999031067 }, "probabilities": { "yes": 0.03526228744730416, "no": 0.9647377125526959 } }, { "domain": "benefits_request", "row": 30, "task": "program", "multi_label": false, "gold": [ "housing_aid" ], "selected": [ "housing_aid" ], "correct": true, "logits": { "unemployment": -5.419792175292969, "health_coverage": -5.82101583480835, "food_assistance": -5.773199558258057, "housing_aid": 3.388573169708252, "childcare": -6.662557125091553, "pension": -6.096384525299072, "disability": -5.812511920928955, "tax_credit": -5.164770603179932, "other": -4.0386061668396 }, "probabilities": { "unemployment": 0.00014927403767744436, "health_coverage": 9.993901363088331e-05, "food_assistance": 0.00010483381822346032, "housing_aid": 0.9986395091147223, "childcare": 4.307827656233929e-05, "pension": 7.588290631253605e-05, "disability": 0.00010079251028351882, "tax_credit": 0.0001926365721485632, "other": 0.0005940537504389153 } }, { "domain": "benefits_request", "row": 30, "task": "asking_status", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.08867212384939194, "no": 0.07237450778484344 }, "probabilities": { "yes": 0.459825135911996, "no": 0.540174864088004 } }, { "domain": "benefits_request", "row": 40, "task": "program", "multi_label": false, "gold": [ "housing_aid" ], "selected": [ "housing_aid" ], "correct": true, "logits": { "unemployment": -5.073297023773193, "health_coverage": -7.981906890869141, "food_assistance": -7.254272937774658, "housing_aid": 2.1716105937957764, "childcare": -8.500934600830078, "pension": -2.8347702026367188, "disability": -4.867631435394287, "tax_credit": -7.938043594360352, "other": -3.292217493057251 }, "probabilities": { "unemployment": 0.0007048440674043717, "health_coverage": 3.845030892060812e-05, "food_assistance": 7.959893261172375e-05, "housing_aid": 0.987453034800194, "childcare": 2.2881735600607994e-05, "pension": 0.006611087339584356, "disability": 0.0008657898275439286, "tax_credit": 4.017440200563014e-05, "other": 0.004184138586134833 } }, { "domain": "benefits_request", "row": 40, "task": "asking_status", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.5164826512336731, "no": 0.5896202325820923 }, "probabilities": { "yes": 0.24859814514632425, "no": 0.7514018548536757 } }, { "domain": "benefits_request", "row": 50, "task": "program", "multi_label": false, "gold": [ "tax_credit" ], "selected": [ "tax_credit" ], "correct": true, "logits": { "unemployment": -2.96087646484375, "health_coverage": -7.450588226318359, "food_assistance": -6.878554344177246, "housing_aid": -6.791663646697998, "childcare": -4.103315353393555, "pension": -6.262441635131836, "disability": -7.098989009857178, "tax_credit": 3.562952995300293, "other": -4.410898208618164 }, "probabilities": { "unemployment": 0.0014644694592463863, "health_coverage": 1.643702727398809e-05, "food_assistance": 2.9124228803008326e-05, "housing_aid": 3.176805198262211e-05, "childcare": 0.0004672242912650435, "pension": 5.392979681182188e-05, "disability": 2.3362583969812256e-05, "tax_credit": 0.9975701710070497, "other": 0.0003435135535975695 } }, { "domain": "benefits_request", "row": 50, "task": "asking_status", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 0.1795753538608551, "no": -0.1602126508951187 }, "probabilities": { "yes": 0.5841390258371236, "no": 0.4158609741628764 } }, { "domain": "benefits_request", "row": 60, "task": "program", "multi_label": false, "gold": [ "food_assistance" ], "selected": [ "food_assistance" ], "correct": true, "logits": { "unemployment": -3.91317081451416, "health_coverage": -5.1639814376831055, "food_assistance": 2.347166061401367, "housing_aid": -5.672212600708008, "childcare": -4.660557270050049, "pension": -5.402055263519287, "disability": -5.569876194000244, "tax_credit": -2.733046770095825, "other": -3.5763015747070312 }, "probabilities": { "unemployment": 0.001885372947323043, "health_coverage": 0.0005397306976908229, "food_assistance": 0.9867952067582462, "housing_aid": 0.0003246796955361158, "childcare": 0.000892917753075069, "pension": 0.00042538578276545355, "disability": 0.00035966590211505936, "tax_credit": 0.006136470230265485, "other": 0.0026405702329827677 } }, { "domain": "benefits_request", "row": 60, "task": "asking_status", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 0.8905655741691589, "no": -0.8225129246711731 }, "probabilities": { "yes": 0.8472351534690878, "no": 0.1527648465309122 } }, { "domain": "benefits_request", "row": 70, "task": "program", "multi_label": false, "gold": [ "unemployment" ], "selected": [ "unemployment" ], "correct": true, "logits": { "unemployment": 5.917146682739258, "health_coverage": -8.337200164794922, "food_assistance": -7.900900840759277, "housing_aid": -7.14553165435791, "childcare": -8.325397491455078, "pension": -7.267385959625244, "disability": -8.480632781982422, "tax_credit": -6.190905570983887, "other": -5.192785739898682 }, "probabilities": { "unemployment": 0.9999726670803194, "health_coverage": 6.4476870534411e-07, "food_assistance": 9.974389861782335e-07, "housing_aid": 2.122946299081589e-06, "childcare": 6.524237860973014e-07, "pension": 1.8793962512391951e-06, "disability": 5.586141804758363e-07, "tax_credit": 5.5147756446706285e-06, "other": 1.4962555827421233e-05 } }, { "domain": "benefits_request", "row": 70, "task": "asking_status", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.8270651698112488, "no": 0.8862558603286743 }, "probabilities": { "yes": 0.15273345889441525, "no": 0.8472665411055847 } }, { "domain": "benefits_request", "row": 80, "task": "program", "multi_label": false, "gold": [ "disability" ], "selected": [ "disability" ], "correct": true, "logits": { "unemployment": -0.6847037076950073, "health_coverage": -7.345560550689697, "food_assistance": -7.149491310119629, "housing_aid": -6.963518142700195, "childcare": -8.725296020507812, "pension": -7.87554407119751, "disability": 1.1298435926437378, "tax_credit": -5.074853420257568, "other": -3.823119640350342 }, "probabilities": { "unemployment": 0.13889206027209705, "health_coverage": 0.00017778865664472586, "food_assistance": 0.00021629966046486375, "housing_aid": 0.00026050913574984926, "childcare": 4.473964641070289e-05, "pension": 0.0001046490114018803, "disability": 0.8525606812678216, "tax_credit": 0.0017221049902522245, "other": 0.006021167359157187 } }, { "domain": "benefits_request", "row": 80, "task": "asking_status", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.4379855692386627, "no": 0.4495203495025635 }, "probabilities": { "yes": 0.29162478627396204, "no": 0.7083752137260381 } }, { "domain": "benefits_request", "row": 90, "task": "program", "multi_label": false, "gold": [ "pension" ], "selected": [ "pension" ], "correct": true, "logits": { "unemployment": -5.857782363891602, "health_coverage": -6.748391151428223, "food_assistance": -7.616694927215576, "housing_aid": -5.101473808288574, "childcare": -6.739470958709717, "pension": 4.921709060668945, "disability": -7.649245738983154, "tax_credit": -7.252867221832275, "other": -5.150960922241211 }, "probabilities": { "unemployment": 2.0819339908482324e-05, "health_coverage": 8.544378407152509e-06, "food_assistance": 3.5857576644922064e-06, "housing_aid": 4.4353468520626233e-05, "childcare": 8.620936859634703e-06, "pension": 0.9998632339499743, "disability": 3.470917550946501e-06, "tax_credit": 5.159282400024589e-06, "other": 4.2211968714246697e-05 } }, { "domain": "benefits_request", "row": 90, "task": "asking_status", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 0.31690290570259094, "no": -0.3793753385543823 }, "probabilities": { "yes": 0.6673620951260663, "no": 0.33263790487393374 } }, { "domain": "clinic_request", "row": 0, "task": "request", "multi_label": false, "gold": [ "other" ], "selected": [ "provider_message" ], "correct": false, "logits": { "appointment": -1.554984211921692, "refill": -5.868758201599121, "lab_results": -6.401317596435547, "referral": -1.7529189586639404, "billing": -8.221247673034668, "insurance": -7.572171211242676, "records": -4.076618671417236, "symptom_question": -1.006243109703064, "cancel_visit": -3.916355609893799, "provider_message": 0.2754872739315033, "other": -1.8371514081954956 }, "probabilities": { "appointment": 0.09309219872224202, "refill": 0.0012458479675265107, "lab_results": 0.0007314378691566035, "referral": 0.07637501695011188, "billing": 0.00011852005704730518, "insurance": 0.00022682043423385008, "records": 0.007477929334103151, "symptom_question": 0.16114933608333243, "cancel_visit": 0.00877774014808512, "provider_message": 0.580599895741304, "other": 0.07020525669285711 } }, { "domain": "clinic_request", "row": 0, "task": "urgent", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -1.819415807723999, "no": 1.9017174243927002 }, "probabilities": { "yes": 0.023634413764172583, "no": 0.9763655862358275 } }, { "domain": "clinic_request", "row": 10, "task": "request", "multi_label": false, "gold": [ "provider_message" ], "selected": [ "cancel_visit" ], "correct": false, "logits": { "appointment": -0.46141311526298523, "refill": -3.4515857696533203, "lab_results": -4.357808589935303, "referral": -3.644169807434082, "billing": -4.5126495361328125, "insurance": -5.052490711212158, "records": -3.9081335067749023, "symptom_question": -1.9991967678070068, "cancel_visit": 0.8725743293762207, "provider_message": -2.174492597579956, "other": -2.085198402404785 }, "probabilities": { "appointment": 0.17985853244477087, "refill": 0.009043063110173936, "lab_results": 0.003653827090943308, "referral": 0.007458944519255292, "billing": 0.0031296907262097357, "insurance": 0.0018241114925271593, "records": 0.005728479968380154, "symptom_question": 0.038643823583701394, "cancel_visit": 0.6827700016642689, "provider_message": 0.03243023376880226, "other": 0.03545929163096692 } }, { "domain": "clinic_request", "row": 10, "task": "urgent", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.0356738567352295, "no": 1.152518630027771 }, "probabilities": { "yes": 0.10081583006582236, "no": 0.8991841699341777 } }, { "domain": "clinic_request", "row": 20, "task": "request", "multi_label": false, "gold": [ "provider_message" ], "selected": [ "records" ], "correct": false, "logits": { "appointment": -2.6483278274536133, "refill": -0.9516804218292236, "lab_results": -6.325307846069336, "referral": -4.645208835601807, "billing": -6.029715538024902, "insurance": -2.092264175415039, "records": 0.4874415397644043, "symptom_question": -2.614018678665161, "cancel_visit": -3.4032602310180664, "provider_message": -2.3899855613708496, "other": -1.265221118927002 }, "probabilities": { "appointment": 0.026186369293120803, "refill": 0.14286304227848748, "lab_results": 0.0006624958407752704, "referral": 0.0035550104801404976, "billing": 0.0008903428273360965, "insurance": 0.045663654331373836, "records": 0.6024522364479131, "symptom_question": 0.027100391335009393, "cancel_visit": 0.012308703525384799, "provider_message": 0.03390563705876617, "other": 0.10441211658169254 } }, { "domain": "clinic_request", "row": 20, "task": "urgent", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -2.7030749320983887, "no": 2.76936411857605 }, "probabilities": { "yes": 0.004183398918412227, "no": 0.9958166010815878 } }, { "domain": "clinic_request", "row": 30, "task": "request", "multi_label": false, "gold": [ "provider_message" ], "selected": [ "symptom_question" ], "correct": false, "logits": { "appointment": -4.46229887008667, "refill": -5.398127555847168, "lab_results": -4.908976078033447, "referral": -5.090249061584473, "billing": -8.462296485900879, "insurance": -8.17043685913086, "records": -4.277422904968262, "symptom_question": 0.019065462052822113, "cancel_visit": -4.749588966369629, "provider_message": -0.4816425144672394, "other": -1.204653024673462 }, "probabilities": { "appointment": 0.0057985303891534695, "refill": 0.002274535415325841, "lab_results": 0.0037096120273722058, "referral": 0.0030945865561406733, "billing": 0.00010620404190295204, "insurance": 0.00014219819077734838, "records": 0.006976033364733759, "symptom_question": 0.5123299424840676, "cancel_visit": 0.004350602790297702, "provider_message": 0.3105238965372987, "other": 0.15069385820292977 } }, { "domain": "clinic_request", "row": 30, "task": "urgent", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.467948317527771, "no": 0.5265734195709229 }, "probabilities": { "yes": 0.2700198756396959, "no": 0.729980124360304 } }, { "domain": "clinic_request", "row": 40, "task": "request", "multi_label": false, "gold": [ "cancel_visit" ], "selected": [ "other" ], "correct": false, "logits": { "appointment": -0.7182739973068237, "refill": -3.980116605758667, "lab_results": -7.287221908569336, "referral": -0.7486107349395752, "billing": -6.218360900878906, "insurance": -8.110553741455078, "records": -2.9670255184173584, "symptom_question": -1.8173506259918213, "cancel_visit": -2.592625141143799, "provider_message": -2.9802098274230957, "other": 0.2973865270614624 }, "probabilities": { "appointment": 0.18274715199688493, "refill": 0.007002455728627119, "lab_results": 0.00025644414486312555, "referral": 0.17728644835256868, "billing": 0.0007467809404902165, "insurance": 0.00011257042763479594, "records": 0.01928547059039228, "symptom_question": 0.060887438293133034, "cancel_visit": 0.028043373649257373, "provider_message": 0.01903287380283822, "other": 0.5045989920733102 } }, { "domain": "clinic_request", "row": 40, "task": "urgent", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -3.6869423389434814, "no": 3.815537452697754 }, "probabilities": { "yes": 0.0005514103142190372, "no": 0.9994485896857811 } }, { "domain": "clinic_request", "row": 50, "task": "request", "multi_label": false, "gold": [ "referral" ], "selected": [ "referral" ], "correct": true, "logits": { "appointment": -1.2402809858322144, "refill": -4.133367538452148, "lab_results": -4.79049015045166, "referral": 4.3576741218566895, "billing": -7.077660083770752, "insurance": -6.354948043823242, "records": -4.82742977142334, "symptom_question": -3.4816598892211914, "cancel_visit": -6.6771650314331055, "provider_message": -3.026876926422119, "other": -3.7635538578033447 }, "probabilities": { "appointment": 0.003685235273516396, "refill": 0.00020418023245858313, "lab_results": 0.00010583491603762221, "referral": 0.9945491088976756, "billing": 1.0747901765520338e-05, "insurance": 2.2140811746695484e-05, "records": 0.00010199674115033253, "symptom_question": 0.0003917840305414116, "cancel_visit": 1.6041924928634115e-05, "provider_message": 0.0006173855506632696, "other": 0.00029554371951602044 } }, { "domain": "clinic_request", "row": 50, "task": "urgent", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.17397335171699524, "no": 0.07402873039245605 }, "probabilities": { "yes": 0.4383153157331826, "no": 0.5616846842668174 } }, { "domain": "clinic_request", "row": 60, "task": "request", "multi_label": false, "gold": [ "other" ], "selected": [ "symptom_question" ], "correct": false, "logits": { "appointment": -2.296959400177002, "refill": -5.233834743499756, "lab_results": -2.589488983154297, "referral": -4.670869827270508, "billing": -6.3180131912231445, "insurance": -4.481428146362305, "records": -3.5301966667175293, "symptom_question": 1.3954741954803467, "cancel_visit": -4.8945817947387695, "provider_message": -1.4613481760025024, "other": -1.9541717767715454 }, "probabilities": { "appointment": 0.021623350832438045, "refill": 0.001146711687731861, "lab_results": 0.016139088793714937, "referral": 0.002013477568573513, "billing": 0.0003877943897199479, "insurance": 0.0024334379254904335, "records": 0.006299917394236211, "symptom_question": 0.8680136006449612, "cancel_visit": 0.0016098667204882378, "provider_message": 0.04986827415097241, "other": 0.030464479891673266 } }, { "domain": "clinic_request", "row": 60, "task": "urgent", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -1.1609300374984741, "no": 1.2307262420654297 }, "probabilities": { "yes": 0.08381116370285462, "no": 0.9161888362971454 } }, { "domain": "clinic_request", "row": 70, "task": "request", "multi_label": false, "gold": [ "billing" ], "selected": [ "billing" ], "correct": true, "logits": { "appointment": -4.451655864715576, "refill": -6.851769924163818, "lab_results": -3.463505506515503, "referral": -4.985381603240967, "billing": 4.649818420410156, "insurance": -2.093477249145508, "records": -2.8536581993103027, "symptom_question": -2.7074921131134033, "cancel_visit": -8.48861312866211, "provider_message": -3.3464126586914062, "other": -3.615715265274048 }, "probabilities": { "appointment": 0.0001111179140965398, "refill": 1.007924004250318e-05, "lab_results": 0.00029849174718844676, "referral": 6.516132956304684e-05, "billing": 0.9965615787578268, "insurance": 0.0011747028984357025, "records": 0.0005492697078725518, "symptom_question": 0.0006357183874436197, "cancel_visit": 1.9613532382666654e-06, "provider_message": 0.000335571528256647, "other": 0.0002563471360359371 } }, { "domain": "clinic_request", "row": 70, "task": "urgent", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -2.4538300037384033, "no": 2.3336777687072754 }, "probabilities": { "yes": 0.008264331429241593, "no": 0.9917356685707585 } }, { "domain": "clinic_request", "row": 80, "task": "request", "multi_label": false, "gold": [ "records" ], "selected": [ "records" ], "correct": true, "logits": { "appointment": -1.4093027114868164, "refill": -5.621933460235596, "lab_results": -8.048778533935547, "referral": 0.5253973603248596, "billing": -7.904317855834961, "insurance": -8.980527877807617, "records": 0.9046927690505981, "symptom_question": -4.966005802154541, "cancel_visit": -5.269042015075684, "provider_message": -2.2558584213256836, "other": -1.930720329284668 }, "probabilities": { "appointment": 0.052282004219354306, "refill": 0.0007741585940770034, "lab_results": 6.836983241732362e-05, "referral": 0.3618943419248666, "billing": 7.899561613056406e-05, "insurance": 2.6928422759131396e-05, "records": 0.528819786096406, "symptom_question": 0.0014917488158304266, "cancel_visit": 0.0011017644285532995, "provider_message": 0.02242320845963652, "other": 0.03103869358996889 } }, { "domain": "clinic_request", "row": 80, "task": "urgent", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.352353811264038, "no": 1.367552399635315 }, "probabilities": { "yes": 0.061808904735345, "no": 0.938191095264655 } }, { "domain": "clinic_request", "row": 90, "task": "request", "multi_label": false, "gold": [ "symptom_question" ], "selected": [ "lab_results" ], "correct": false, "logits": { "appointment": -4.495497226715088, "refill": -0.7337647676467896, "lab_results": 0.2252475619316101, "referral": -5.430910110473633, "billing": -7.057960033416748, "insurance": -5.960830211639404, "records": -4.191606044769287, "symptom_question": -0.47535043954849243, "cancel_visit": -5.50755500793457, "provider_message": -3.688472270965576, "other": -1.4241927862167358 }, "probabilities": { "appointment": 0.004197899252819613, "refill": 0.1806057865788191, "lab_results": 0.47122185408795186, "referral": 0.0016473556047743915, "billing": 0.00032371927160217465, "insurance": 0.0009697191721339059, "records": 0.005688663894910332, "symptom_question": 0.2338619559460925, "cancel_visit": 0.0015258115631132285, "provider_message": 0.009408458997897972, "other": 0.09054877562988492 } }, { "domain": "clinic_request", "row": 90, "task": "urgent", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -2.838996648788452, "no": 2.912750244140625 }, "probabilities": { "yes": 0.0031671628815402933, "no": 0.9968328371184597 } }, { "domain": "document_type", "row": 0, "task": "doc_type", "multi_label": false, "gold": [ "resume" ], "selected": [ "resume" ], "correct": true, "logits": { "invoice": -3.7250711917877197, "receipt": -3.075928211212158, "contract": -3.677985429763794, "resume": -0.7708860635757446, "bank_statement": -2.4320929050445557, "clinical_note": -1.9252147674560547, "syllabus": -3.245715618133545, "news_article": -2.5456581115722656, "lease": -3.2199370861053467, "research_abstract": -2.635937452316284, "recipe": -3.821949005126953, "sports_recap": -2.7633581161499023 }, "probabilities": { "invoice": 0.02180509524431054, "receipt": 0.04173276909689279, "contract": 0.022856360359310712, "resume": 0.41835434866994253, "bank_statement": 0.07944952831063246, "clinical_note": 0.13189420099847143, "syllabus": 0.03521595485869846, "news_article": 0.07092030270833112, "lease": 0.0361355727528117, "research_abstract": 0.06479817301578139, "recipe": 0.01979176329501445, "sports_recap": 0.0570459306898024 } }, { "domain": "document_type", "row": 10, "task": "doc_type", "multi_label": false, "gold": [ "lease" ], "selected": [ "lease" ], "correct": true, "logits": { "invoice": -6.5069169998168945, "receipt": -5.813621520996094, "contract": -5.54586935043335, "resume": -4.562626838684082, "bank_statement": -6.560192108154297, "clinical_note": -4.923052787780762, "syllabus": -5.648794174194336, "news_article": -5.030187606811523, "lease": 2.6681132316589355, "research_abstract": -5.132631301879883, "recipe": -5.441455364227295, "sports_recap": -4.675322532653809 }, "probabilities": { "invoice": 0.00010318511012805023, "receipt": 0.00020640082687025453, "contract": 0.0002697706653081091, "resume": 0.0007211266077418131, "bank_statement": 9.783177799092051e-05, "clinical_note": 0.0005028987074436203, "syllabus": 0.00024338568996157481, "news_article": 0.0004518064833333184, "lease": 0.9960520481785419, "research_abstract": 0.0004078136210449951, "recipe": 0.00029946160026249964, "sports_recap": 0.0006442707313731039 } }, { "domain": "document_type", "row": 20, "task": "doc_type", "multi_label": false, "gold": [ "recipe" ], "selected": [ "research_abstract" ], "correct": false, "logits": { "invoice": -3.1685709953308105, "receipt": -3.168768882751465, "contract": -3.682706356048584, "resume": -3.035874843597412, "bank_statement": -2.9266514778137207, "clinical_note": -2.4806466102600098, "syllabus": -3.3263442516326904, "news_article": -2.313873052597046, "lease": -4.211215019226074, "research_abstract": -1.183956265449524, "recipe": -1.1842308044433594, "sports_recap": -3.031724214553833 }, "probabilities": { "invoice": 0.038084549682098, "receipt": 0.03807701397443009, "contract": 0.02277522492401388, "resume": 0.04348886078811123, "bank_statement": 0.04850797402177836, "clinical_note": 0.07577232078424322, "syllabus": 0.032525858639841726, "news_article": 0.0895239889624162, "lease": 0.013425617771522545, "research_abstract": 0.27711245807106094, "recipe": 0.27703639033790006, "sports_recap": 0.043669742042583753 } }, { "domain": "document_type", "row": 30, "task": "doc_type", "multi_label": false, "gold": [ "syllabus" ], "selected": [ "syllabus" ], "correct": true, "logits": { "invoice": -4.979419708251953, "receipt": -4.9457783699035645, "contract": -4.938449859619141, "resume": -4.510499477386475, "bank_statement": -4.699284553527832, "clinical_note": -4.7235517501831055, "syllabus": 2.6496386528015137, "news_article": -4.497910022735596, "lease": -5.070171356201172, "research_abstract": -4.8895721435546875, "recipe": -4.8780317306518555, "sports_recap": -4.6557440757751465 }, "probabilities": { "invoice": 0.0004829714141216664, "receipt": 0.0004994956084891295, "contract": 0.0005031696132404502, "resume": 0.0007719175152254511, "bank_statement": 0.0006391202508152483, "clinical_note": 0.0006237972688743149, "syllabus": 0.9935263161442155, "news_article": 0.0007816969655857021, "lease": 0.00044107098223289734, "research_abstract": 0.0005283743520532515, "recipe": 0.0005345073307300716, "sports_recap": 0.0006675625544162988 } }, { "domain": "document_type", "row": 40, "task": "doc_type", "multi_label": false, "gold": [ "research_abstract" ], "selected": [ "news_article" ], "correct": false, "logits": { "invoice": -5.489402770996094, "receipt": -5.270121097564697, "contract": -6.28785514831543, "resume": -3.6640872955322266, "bank_statement": -5.304237365722656, "clinical_note": -3.5591349601745605, "syllabus": -5.191082954406738, "news_article": 2.179628372192383, "lease": -6.140591144561768, "research_abstract": -3.8490238189697266, "recipe": -5.708310127258301, "sports_recap": -4.18023157119751 }, "probabilities": { "invoice": 0.0004609285852725243, "receipt": 0.0005739399601385236, "contract": 0.0002074293378177275, "resume": 0.0028599476338950565, "bank_statement": 0.0005546895143652381, "clinical_note": 0.0031764227753651006, "syllabus": 0.0006211439974125373, "news_article": 0.9868509181278943, "lease": 0.00024034003780708886, "research_abstract": 0.0023770656132880777, "recipe": 0.0003703082482183919, "sports_recap": 0.0017068661685254857 } }, { "domain": "document_type", "row": 50, "task": "doc_type", "multi_label": false, "gold": [ "clinical_note" ], "selected": [ "clinical_note" ], "correct": true, "logits": { "invoice": -3.83916974067688, "receipt": -3.6363377571105957, "contract": -4.873356342315674, "resume": -4.109942436218262, "bank_statement": -3.3195791244506836, "clinical_note": 1.1163945198059082, "syllabus": -4.6104350090026855, "news_article": -2.9144935607910156, "lease": -5.046952247619629, "research_abstract": -3.9398691654205322, "recipe": -5.163342475891113, "sports_recap": -2.9458577632904053 }, "probabilities": { "invoice": 0.006498444749480394, "receipt": 0.007959728271502467, "contract": 0.0023102974380899825, "resume": 0.004956947775214681, "bank_statement": 0.010926089869058688, "clinical_note": 0.9225366825852922, "syllabus": 0.003005060228762126, "news_article": 0.016382915578768455, "lease": 0.0019421204119072517, "research_abstract": 0.005875924763626095, "recipe": 0.001728735374405286, "sports_recap": 0.01587705295389248 } }, { "domain": "document_type", "row": 60, "task": "doc_type", "multi_label": false, "gold": [ "bank_statement" ], "selected": [ "bank_statement" ], "correct": true, "logits": { "invoice": -2.0594074726104736, "receipt": -1.2619324922561646, "contract": -3.406510829925537, "resume": -3.2280163764953613, "bank_statement": -1.0240315198898315, "clinical_note": -2.3052072525024414, "syllabus": -3.444019079208374, "news_article": -3.2392072677612305, "lease": -3.5597338676452637, "research_abstract": -2.584282398223877, "recipe": -3.745607376098633, "sports_recap": -3.056366205215454 }, "probabilities": { "invoice": 0.10734643179186806, "receipt": 0.23830140144836778, "contract": 0.027909243088140592, "resume": 0.033363162790780256, "bank_statement": 0.3023052406256565, "clinical_note": 0.08395336825431714, "syllabus": 0.02688180538877996, "news_article": 0.03299188062767904, "lease": 0.023944409904995755, "research_abstract": 0.06350930046217805, "recipe": 0.019882926154564853, "sports_recap": 0.039610829462672005 } }, { "domain": "document_type", "row": 70, "task": "doc_type", "multi_label": false, "gold": [ "receipt" ], "selected": [ "receipt" ], "correct": true, "logits": { "invoice": -7.048818588256836, "receipt": 4.925084114074707, "contract": -8.00182056427002, "resume": -7.2457733154296875, "bank_statement": -6.937615394592285, "clinical_note": -6.93604040145874, "syllabus": -6.175173759460449, "news_article": -5.27533483505249, "lease": -8.392218589782715, "research_abstract": -7.056257247924805, "recipe": -6.8412933349609375, "sports_recap": -6.939754486083984 }, "probabilities": { "invoice": 6.306020877158669e-06, "receipt": 0.9998970214689898, "contract": 2.431486736019324e-06, "resume": 5.178679725961755e-06, "bank_statement": 7.047747522818304e-06, "clinical_note": 7.058856422697819e-06, "syllabus": 1.5106871410274904e-05, "news_article": 3.715092333161695e-05, "lease": 1.6455996888443513e-06, "research_abstract": 6.259286569796172e-06, "recipe": 7.760370866244695e-06, "sports_recap": 7.032687858798576e-06 } }, { "domain": "document_type", "row": 80, "task": "doc_type", "multi_label": false, "gold": [ "receipt" ], "selected": [ "resume" ], "correct": false, "logits": { "invoice": -2.7480809688568115, "receipt": -2.546323299407959, "contract": -2.268900156021118, "resume": -1.9585072994232178, "bank_statement": -2.008423328399658, "clinical_note": -2.042346954345703, "syllabus": -2.6906614303588867, "news_article": -2.2098546028137207, "lease": -2.881026029586792, "research_abstract": -2.5340919494628906, "recipe": -2.744832754135132, "sports_recap": -2.744662284851074 }, "probabilities": { "invoice": 0.058636661440939375, "receipt": 0.07174497320114633, "contract": 0.09468361220673371, "resume": 0.1291447401781197, "bank_statement": 0.12285659283765171, "clinical_note": 0.11875875153467304, "syllabus": 0.06210209109004865, "news_article": 0.10044260691143761, "lease": 0.051337170558543505, "research_abstract": 0.07262789975946506, "recipe": 0.058827435577805405, "sports_recap": 0.05883746470343594 } }, { "domain": "document_type", "row": 90, "task": "doc_type", "multi_label": false, "gold": [ "bank_statement" ], "selected": [ "bank_statement" ], "correct": true, "logits": { "invoice": -2.7306032180786133, "receipt": -3.084547281265259, "contract": -3.329408884048462, "resume": -2.649756669998169, "bank_statement": -0.5615575909614563, "clinical_note": -2.884629726409912, "syllabus": -2.823819160461426, "news_article": -2.8093783855438232, "lease": -3.3329505920410156, "research_abstract": -2.747401475906372, "recipe": -3.4279396533966064, "sports_recap": -1.3441824913024902 }, "probabilities": { "invoice": 0.04806007542963324, "receipt": 0.033734050795799955, "contract": 0.026407449115724953, "resume": 0.0521069505575869, "bank_statement": 0.42052226501321555, "clinical_note": 0.0411994658499593, "syllabus": 0.04378257282123485, "news_article": 0.044419414275719764, "lease": 0.026314087070272194, "research_abstract": 0.047259492912816815, "recipe": 0.023929580332684462, "sports_recap": 0.192264595825352 } }, { "domain": "email_triage", "row": 0, "task": "category", "multi_label": false, "gold": [ "calendar" ], "selected": [ "support" ], "correct": false, "logits": { "billing": -2.89577317237854, "support": 0.8857744336128235, "sales": -5.484638690948486, "legal": -6.591725826263428, "recruiting": -5.900564193725586, "calendar": -0.839963972568512, "newsletter": -5.728246212005615, "security": -6.495951175689697 }, "probabilities": { "billing": 0.018891904423576376, "support": 0.8290505239349012, "sales": 0.001418880207653565, "legal": 0.0004689687419933027, "recruiting": 0.0009360770117101707, "calendar": 0.1476054289119238, "newsletter": 0.001112111491896118, "security": 0.0005161052763455552 } }, { "domain": "email_triage", "row": 0, "task": "action", "multi_label": false, "gold": [ "escalate" ], "selected": [ "reply" ], "correct": false, "logits": { "reply": 1.728569507598877, "approve": -2.9683830738067627, "fyi": -1.3275612592697144, "escalate": -3.4659032821655273 }, "probabilities": { "reply": 0.9418504874992281, "approve": 0.008592536369363775, "fyi": 0.04433239956805044, "escalate": 0.00522457656335771 } }, { "domain": "email_triage", "row": 0, "task": "needs_reply", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 2.309201240539551, "no": -2.1786556243896484 }, "probabilities": { "yes": 0.9888803206947427, "no": 0.011119679305257221 } }, { "domain": "email_triage", "row": 0, "task": "is_phishing", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.8848665952682495, "no": 0.9666911363601685 }, "probabilities": { "yes": 0.13569010519415328, "no": 0.8643098948058467 } }, { "domain": "email_triage", "row": 10, "task": "category", "multi_label": false, "gold": [ "calendar" ], "selected": [ "calendar" ], "correct": true, "logits": { "billing": -2.1154823303222656, "support": -2.5346174240112305, "sales": -3.0950393676757812, "legal": -3.0313785076141357, "recruiting": -2.611849784851074, "calendar": 0.40290096402168274, "newsletter": -3.0928304195404053, "security": -2.837083339691162 }, "probabilities": { "billing": 0.06130159476356287, "support": 0.040312869670448746, "sales": 0.02301736246599608, "legal": 0.024530314544326855, "recruiting": 0.03731660507246587, "calendar": 0.7606620349149578, "newsletter": 0.023068262823324172, "security": 0.029790955744917618 } }, { "domain": "email_triage", "row": 10, "task": "action", "multi_label": false, "gold": [ "approve" ], "selected": [ "approve" ], "correct": true, "logits": { "reply": -2.9633238315582275, "approve": 2.8134188652038574, "fyi": -1.9040493965148926, "escalate": -4.219578266143799 }, "probabilities": { "reply": 0.003059270425729091, "approve": 0.9872459139465566, "fyi": 0.008823784852897101, "escalate": 0.0008710307748173379 } }, { "domain": "email_triage", "row": 10, "task": "needs_reply", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 0.20489415526390076, "no": -0.05513251945376396 }, "probabilities": { "yes": 0.5646428490281643, "no": 0.4353571509718357 } }, { "domain": "email_triage", "row": 10, "task": "is_phishing", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.9694071412086487, "no": 1.0326260328292847 }, "probabilities": { "yes": 0.11898961701004812, "no": 0.8810103829899519 } }, { "domain": "email_triage", "row": 20, "task": "category", "multi_label": false, "gold": [ "newsletter" ], "selected": [ "newsletter" ], "correct": true, "logits": { "billing": -7.269685745239258, "support": -3.007955312728882, "sales": -6.071320533752441, "legal": -5.4139723777771, "recruiting": -6.134468078613281, "calendar": -5.4102935791015625, "newsletter": 3.062192440032959, "security": -4.888043403625488 }, "probabilities": { "billing": 3.2469948792083254e-05, "support": 0.002303178585503198, "sales": 0.00010762793365731228, "legal": 0.00020768621865035073, "recruiting": 0.00010104163662009988, "calendar": 0.00020845166152830803, "newsletter": 0.9966881327088103, "security": 0.00035141130643834436 } }, { "domain": "email_triage", "row": 20, "task": "action", "multi_label": false, "gold": [ "reply" ], "selected": [ "reply" ], "correct": true, "logits": { "reply": 1.0650805234909058, "approve": -2.832310914993286, "fyi": -1.212580919265747, "escalate": -2.0393412113189697 }, "probabilities": { "reply": 0.8564071447383357, "approve": 0.017380596858874633, "fyi": 0.0878020155635741, "escalate": 0.03841024283921547 } }, { "domain": "email_triage", "row": 20, "task": "needs_reply", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 1.7065224647521973, "no": -1.573486089706421 }, "probabilities": { "yes": 0.963736582591025, "no": 0.036263417408975034 } }, { "domain": "email_triage", "row": 20, "task": "is_phishing", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": 0.019573889672756195, "no": 0.020024508237838745 }, "probabilities": { "yes": 0.4998873453606356, "no": 0.5001126546393644 } }, { "domain": "email_triage", "row": 30, "task": "category", "multi_label": false, "gold": [ "sales" ], "selected": [ "billing" ], "correct": false, "logits": { "billing": -1.0142654180526733, "support": -1.1812857389450073, "sales": -1.2029560804367065, "legal": -4.519486427307129, "recruiting": -3.1794562339782715, "calendar": -3.2188451290130615, "newsletter": -3.0487184524536133, "security": -2.088526964187622 }, "probabilities": { "billing": 0.2939800008870908, "support": 0.24876070740064868, "sales": 0.24342796775406314, "legal": 0.008831198462089946, "recruiting": 0.033727749468903535, "calendar": 0.03242507456683578, "newsletter": 0.03843846713286483, "security": 0.10040883432750324 } }, { "domain": "email_triage", "row": 30, "task": "action", "multi_label": false, "gold": [ "fyi" ], "selected": [ "fyi" ], "correct": true, "logits": { "reply": -1.5715054273605347, "approve": -2.2517991065979004, "fyi": 2.0699071884155273, "escalate": -3.8001623153686523 }, "probabilities": { "reply": 0.025151016094649228, "approve": 0.012738190637325647, "fyi": 0.9594027091723542, "escalate": 0.002708084095670958 } }, { "domain": "email_triage", "row": 30, "task": "needs_reply", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.568884253501892, "no": 1.7752490043640137 }, "probabilities": { "yes": 0.03408780437694906, "no": 0.9659121956230509 } }, { "domain": "email_triage", "row": 30, "task": "is_phishing", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.859940528869629, "no": 1.88909912109375 }, "probabilities": { "yes": 0.02299893908147551, "no": 0.9770010609185245 } }, { "domain": "email_triage", "row": 40, "task": "category", "multi_label": false, "gold": [ "calendar" ], "selected": [ "security" ], "correct": false, "logits": { "billing": -3.9957008361816406, "support": -1.8618959188461304, "sales": -2.718123435974121, "legal": -4.168544769287109, "recruiting": -2.403585910797119, "calendar": -1.5431973934173584, "newsletter": -3.6225626468658447, "security": 0.09891211241483688 }, "probabilities": { "billing": 0.01088422736276006, "support": 0.09193847723730476, "sales": 0.039051922645070646, "legal": 0.009156561968022694, "recruiting": 0.05348651872609041, "calendar": 0.12644635576144206, "newsletter": 0.015807000312280654, "security": 0.6532289359870288 } }, { "domain": "email_triage", "row": 40, "task": "action", "multi_label": false, "gold": [ "fyi" ], "selected": [ "fyi" ], "correct": true, "logits": { "reply": -0.81154465675354, "approve": -1.4601526260375977, "fyi": 0.9227949976921082, "escalate": -2.5067951679229736 }, "probabilities": { "reply": 0.13565738276911132, "approve": 0.07091801513764916, "fyi": 0.7685242498026708, "escalate": 0.024900352290568657 } }, { "domain": "email_triage", "row": 40, "task": "needs_reply", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 0.49857789278030396, "no": -0.24192047119140625 }, "probabilities": { "yes": 0.6771048251033982, "no": 0.3228951748966018 } }, { "domain": "email_triage", "row": 40, "task": "is_phishing", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -1.076368808746338, "no": 1.1651476621627808 }, "probabilities": { "yes": 0.09608375298777945, "no": 0.9039162470122205 } }, { "domain": "email_triage", "row": 50, "task": "category", "multi_label": false, "gold": [ "sales" ], "selected": [ "legal" ], "correct": false, "logits": { "billing": -5.026220321655273, "support": -3.158890724182129, "sales": -4.027879238128662, "legal": 1.2835900783538818, "recruiting": -4.614371299743652, "calendar": -2.7034664154052734, "newsletter": -2.473069667816162, "security": -3.6945619583129883 }, "probabilities": { "billing": 0.0016993115203862961, "support": 0.010996233207779265, "sales": 0.0046115510996565185, "legal": 0.9345205163670348, "recruiting": 0.0025652917194508565, "calendar": 0.01733932576319475, "newsletter": 0.02183193562751268, "security": 0.006435834694984815 } }, { "domain": "email_triage", "row": 50, "task": "action", "multi_label": false, "gold": [ "fyi" ], "selected": [ "fyi" ], "correct": true, "logits": { "reply": -2.000962495803833, "approve": -2.4509263038635254, "fyi": 1.4315190315246582, "escalate": -2.570450782775879 }, "probabilities": { "reply": 0.030159702933041373, "approve": 0.019231371642716454, "fyi": 0.9335441161405873, "escalate": 0.01706480928365492 } }, { "domain": "email_triage", "row": 50, "task": "needs_reply", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.5439344644546509, "no": 1.7183200120925903 }, "probabilities": { "yes": 0.03688902805954409, "no": 0.9631109719404559 } }, { "domain": "email_triage", "row": 50, "task": "is_phishing", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.7048074007034302, "no": 1.7457106113433838 }, "probabilities": { "yes": 0.030753414885493727, "no": 0.9692465851145062 } }, { "domain": "email_triage", "row": 60, "task": "category", "multi_label": false, "gold": [ "billing" ], "selected": [ "billing" ], "correct": true, "logits": { "billing": 1.0630384683609009, "support": -2.753986358642578, "sales": -2.161346673965454, "legal": -4.560564994812012, "recruiting": -4.56361198425293, "calendar": -4.199149131774902, "newsletter": -3.9771087169647217, "security": -3.430695056915283 }, "probabilities": { "billing": 0.9159009104485204, "support": 0.020143534238856998, "sales": 0.036434751842344314, "legal": 0.0033078707990628936, "recruiting": 0.003297807091457708, "calendar": 0.004747986257281329, "newsletter": 0.0059284393352432695, "security": 0.010238699987233214 } }, { "domain": "email_triage", "row": 60, "task": "action", "multi_label": false, "gold": [ "fyi" ], "selected": [ "fyi" ], "correct": true, "logits": { "reply": -0.461653470993042, "approve": -2.3061320781707764, "fyi": 0.3521992862224579, "escalate": -2.0470712184906006 }, "probabilities": { "reply": 0.2762770358746944, "approve": 0.04368153655177148, "fyi": 0.6234427076950526, "escalate": 0.05659871987848143 } }, { "domain": "email_triage", "row": 60, "task": "needs_reply", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.6459429860115051, "no": 0.8843570351600647 }, "probabilities": { "yes": 0.17794979334923228, "no": 0.8220502066507677 } }, { "domain": "email_triage", "row": 60, "task": "is_phishing", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -1.7140860557556152, "no": 1.7871320247650146 }, "probabilities": { "yes": 0.029277592540817503, "no": 0.9707224074591825 } }, { "domain": "email_triage", "row": 70, "task": "category", "multi_label": false, "gold": [ "recruiting" ], "selected": [ "recruiting" ], "correct": true, "logits": { "billing": -3.6260643005371094, "support": -3.4940147399902344, "sales": -3.8983874320983887, "legal": -3.311235189437866, "recruiting": 1.735058069229126, "calendar": -3.4332501888275146, "newsletter": -4.566754341125488, "security": -3.3318915367126465 }, "probabilities": { "billing": 0.004541704209091082, "support": 0.005182833314333215, "sales": 0.003458998812015688, "legal": 0.006222249075955416, "recruiting": 0.9672187527724347, "calendar": 0.005507531007775778, "newsletter": 0.001772892294210338, "security": 0.006095038514183804 } }, { "domain": "email_triage", "row": 70, "task": "action", "multi_label": false, "gold": [ "reply" ], "selected": [ "reply" ], "correct": true, "logits": { "reply": 0.5363346934318542, "approve": -1.802575707435608, "fyi": -0.6900551915168762, "escalate": -2.3273367881774902 }, "probabilities": { "reply": 0.6911608386325484, "approve": 0.06665047397127176, "fyi": 0.20275182087947874, "escalate": 0.039436866516701155 } }, { "domain": "email_triage", "row": 70, "task": "needs_reply", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 1.4208600521087646, "no": -1.2754311561584473 }, "probabilities": { "yes": 0.9368074415859109, "no": 0.06319255841408918 } }, { "domain": "email_triage", "row": 70, "task": "is_phishing", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.4197630286216736, "no": 0.4661276340484619 }, "probabilities": { "yes": 0.29195857777685785, "no": 0.7080414222231421 } }, { "domain": "email_triage", "row": 80, "task": "category", "multi_label": false, "gold": [ "billing" ], "selected": [ "billing" ], "correct": true, "logits": { "billing": -0.1610155999660492, "support": -2.9109580516815186, "sales": -3.1118862628936768, "legal": -2.7499616146087646, "recruiting": -3.4857187271118164, "calendar": -0.7217138409614563, "newsletter": -3.6087005138397217, "security": -3.0677294731140137 }, "probabilities": { "billing": 0.5306186423602005, "support": 0.03392326709214879, "sales": 0.027748253815596174, "legal": 0.03984900996682916, "recruiting": 0.01909335652117019, "calendar": 0.3028826190509242, "newsletter": 0.016883868875686957, "security": 0.029000982317444052 } }, { "domain": "email_triage", "row": 80, "task": "action", "multi_label": false, "gold": [ "escalate" ], "selected": [ "escalate" ], "correct": true, "logits": { "reply": -5.250533580780029, "approve": -5.332393169403076, "fyi": -3.7545340061187744, "escalate": 5.370587348937988 }, "probabilities": { "reply": 2.439147952943857e-05, "approve": 2.2474341553296935e-05, "fyi": 0.00010887859415265506, "escalate": 0.9998442555847646 } }, { "domain": "email_triage", "row": 80, "task": "needs_reply", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.7261035442352295, "no": 0.9898627996444702 }, "probabilities": { "yes": 0.15239145393347894, "no": 0.847608546066521 } }, { "domain": "email_triage", "row": 80, "task": "is_phishing", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.8989808559417725, "no": 1.0457463264465332 }, "probabilities": { "yes": 0.12512944438725676, "no": 0.8748705556127433 } }, { "domain": "email_triage", "row": 90, "task": "category", "multi_label": false, "gold": [ "newsletter" ], "selected": [ "support" ], "correct": false, "logits": { "billing": -3.2206850051879883, "support": -0.6221319437026978, "sales": -3.2006003856658936, "legal": -2.278224468231201, "recruiting": -2.9434871673583984, "calendar": -1.4507381916046143, "newsletter": -2.777177333831787, "security": -1.6663508415222168 }, "probabilities": { "billing": 0.03173503717272189, "support": 0.42665443744257614, "sales": 0.03237886721775914, "legal": 0.08144124834307039, "recruiting": 0.0418720768832316, "calendar": 0.18630184066825067, "newsletter": 0.04944836607007918, "security": 0.1501681262023109 } }, { "domain": "email_triage", "row": 90, "task": "action", "multi_label": false, "gold": [ "approve" ], "selected": [ "reply" ], "correct": false, "logits": { "reply": -0.10827504098415375, "approve": -1.7018756866455078, "fyi": -0.38211527466773987, "escalate": -1.7330751419067383 }, "probabilities": { "reply": 0.4628349826163612, "approve": 0.09404467505038824, "fyi": 0.3519645104176137, "escalate": 0.09115583191563688 } }, { "domain": "email_triage", "row": 90, "task": "needs_reply", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.21192432940006256, "no": 0.3583466112613678 }, "probabilities": { "yes": 0.36117430905827486, "no": 0.638825690941725 } }, { "domain": "email_triage", "row": 90, "task": "is_phishing", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -1.3624850511550903, "no": 1.4009857177734375 }, "probabilities": { "yes": 0.05933036489351021, "no": 0.9406696351064898 } }, { "domain": "news_topic", "row": 0, "task": "topic", "multi_label": false, "gold": [ "world" ], "selected": [ "culture" ], "correct": false, "logits": { "politics": -4.4911699295043945, "business": -4.215949058532715, "technology": -6.466953754425049, "science": -5.890413761138916, "health": -6.809118270874023, "sports": -5.089170455932617, "entertainment": -1.320483684539795, "world": -2.5787503719329834, "culture": 0.6200532913208008, "weather": -1.5641154050827026 }, "probabilities": { "politics": 0.004576876905762876, "business": 0.006026929868388298, "technology": 0.0006345958383925408, "science": 0.0011294977169012713, "health": 0.00045070985923278284, "sports": 0.002516870691543686, "entertainment": 0.10903872694351162, "world": 0.030982930530766385, "culture": 0.7591808993392954, "weather": 0.08546196230620508 } }, { "domain": "news_topic", "row": 10, "task": "topic", "multi_label": false, "gold": [ "sports" ], "selected": [ "sports" ], "correct": true, "logits": { "politics": -1.4367201328277588, "business": 0.27820128202438354, "technology": -6.789919853210449, "science": -6.231821537017822, "health": -6.44338321685791, "sports": 0.6113623976707458, "entertainment": -3.3913960456848145, "world": -4.34600305557251, "culture": -4.449847221374512, "weather": -5.896869659423828 }, "probabilities": { "politics": 0.06855992876127935, "business": 0.38093534024071957, "technology": 0.00032449294469162013, "science": 0.0005670015911641936, "health": 0.00045888536534745137, "sports": 0.5315465443949482, "entertainment": 0.009708796423557571, "world": 0.003737531297044987, "culture": 0.0033688827010309444, "weather": 0.0007925962802160808 } }, { "domain": "news_topic", "row": 20, "task": "topic", "multi_label": false, "gold": [ "entertainment" ], "selected": [ "entertainment" ], "correct": true, "logits": { "politics": -2.594618797302246, "business": -4.284222602844238, "technology": -4.277697563171387, "science": -2.0581610202789307, "health": -2.045790195465088, "sports": -5.109347820281982, "entertainment": 1.653327465057373, "world": -3.2869067192077637, "culture": -3.27055025100708, "weather": -3.763923168182373 }, "probabilities": { "politics": 0.013127887387435085, "business": 0.00242331144415875, "technology": 0.0024391753474855703, "science": 0.022447888419574142, "health": 0.02272731210185298, "sports": 0.001061846902397828, "entertainment": 0.9184477695232738, "world": 0.006569586243001029, "culture": 0.006677925076468758, "weather": 0.004077297554351985 } }, { "domain": "news_topic", "row": 30, "task": "topic", "multi_label": false, "gold": [ "politics" ], "selected": [ "health" ], "correct": false, "logits": { "politics": -1.4348149299621582, "business": -1.4469753503799438, "technology": -4.922694206237793, "science": -3.8409793376922607, "health": 0.8038437366485596, "sports": -6.062755107879639, "entertainment": -5.367118835449219, "world": -1.4530932903289795, "culture": -4.453902721405029, "weather": -5.795170783996582 }, "probabilities": { "politics": 0.07960351050305628, "business": 0.07864136028074561, "technology": 0.0024331310293322266, "science": 0.007177088403220664, "health": 0.746739846650838, "sports": 0.0007781141958768111, "entertainment": 0.0015601068120619465, "world": 0.07816170590545828, "culture": 0.003888293615871176, "weather": 0.00101684260353901 } }, { "domain": "news_topic", "row": 40, "task": "topic", "multi_label": false, "gold": [ "business" ], "selected": [ "business" ], "correct": true, "logits": { "politics": -1.7284777164459229, "business": 2.7605912685394287, "technology": -6.548844814300537, "science": -9.675893783569336, "health": -9.90114688873291, "sports": -9.846331596374512, "entertainment": -7.710562705993652, "world": -3.155701160430908, "culture": -6.631889820098877, "weather": -7.757523536682129 }, "probabilities": { "politics": 0.011074217259446044, "business": 0.9860318162369943, "technology": 8.930056530031105e-05, "science": 3.915561919170506e-06, "health": 3.1258484295033354e-06, "sports": 3.30197586458523e-06, "entertainment": 2.79464429437446e-05, "world": 0.0026575275368030766, "culture": 8.218417887373163e-05, "weather": 2.6664393425583294e-05 } }, { "domain": "news_topic", "row": 50, "task": "topic", "multi_label": false, "gold": [ "weather" ], "selected": [ "politics" ], "correct": false, "logits": { "politics": 0.46347182989120483, "business": 0.03134285286068916, "technology": -4.692283630371094, "science": -2.062656879425049, "health": -0.28032633662223816, "sports": -5.978346824645996, "entertainment": -4.865292549133301, "world": -2.115712881088257, "culture": -5.2394232749938965, "weather": -0.5946142077445984 }, "probabilities": { "politics": 0.3783722167894935, "business": 0.24561111078218684, "technology": 0.0021817404969674305, "science": 0.030257669886490948, "health": 0.17984228009838066, "sports": 0.000602938400268088, "entertainment": 0.0018351276306891626, "world": 0.028694162378822004, "culture": 0.001262360406236208, "weather": 0.13134039313046525 } }, { "domain": "news_topic", "row": 60, "task": "topic", "multi_label": false, "gold": [ "weather" ], "selected": [ "weather" ], "correct": true, "logits": { "politics": -4.520905494689941, "business": -7.760076999664307, "technology": -5.41467809677124, "science": 0.048960696905851364, "health": -6.29864501953125, "sports": -8.51423168182373, "entertainment": -6.5126752853393555, "world": -1.7746055126190186, "culture": -6.870851516723633, "weather": 1.214159607887268 }, "probabilities": { "politics": 0.0023611816090783316, "business": 9.254971407085567e-05, "technology": 0.0009659816637932627, "science": 0.22792767578133136, "health": 0.0003990863979608004, "sports": 4.353613432437537e-05, "entertainment": 0.00032219200732913584, "world": 0.03679868672148695, "culture": 0.00022519606719422964, "weather": 0.7308639139034306 } }, { "domain": "news_topic", "row": 70, "task": "topic", "multi_label": false, "gold": [ "entertainment" ], "selected": [ "culture" ], "correct": false, "logits": { "politics": -0.6818933486938477, "business": -2.2841343879699707, "technology": -5.922938346862793, "science": -6.314449310302734, "health": -6.885383605957031, "sports": -5.663727283477783, "entertainment": -0.774067759513855, "world": -3.794055938720703, "culture": 0.3319292664527893, "weather": -6.213222980499268 }, "probabilities": { "politics": 0.2026042583695891, "business": 0.04081352701405952, "technology": 0.001072733015435308, "science": 0.0007252046766067466, "health": 0.0004097386969131871, "sports": 0.0013901625398023481, "entertainment": 0.18476415969971874, "world": 0.009016822688345732, "culture": 0.5584009347055573, "weather": 0.0008024585939720128 } }, { "domain": "news_topic", "row": 80, "task": "topic", "multi_label": false, "gold": [ "weather" ], "selected": [ "weather" ], "correct": true, "logits": { "politics": -1.4112129211425781, "business": -4.451791286468506, "technology": -1.3879740238189697, "science": -1.477954626083374, "health": -0.3165713846683502, "sports": -5.515611171722412, "entertainment": -4.811525821685791, "world": -1.0877299308776855, "culture": -4.631607532501221, "weather": 0.832017719745636 }, "probabilities": { "politics": 0.059205481508239295, "business": 0.0028304501576762683, "technology": 0.06059746302782818, "science": 0.055382985503542266, "health": 0.17691257201443125, "sports": 0.0009768871698620876, "entertainment": 0.0019752623601342447, "world": 0.0818179879409862, "culture": 0.0023646251709286705, "weather": 0.5579362851463715 } }, { "domain": "news_topic", "row": 90, "task": "topic", "multi_label": false, "gold": [ "entertainment" ], "selected": [ "entertainment" ], "correct": true, "logits": { "politics": -3.7396554946899414, "business": -5.161128997802734, "technology": -6.0428314208984375, "science": -5.598080635070801, "health": -4.474094867706299, "sports": -4.870988845825195, "entertainment": 2.0747592449188232, "world": -3.3767313957214355, "culture": -1.4536885023117065, "weather": -4.894647121429443 }, "probabilities": { "politics": 0.002865472405056804, "business": 0.0006916050120166013, "technology": 0.00028637798888314425, "science": 0.000446778686498486, "health": 0.0013747801836260396, "sports": 0.0009244094976558572, "entertainment": 0.9602061149696567, "world": 0.004119193220295902, "culture": 0.028182471798992137, "weather": 0.0009027962373183202 } }, { "domain": "paper_field", "row": 0, "task": "field", "multi_label": false, "gold": [ "chemistry" ], "selected": [ "chemistry" ], "correct": true, "logits": { "biology": -2.4740588665008545, "medicine": -4.2532734870910645, "chemistry": -0.6083477139472961, "physics": -1.8198702335357666, "computer_science": -2.940880060195923, "mathematics": -2.545633316040039, "earth_science": -4.095777988433838, "engineering": -0.6251453757286072, "economics": -3.313037872314453, "other": -2.390737533569336 }, "probabilities": { "biology": 0.05213678926146652, "medicine": 0.008799159511248433, "chemistry": 0.3368312220031297, "physics": 0.100289345238068, "computer_science": 0.032689359707123014, "mathematics": 0.04853554314483874, "earth_science": 0.010300080445981989, "engineering": 0.3312205004865465, "economics": 0.02253099285521238, "other": 0.05666700734638476 } }, { "domain": "paper_field", "row": 10, "task": "field", "multi_label": false, "gold": [ "earth_science" ], "selected": [ "economics" ], "correct": false, "logits": { "biology": -5.744741439819336, "medicine": -5.224399089813232, "chemistry": -8.464914321899414, "physics": -7.403064727783203, "computer_science": -6.893973350524902, "mathematics": -5.666950702667236, "earth_science": -4.524603366851807, "engineering": -8.631370544433594, "economics": 4.307960033416748, "other": -6.3074846267700195 }, "probabilities": { "biology": 4.305375844852985e-05, "medicine": 7.244240854812436e-05, "chemistry": 2.8356654868038806e-06, "physics": 8.199935130811386e-06, "computer_science": 1.3642877909882551e-05, "mathematics": 4.653665438777221e-05, "earth_science": 0.0001458512992346584, "engineering": 2.4008442028990035e-06, "economics": 0.9996405112295111, "other": 2.4525327139293255e-05 } }, { "domain": "paper_field", "row": 20, "task": "field", "multi_label": false, "gold": [ "physics" ], "selected": [ "biology" ], "correct": false, "logits": { "biology": -0.34375429153442383, "medicine": -2.9331846237182617, "chemistry": -6.988465785980225, "physics": -1.1581889390945435, "computer_science": -3.2428088188171387, "mathematics": -3.42608904838562, "earth_science": -3.1183688640594482, "engineering": -3.8750874996185303, "economics": -2.1254966259002686, "other": -2.306859016418457 }, "probabilities": { "biology": 0.49490583528121257, "medicine": 0.037149012171214295, "chemistry": 0.0006438149224196716, "physics": 0.21918866986307317, "computer_science": 0.027257071320200808, "mathematics": 0.022692461024470353, "earth_science": 0.030869015314508447, "engineering": 0.014483850999943489, "economics": 0.0833147145100423, "other": 0.069495554592915 } }, { "domain": "paper_field", "row": 30, "task": "field", "multi_label": false, "gold": [ "biology" ], "selected": [ "biology" ], "correct": true, "logits": { "biology": 0.7041404247283936, "medicine": -1.0934617519378662, "chemistry": -3.6416685581207275, "physics": -4.42454719543457, "computer_science": -4.630671977996826, "mathematics": -3.8602561950683594, "earth_science": -4.4900054931640625, "engineering": -3.4201529026031494, "economics": -5.6993560791015625, "other": -2.810960292816162 }, "probabilities": { "biology": 0.7981212313747809, "medicine": 0.1322452731418433, "chemistry": 0.010344464255433545, "physics": 0.0047283338306198875, "computer_science": 0.003847594324431575, "mathematics": 0.00831336021492891, "earth_science": 0.0044287376430841765, "engineering": 0.012909547805956932, "economics": 0.001321495435593316, "other": 0.023739961973327468 } }, { "domain": "paper_field", "row": 40, "task": "field", "multi_label": false, "gold": [ "computer_science" ], "selected": [ "chemistry" ], "correct": false, "logits": { "biology": -4.980285167694092, "medicine": -4.657288551330566, "chemistry": 0.7557374835014343, "physics": -2.8064661026000977, "computer_science": -1.4894213676452637, "mathematics": -2.576845169067383, "earth_science": -4.958426475524902, "engineering": -2.9611990451812744, "economics": -5.00317907333374, "other": -3.2484471797943115 }, "probabilities": { "biology": 0.002631162871771853, "medicine": 0.0036343218101231715, "chemistry": 0.8152122886795232, "physics": 0.023132648395173237, "computer_science": 0.08633971636976724, "mathematics": 0.029103717165368446, "earth_science": 0.002689309844206499, "engineering": 0.019816442602441857, "economics": 0.00257160958208095, "other": 0.014868782679543464 } }, { "domain": "paper_field", "row": 50, "task": "field", "multi_label": false, "gold": [ "economics" ], "selected": [ "computer_science" ], "correct": false, "logits": { "biology": -2.7974796295166016, "medicine": -5.039783954620361, "chemistry": -5.376378536224365, "physics": -2.2177586555480957, "computer_science": 0.153746098279953, "mathematics": -2.6854195594787598, "earth_science": -3.7229933738708496, "engineering": -2.1973345279693604, "economics": -3.3880393505096436, "other": -2.2548940181732178 }, "probabilities": { "biology": 0.03608967564314217, "medicine": 0.0038332097462261093, "chemistry": 0.002737672000666293, "physics": 0.0644395648271904, "computer_science": 0.6903733643277971, "mathematics": 0.04036919124937593, "earth_science": 0.014303340148596251, "engineering": 0.06576921901247017, "economics": 0.01999429754556762, "other": 0.06209046549896794 } }, { "domain": "paper_field", "row": 60, "task": "field", "multi_label": false, "gold": [ "computer_science" ], "selected": [ "medicine" ], "correct": false, "logits": { "biology": -2.098292350769043, "medicine": -1.0257047414779663, "chemistry": -3.1053061485290527, "physics": -1.8247663974761963, "computer_science": -2.215550184249878, "mathematics": -1.8337585926055908, "earth_science": -2.4850528240203857, "engineering": -2.369713544845581, "economics": -3.388606548309326, "other": -2.2818174362182617 }, "probabilities": { "biology": 0.09667368529352653, "medicine": 0.282570717714494, "chemistry": 0.03531569350655383, "physics": 0.12708640271258162, "computer_science": 0.0859773083189341, "mathematics": 0.1259487396909402, "earth_science": 0.06566596566824957, "engineering": 0.0736939012007215, "economics": 0.0266030811190029, "other": 0.08046450477499573 } }, { "domain": "paper_field", "row": 70, "task": "field", "multi_label": false, "gold": [ "other" ], "selected": [ "other" ], "correct": true, "logits": { "biology": -1.3266279697418213, "medicine": -3.3372583389282227, "chemistry": -3.800971031188965, "physics": -3.046759605407715, "computer_science": -2.874542236328125, "mathematics": -2.3531692028045654, "earth_science": -1.5980644226074219, "engineering": -3.0618414878845215, "economics": -2.2624733448028564, "other": -1.040452003479004 }, "probabilities": { "biology": 0.21596745741057302, "medicine": 0.028918958023090798, "chemistry": 0.018188411758930493, "physics": 0.03866737033268088, "computer_science": 0.04593436223773427, "mathematics": 0.0773690247242942, "earth_science": 0.1646284774548051, "engineering": 0.038088569273696575, "economics": 0.08471412570559443, "other": 0.2875232430786002 } }, { "domain": "paper_field", "row": 80, "task": "field", "multi_label": false, "gold": [ "medicine" ], "selected": [ "medicine" ], "correct": true, "logits": { "biology": -3.099961519241333, "medicine": -0.8235671520233154, "chemistry": -3.360417604446411, "physics": -2.813403606414795, "computer_science": -3.0395092964172363, "mathematics": -1.0906496047973633, "earth_science": -3.658932685852051, "engineering": -3.8606033325195312, "economics": -3.3925724029541016, "other": -1.6322026252746582 }, "probabilities": { "biology": 0.036377521365780256, "medicine": 0.3543713611980772, "chemistry": 0.028036155744449866, "physics": 0.048448866936206154, "computer_science": 0.03864445361196248, "mathematics": 0.27131023736392773, "earth_science": 0.020800559229436324, "engineering": 0.017001630066242144, "economics": 0.027148998424958944, "other": 0.1578602160589589 } }, { "domain": "paper_field", "row": 90, "task": "field", "multi_label": false, "gold": [ "medicine" ], "selected": [ "medicine" ], "correct": true, "logits": { "biology": -0.8286880850791931, "medicine": -0.22314561903476715, "chemistry": -5.718775272369385, "physics": -4.21743106842041, "computer_science": -3.366868734359741, "mathematics": -2.424490451812744, "earth_science": -4.123588562011719, "engineering": -5.474828720092773, "economics": -4.7187089920043945, "other": -1.7779312133789062 }, "probabilities": { "biology": 0.27705262397795744, "medicine": 0.5076285261996099, "chemistry": 0.002083648157187345, "physics": 0.009350824143091395, "computer_science": 0.021889932263430688, "mathematics": 0.05617125221882485, "earth_science": 0.010270821278835164, "engineering": 0.0026593103742771395, "economics": 0.0056643183432301, "other": 0.10722874304355604 } }, { "domain": "product_feedback", "row": 0, "task": "feedback_type", "multi_label": false, "gold": [ "bug" ], "selected": [ "bug" ], "correct": true, "logits": { "bug": 0.9414206743240356, "feature_request": -1.4569158554077148, "how_to": -1.6269164085388184, "praise": -3.205777645111084 }, "probabilities": { "bug": 0.8450652657290708, "feature_request": 0.07679022337087808, "how_to": 0.06478517388188257, "praise": 0.013359337018168512 } }, { "domain": "product_feedback", "row": 0, "task": "product_area", "multi_label": true, "gold": [ "notifications" ], "selected": [ "notifications" ], "correct": true, "logits": { "billing": -3.9565165042877197, "mobile": -2.3725128173828125, "search": -3.6694204807281494, "checkout": -3.069188117980957, "shipping": -3.5638554096221924, "notifications": 1.41297447681427, "admin": -4.145140647888184, "integrations": -4.197084426879883 }, "probabilities": { "billing": 0.018770562334085576, "mobile": 0.08529289001078703, "search": 0.024857587640000595, "checkout": 0.04439625913361095, "shipping": 0.027548947537018982, "notifications": 0.8042346728488591, "admin": 0.015594177161909164, "integrations": 0.014816530167929361 } }, { "domain": "product_feedback", "row": 10, "task": "feedback_type", "multi_label": false, "gold": [ "feature_request" ], "selected": [ "praise" ], "correct": false, "logits": { "bug": -3.0034773349761963, "feature_request": -2.379892110824585, "how_to": -2.124906301498413, "praise": 2.740675687789917 }, "probabilities": { "bug": 0.0031482972363779498, "feature_request": 0.005873478048846002, "how_to": 0.007579390445654366, "praise": 0.9833988342691217 } }, { "domain": "product_feedback", "row": 10, "task": "product_area", "multi_label": true, "gold": [ "integrations" ], "selected": [ "integrations" ], "correct": true, "logits": { "billing": -7.589778900146484, "mobile": -7.829875946044922, "search": -6.628497123718262, "checkout": -5.615575790405273, "shipping": -7.7277092933654785, "notifications": -8.61667251586914, "admin": -7.308248519897461, "integrations": 5.668712615966797 }, "probabilities": { "billing": 0.000505337333175926, "mobile": 0.000397516727973315, "search": 0.001320402850636259, "checkout": 0.00362750609909026, "shipping": 0.0004402579960183446, "notifications": 0.0001810289560688031, "admin": 0.000669540912908787, "integrations": 0.9965595705569641 } }, { "domain": "product_feedback", "row": 20, "task": "feedback_type", "multi_label": false, "gold": [ "praise" ], "selected": [ "praise" ], "correct": true, "logits": { "bug": -1.0476818084716797, "feature_request": -4.280501365661621, "how_to": -3.5583155155181885, "praise": 6.37716007232666 }, "probabilities": { "bug": 0.0005958569963723494, "feature_request": 2.350424732640391e-05, "how_to": 4.839357186793876e-05, "praise": 0.9993322451844332 } }, { "domain": "product_feedback", "row": 20, "task": "product_area", "multi_label": true, "gold": [ "billing", "checkout", "mobile", "notifications", "search", "shipping" ], "selected": [ "notifications" ], "correct": false, "logits": { "billing": -4.635317325592041, "mobile": -4.675848007202148, "search": -5.236105918884277, "checkout": -5.553687572479248, "shipping": -6.595872402191162, "notifications": 3.3325204849243164, "admin": -6.150476455688477, "integrations": -6.944016456604004 }, "probabilities": { "billing": 0.009609783536595582, "mobile": 0.00923160379770697, "search": 0.005292774187889757, "checkout": 0.003858204961637522, "shipping": 0.0013641312978630218, "notifications": 0.9655277597999163, "admin": 0.0021279277700727533, "integrations": 0.0009634592350286708 } }, { "domain": "product_feedback", "row": 30, "task": "feedback_type", "multi_label": false, "gold": [ "how_to" ], "selected": [ "how_to" ], "correct": true, "logits": { "bug": -3.1901907920837402, "feature_request": -2.32448148727417, "how_to": 2.6014413833618164, "praise": -5.475304126739502 }, "probabilities": { "bug": 0.003020913877665397, "feature_request": 0.007179779691240269, "how_to": 0.9894918905973332, "praise": 0.00030741583376114407 } }, { "domain": "product_feedback", "row": 30, "task": "product_area", "multi_label": true, "gold": [ "checkout" ], "selected": [ "checkout" ], "correct": true, "logits": { "billing": -4.626077651977539, "mobile": -5.659435749053955, "search": -3.9238789081573486, "checkout": 1.8474452495574951, "shipping": -5.769840240478516, "notifications": -6.161190032958984, "admin": -2.3357882499694824, "integrations": -6.031501770019531 }, "probabilities": { "billing": 0.009698121168532933, "mobile": 0.0034723830084420564, "search": 0.01938122601410312, "checkout": 0.8638268668972444, "shipping": 0.0031105502519696952, "notifications": 0.0021052994789205505, "admin": 0.08820204711558781, "integrations": 0.002396128966808831 } }, { "domain": "product_feedback", "row": 40, "task": "feedback_type", "multi_label": false, "gold": [ "feature_request" ], "selected": [ "feature_request" ], "correct": true, "logits": { "bug": -0.37438634037971497, "feature_request": -0.019676223397254944, "how_to": -1.709827184677124, "praise": -2.8719940185546875 }, "probabilities": { "bug": 0.36086868645143866, "feature_request": 0.5145147686393253, "how_to": 0.09492368931338563, "praise": 0.02969285559585046 } }, { "domain": "product_feedback", "row": 40, "task": "product_area", "multi_label": true, "gold": [ "mobile" ], "selected": [], "correct": false, "logits": { "billing": -1.8436195850372314, "mobile": -0.1043936088681221, "search": -4.942628860473633, "checkout": -5.3129377365112305, "shipping": -5.986028671264648, "notifications": -5.149019241333008, "admin": -0.7737560868263245, "integrations": -6.205305099487305 }, "probabilities": { "billing": 0.13662377274854695, "mobile": 0.4739252737357539, "search": 0.007085255561871444, "checkout": 0.004903269426808733, "shipping": 0.0025073242154835577, "notifications": 0.005771590699157172, "admin": 0.31566714990775085, "integrations": 0.0020146259018345245 } }, { "domain": "product_feedback", "row": 50, "task": "feedback_type", "multi_label": false, "gold": [ "bug" ], "selected": [ "bug" ], "correct": true, "logits": { "bug": 1.9703173637390137, "feature_request": -3.191213369369507, "how_to": -1.8353867530822754, "praise": -4.796311378479004 }, "probabilities": { "bug": 0.9716964100966661, "feature_request": 0.005570655256211862, "how_to": 0.02161395786743544, "praise": 0.0011189767796865198 } }, { "domain": "product_feedback", "row": 50, "task": "product_area", "multi_label": true, "gold": [ "checkout" ], "selected": [ "checkout" ], "correct": true, "logits": { "billing": -4.466052055358887, "mobile": -4.916243553161621, "search": -4.681847095489502, "checkout": 2.4039063453674316, "shipping": -5.456897258758545, "notifications": -5.857413291931152, "admin": -5.174821853637695, "integrations": -3.125593900680542 }, "probabilities": { "billing": 0.011362019419863917, "mobile": 0.007273312546470595, "search": 0.009176895084286863, "checkout": 0.9171246972524596, "shipping": 0.0042486459117511725, "notifications": 0.0028504801010289294, "admin": 0.005625400483961779, "integrations": 0.042063790517863194 } }, { "domain": "product_feedback", "row": 60, "task": "feedback_type", "multi_label": false, "gold": [ "feature_request" ], "selected": [ "feature_request" ], "correct": true, "logits": { "bug": -1.5237855911254883, "feature_request": 3.0302352905273438, "how_to": -2.0846328735351562, "praise": -5.548869609832764 }, "probabilities": { "bug": 0.010351723912892198, "feature_request": 0.9835553837202882, "how_to": 0.005907990667699996, "praise": 0.00018490169911962196 } }, { "domain": "product_feedback", "row": 60, "task": "product_area", "multi_label": true, "gold": [ "billing", "integrations", "mobile", "notifications", "shipping" ], "selected": [ "mobile" ], "correct": false, "logits": { "billing": -0.397719144821167, "mobile": 0.035697244107723236, "search": -5.3560872077941895, "checkout": -5.475062370300293, "shipping": -3.627145290374756, "notifications": -2.0809531211853027, "admin": -4.7319207191467285, "integrations": -4.1676106452941895 }, "probabilities": { "billing": 0.401860462994387, "mobile": 0.5089233634652457, "search": 0.004697168294113839, "checkout": 0.00417248463849236, "shipping": 0.02590317171761815, "notifications": 0.1109619070441041, "admin": 0.008732603965770136, "integrations": 0.015252968533547106 } }, { "domain": "product_feedback", "row": 70, "task": "feedback_type", "multi_label": false, "gold": [ "feature_request" ], "selected": [ "bug" ], "correct": false, "logits": { "bug": -0.0006429851055145264, "feature_request": -1.1120331287384033, "how_to": -0.5855810046195984, "praise": -4.2965497970581055 }, "probabilities": { "bug": 0.526352980704622, "feature_request": 0.1732233686622235, "how_to": 0.2932525063024268, "praise": 0.007171144330727788 } }, { "domain": "product_feedback", "row": 70, "task": "product_area", "multi_label": true, "gold": [ "mobile", "shipping" ], "selected": [ "mobile" ], "correct": false, "logits": { "billing": -7.232300758361816, "mobile": 1.5351924896240234, "search": -5.1819257736206055, "checkout": -6.0301899909973145, "shipping": -1.4215998649597168, "notifications": -5.8712849617004395, "admin": -5.795352935791016, "integrations": -6.2572832107543945 }, "probabilities": { "billing": 0.0007223336887479908, "mobile": 0.8227647675618075, "search": 0.0055858021293833425, "checkout": 0.002399266674706648, "shipping": 0.194410897752851, "notifications": 0.002811322562779685, "admin": 0.0030324331071428894, "integrations": 0.0019127795710144537 } }, { "domain": "product_feedback", "row": 80, "task": "feedback_type", "multi_label": false, "gold": [ "bug" ], "selected": [ "bug" ], "correct": true, "logits": { "bug": 1.2783427238464355, "feature_request": -1.8719159364700317, "how_to": -1.1289310455322266, "praise": -3.188013792037964 }, "probabilities": { "bug": 0.873827497138265, "feature_request": 0.037435682389394986, "how_to": 0.07869732894197222, "praise": 0.01003949153036775 } }, { "domain": "product_feedback", "row": 80, "task": "product_area", "multi_label": true, "gold": [ "checkout", "integrations", "mobile" ], "selected": [ "integrations" ], "correct": false, "logits": { "billing": -6.897904872894287, "mobile": -5.701175689697266, "search": -6.766799449920654, "checkout": -5.776104927062988, "shipping": -6.636965751647949, "notifications": -6.663922309875488, "admin": -4.5836663246154785, "integrations": 4.264232635498047 }, "probabilities": { "billing": 0.001008880213828451, "mobile": 0.0033309019644678787, "search": 0.0011500496525739372, "checkout": 0.00309118458952153, "shipping": 0.0013092826434840023, "notifications": 0.0012745047241680258, "admin": 0.01011402861059297, "integrations": 0.9861323616273261 } }, { "domain": "product_feedback", "row": 90, "task": "feedback_type", "multi_label": false, "gold": [ "bug" ], "selected": [ "bug" ], "correct": true, "logits": { "bug": 1.037614107131958, "feature_request": -2.2700510025024414, "how_to": -0.8981501460075378, "praise": -4.450148105621338 }, "probabilities": { "bug": 0.8438444278199917, "feature_request": 0.030886001048360402, "how_to": 0.12177850930722939, "praise": 0.003491061824418616 } }, { "domain": "product_feedback", "row": 90, "task": "product_area", "multi_label": true, "gold": [ "admin", "billing", "checkout", "shipping" ], "selected": [ "checkout" ], "correct": false, "logits": { "billing": -2.773326873779297, "mobile": -4.431787014007568, "search": -3.1190476417541504, "checkout": 0.13343007862567902, "shipping": -1.394026279449463, "notifications": -4.954594612121582, "admin": -2.526233673095703, "integrations": -4.9714035987854 }, "probabilities": { "billing": 0.05878267619645837, "mobile": 0.011753431213758004, "search": 0.04232836045187198, "checkout": 0.5333081173195959, "shipping": 0.1987657621240243, "notifications": 0.007001570425729209, "admin": 0.07403944306132117, "integrations": 0.006885668318736012 } }, { "domain": "restaurant_review", "row": 0, "task": "sentiment", "multi_label": false, "gold": [ "negative" ], "selected": [ "negative" ], "correct": true, "logits": { "negative": 3.4311156272888184, "neutral": -3.9323270320892334, "positive": -3.942883014678955 }, "probabilities": { "negative": 0.998740222440053, "neutral": 0.0006332132966004076, "positive": 0.000626564263346564 } }, { "domain": "restaurant_review", "row": 0, "task": "aspects", "multi_label": true, "gold": [ "food" ], "selected": [ "food", "service" ], "correct": false, "logits": { "food": 2.103090286254883, "service": 0.7128744721412659, "price": -4.644192695617676, "ambiance": -2.2697980403900146, "wait": -2.2563085556030273, "cleanliness": -3.6692423820495605 }, "probabilities": { "food": 0.8912031756289555, "service": 0.6710360016838312, "price": 0.009525679380542624, "ambiance": 0.09365535397625682, "wait": 0.09480668923671662, "cleanliness": 0.02486190506173712 } }, { "domain": "restaurant_review", "row": 10, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "neutral" ], "correct": true, "logits": { "negative": -2.3676412105560303, "neutral": 2.1755969524383545, "positive": -3.772178888320923 }, "probabilities": { "negative": 0.010499772534703466, "neutral": 0.986922737829678, "positive": 0.0025774896356185214 } }, { "domain": "restaurant_review", "row": 10, "task": "aspects", "multi_label": true, "gold": [ "cleanliness", "food", "price", "service", "wait" ], "selected": [ "food", "price", "service" ], "correct": false, "logits": { "food": 2.529005527496338, "service": 1.5681469440460205, "price": 1.4169182777404785, "ambiance": -0.44887205958366394, "wait": -0.19547057151794434, "cleanliness": -0.954241156578064 }, "probabilities": { "food": 0.9261503644075083, "service": 0.8275192800116351, "price": 0.8048548449392858, "ambiance": 0.38962897743386754, "wait": 0.45128736249742474, "cleanliness": 0.27803269152070953 } }, { "domain": "restaurant_review", "row": 20, "task": "sentiment", "multi_label": false, "gold": [ "negative" ], "selected": [ "negative" ], "correct": true, "logits": { "negative": 4.301956653594971, "neutral": -4.812624454498291, "positive": -4.799843788146973 }, "probabilities": { "negative": 0.9997785347058125, "neutral": 0.00011002503821747758, "positive": 0.00011144025597012117 } }, { "domain": "restaurant_review", "row": 20, "task": "aspects", "multi_label": true, "gold": [ "ambiance", "food", "service", "wait" ], "selected": [ "ambiance", "service", "wait" ], "correct": false, "logits": { "food": -1.5308114290237427, "service": 0.0047189295291900635, "price": -8.005084991455078, "ambiance": 4.147347927093506, "wait": 0.532701849937439, "cleanliness": -5.072324752807617 }, "probabilities": { "food": 0.17787499505545018, "service": 0.5011797301930832, "price": 0.00033364977368942615, "ambiance": 0.9844396705741428, "wait": 0.6301130548050657, "cleanliness": 0.006228791014553975 } }, { "domain": "restaurant_review", "row": 30, "task": "sentiment", "multi_label": false, "gold": [ "positive" ], "selected": [ "positive" ], "correct": true, "logits": { "negative": -3.5751049518585205, "neutral": -2.9166877269744873, "positive": 2.667039394378662 }, "probabilities": { "negative": 0.0019346432265301388, "neutral": 0.0037372130330406337, "positive": 0.9943281437404293 } }, { "domain": "restaurant_review", "row": 30, "task": "aspects", "multi_label": true, "gold": [ "cleanliness", "food", "price", "service", "wait" ], "selected": [ "ambiance", "cleanliness", "food", "price", "service" ], "correct": false, "logits": { "food": 3.574169874191284, "service": 0.9684615135192871, "price": 1.2832040786743164, "ambiance": 0.47929900884628296, "wait": -2.525881052017212, "cleanliness": 1.2533117532730103 }, "probabilities": { "food": 0.9727260345524835, "service": 0.7248127386168017, "price": 0.7829946878935522, "ambiance": 0.6175823322590046, "wait": 0.07406362154898152, "cleanliness": 0.7778726150083526 } }, { "domain": "restaurant_review", "row": 40, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "neutral" ], "correct": true, "logits": { "negative": -3.897585391998291, "neutral": 0.9461735486984253, "positive": -0.6715841889381409 }, "probabilities": { "negative": 0.006530637504691541, "neutral": 0.8290359388765195, "positive": 0.16443342361878896 } }, { "domain": "restaurant_review", "row": 40, "task": "aspects", "multi_label": true, "gold": [ "ambiance" ], "selected": [ "ambiance", "food" ], "correct": false, "logits": { "food": 0.5122835636138916, "service": -0.2713721692562103, "price": -8.365301132202148, "ambiance": 2.8651633262634277, "wait": -3.04152512550354, "cleanliness": -1.3161911964416504 }, "probabilities": { "food": 0.6253416423295901, "service": 0.43257026054541126, "price": 0.0002327527301231693, "ambiance": 0.9460972223721528, "wait": 0.04558477144955111, "cleanliness": 0.21145267710101306 } }, { "domain": "restaurant_review", "row": 50, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "positive" ], "correct": false, "logits": { "negative": -1.1731927394866943, "neutral": -3.25830340385437, "positive": 1.155819058418274 }, "probabilities": { "negative": 0.08778025754560445, "neutral": 0.010910503552117132, "positive": 0.9013092389022785 } }, { "domain": "restaurant_review", "row": 50, "task": "aspects", "multi_label": true, "gold": [ "ambiance", "food", "price", "service", "wait" ], "selected": [ "ambiance", "food", "price", "service", "wait" ], "correct": true, "logits": { "food": 2.9249746799468994, "service": 1.7163939476013184, "price": 1.1792762279510498, "ambiance": 0.011986203491687775, "wait": 1.1121999025344849, "cleanliness": -4.975079536437988 }, "probabilities": { "food": 0.9490673058261977, "service": 0.847663770702401, "price": 0.7648176426689571, "ambiance": 0.5029965149974632, "wait": 0.7525390137520352, "cleanliness": 0.006860576826965758 } }, { "domain": "restaurant_review", "row": 60, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "negative" ], "correct": false, "logits": { "negative": 1.1292475461959839, "neutral": -2.654534101486206, "positive": -1.687299370765686 }, "probabilities": { "negative": 0.9237459967485244, "neutral": 0.021002794302075443, "positive": 0.05525120894940007 } }, { "domain": "restaurant_review", "row": 60, "task": "aspects", "multi_label": true, "gold": [ "cleanliness", "food", "service", "wait" ], "selected": [ "cleanliness", "food", "service", "wait" ], "correct": true, "logits": { "food": 0.8698095083236694, "service": 0.27213793992996216, "price": -5.547048568725586, "ambiance": -0.18930374085903168, "wait": 2.4395177364349365, "cleanliness": 0.3076370358467102 }, "probabilities": { "food": 0.7047060590967447, "service": 0.5676176906259038, "price": 0.003883805001956277, "ambiance": 0.45281489079234283, "wait": 0.9197915159948119, "cleanliness": 0.576308383849247 } }, { "domain": "restaurant_review", "row": 70, "task": "sentiment", "multi_label": false, "gold": [ "negative" ], "selected": [ "negative" ], "correct": true, "logits": { "negative": 4.128564834594727, "neutral": -4.555366039276123, "positive": -4.814444541931152 }, "probabilities": { "negative": 0.9997001583281692, "neutral": 0.0001692335641973484, "positive": 0.00013060810763358287 } }, { "domain": "restaurant_review", "row": 70, "task": "aspects", "multi_label": true, "gold": [ "cleanliness", "food", "service", "wait" ], "selected": [ "cleanliness", "food" ], "correct": false, "logits": { "food": 1.6131587028503418, "service": -0.4202395975589752, "price": -7.117048263549805, "ambiance": -2.341768741607666, "wait": -1.3793755769729614, "cleanliness": 3.1609997749328613 }, "probabilities": { "food": 0.8338494690403091, "service": 0.39645941796558415, "price": 0.000810500100911339, "ambiance": 0.08772226436675057, "wait": 0.20110930354789522, "cleanliness": 0.9593399623626557 } }, { "domain": "restaurant_review", "row": 80, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "neutral" ], "correct": true, "logits": { "negative": -1.210484504699707, "neutral": 1.0471950769424438, "positive": -2.489358901977539 }, "probabilities": { "negative": 0.09225748771630435, "neutral": 0.8820626003355513, "positive": 0.02567991194814436 } }, { "domain": "restaurant_review", "row": 80, "task": "aspects", "multi_label": true, "gold": [ "ambiance", "food", "price", "service", "wait" ], "selected": [ "ambiance", "food", "price", "service", "wait" ], "correct": true, "logits": { "food": 2.9053215980529785, "service": 1.385537028312683, "price": 1.2073744535446167, "ambiance": 0.29535552859306335, "wait": 0.5588858127593994, "cleanliness": -4.6763763427734375 }, "probabilities": { "food": 0.9481088764252061, "service": 0.7998787992198415, "price": 0.7698340591953813, "ambiance": 0.5733067465262822, "wait": 0.6361946996793875, "cleanliness": 0.009226772691895148 } }, { "domain": "restaurant_review", "row": 90, "task": "sentiment", "multi_label": false, "gold": [ "positive" ], "selected": [ "positive" ], "correct": true, "logits": { "negative": -2.9986562728881836, "neutral": -1.493586540222168, "positive": 1.315169334411621 }, "probabilities": { "negative": 0.012464121687664935, "neutral": 0.056144233898463035, "positive": 0.9313916444138721 } }, { "domain": "restaurant_review", "row": 90, "task": "aspects", "multi_label": true, "gold": [ "ambiance", "cleanliness", "food", "service" ], "selected": [ "cleanliness" ], "correct": false, "logits": { "food": -1.211390495300293, "service": -2.103541851043701, "price": -8.122198104858398, "ambiance": -2.4394760131835938, "wait": -5.954218864440918, "cleanliness": 7.269656181335449 }, "probabilities": { "food": 0.2294551113069971, "service": 0.10875304834901758, "price": 0.0002967872706669829, "ambiance": 0.08021156219543728, "wait": 0.0025881540650593963, "cleanliness": 0.9993041331965979 } }, { "domain": "review_sentiment", "row": 0, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "neutral" ], "correct": true, "logits": { "negative": -1.0817779302597046, "neutral": 1.3606643676757812, "positive": -4.214090824127197 }, "probabilities": { "negative": 0.07971486076464313, "neutral": 0.9168082321385121, "positive": 0.0034769070968447406 } }, { "domain": "review_sentiment", "row": 10, "task": "sentiment", "multi_label": false, "gold": [ "negative" ], "selected": [ "negative" ], "correct": true, "logits": { "negative": 2.075194835662842, "neutral": -2.5304882526397705, "positive": -3.545086145401001 }, "probabilities": { "negative": 0.9865644765220017, "neutral": 0.00986058595063545, "positive": 0.0035749375273628017 } }, { "domain": "review_sentiment", "row": 20, "task": "sentiment", "multi_label": false, "gold": [ "negative" ], "selected": [ "negative" ], "correct": true, "logits": { "negative": 3.2312841415405273, "neutral": -2.850769281387329, "positive": -4.806124687194824 }, "probabilities": { "negative": 0.997400148775098, "neutral": 0.002277546163559687, "positive": 0.00032230506134235583 } }, { "domain": "review_sentiment", "row": 30, "task": "sentiment", "multi_label": false, "gold": [ "positive" ], "selected": [ "positive" ], "correct": true, "logits": { "negative": -3.8353404998779297, "neutral": -2.7773284912109375, "positive": 2.9101033210754395 }, "probabilities": { "negative": 0.0011708820266562296, "neutral": 0.0033728879721098286, "positive": 0.9954562300012338 } }, { "domain": "review_sentiment", "row": 40, "task": "sentiment", "multi_label": false, "gold": [ "negative" ], "selected": [ "negative" ], "correct": true, "logits": { "negative": 2.4293534755706787, "neutral": -2.913551092147827, "positive": -3.5487117767333984 }, "probabilities": { "negative": 0.9927374458968075, "neutral": 0.004747231739284028, "positive": 0.0025153223639085513 } }, { "domain": "review_sentiment", "row": 50, "task": "sentiment", "multi_label": false, "gold": [ "positive" ], "selected": [ "negative" ], "correct": false, "logits": { "negative": 1.2212960720062256, "neutral": -1.9321377277374268, "positive": -2.0308170318603516 }, "probabilities": { "negative": 0.9247292628987339, "neutral": 0.03949077917975803, "positive": 0.03577995792150801 } }, { "domain": "review_sentiment", "row": 60, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "neutral" ], "correct": true, "logits": { "negative": -2.30391788482666, "neutral": 1.3322025537490845, "positive": -1.5462404489517212 }, "probabilities": { "negative": 0.024344133089506333, "neutral": 0.9237221455962471, "positive": 0.051933721314246506 } }, { "domain": "review_sentiment", "row": 70, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "neutral" ], "correct": true, "logits": { "negative": -0.8435385227203369, "neutral": -0.4499221444129944, "positive": -0.9024451375007629 }, "probabilities": { "negative": 0.29196001164253454, "neutral": 0.43278160519451053, "positive": 0.27525838316295503 } }, { "domain": "review_sentiment", "row": 80, "task": "sentiment", "multi_label": false, "gold": [ "negative" ], "selected": [ "negative" ], "correct": true, "logits": { "negative": 2.3499996662139893, "neutral": -2.763094186782837, "positive": -3.4606552124023438 }, "probabilities": { "negative": 0.9910676020610465, "neutral": 0.005963686886942593, "positive": 0.0029687110520109903 } }, { "domain": "review_sentiment", "row": 90, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "neutral" ], "correct": true, "logits": { "negative": -2.51008939743042, "neutral": 2.128190040588379, "positive": -3.475864887237549 }, "probabilities": { "negative": 0.009546809683498655, "neutral": 0.986818835714034, "positive": 0.0036343546024674004 } }, { "domain": "screen_tags", "row": 0, "task": "genres", "multi_label": true, "gold": [ "drama", "kids" ], "selected": [ "kids" ], "correct": false, "logits": { "comedy": -2.9973196983337402, "drama": -2.462252140045166, "action": -3.7580924034118652, "documentary": -1.5090152025222778, "horror": -6.180383205413818, "romance": -3.1607189178466797, "sci_fi": -3.8704850673675537, "kids": 2.448904037475586 }, "probabilities": { "comedy": 0.047547107242527205, "drama": 0.07854717806499771, "action": 0.022796399804035125, "documentary": 0.18108478594613675, "horror": 0.0020653600668042965, "romance": 0.040670994386692, "sci_fi": 0.020422481087967286, "kids": 0.9204812682464156 } }, { "domain": "screen_tags", "row": 0, "task": "format", "multi_label": false, "gold": [ "series" ], "selected": [ "series" ], "correct": true, "logits": { "film": -2.646592617034912, "series": 2.226633071899414 }, "probabilities": { "film": 0.007590595443744671, "series": 0.9924094045562554 } }, { "domain": "screen_tags", "row": 10, "task": "genres", "multi_label": true, "gold": [ "action", "drama" ], "selected": [], "correct": false, "logits": { "comedy": -3.556852340698242, "drama": -0.45208778977394104, "action": -2.0964674949645996, "documentary": -0.44363290071487427, "horror": -3.9267542362213135, "romance": -0.8156840801239014, "sci_fi": -2.3505938053131104, "kids": -2.0397205352783203 }, "probabilities": { "comedy": 0.027737181798639553, "drama": 0.38886449014137686, "action": 0.10944063638541156, "documentary": 0.3908756630072997, "horror": 0.019326654148496093, "romance": 0.3066805771259729, "sci_fi": 0.08701858520455467, "kids": 0.11509519198767941 } }, { "domain": "screen_tags", "row": 10, "task": "format", "multi_label": false, "gold": [ "series" ], "selected": [ "series" ], "correct": true, "logits": { "film": -1.857682228088379, "series": 1.7427105903625488 }, "probabilities": { "film": 0.026586825557838454, "series": 0.9734131744421616 } }, { "domain": "screen_tags", "row": 20, "task": "genres", "multi_label": true, "gold": [ "drama", "romance" ], "selected": [ "romance" ], "correct": false, "logits": { "comedy": -5.583466053009033, "drama": -0.8238534331321716, "action": -4.970945835113525, "documentary": -2.23211407661438, "horror": -5.625825881958008, "romance": 4.85933256149292, "sci_fi": -3.3909053802490234, "kids": -5.305689334869385 }, "probabilities": { "comedy": 0.00374543124684549, "drama": 0.30494629578264726, "action": 0.006888799330569772, "documentary": 0.09690347367074281, "horror": 0.0035906465673607615, "romance": 0.9923040287363947, "sci_fi": 0.03258090619311728, "kids": 0.0049387632463772455 } }, { "domain": "screen_tags", "row": 20, "task": "format", "multi_label": false, "gold": [ "series" ], "selected": [ "series" ], "correct": true, "logits": { "film": -1.0676794052124023, "series": 1.0073566436767578 }, "probabilities": { "film": 0.11154696753504831, "series": 0.8884530324649517 } }, { "domain": "screen_tags", "row": 30, "task": "genres", "multi_label": true, "gold": [ "drama", "romance" ], "selected": [], "correct": false, "logits": { "comedy": -3.2204482555389404, "drama": -0.9216711521148682, "action": -2.890458106994629, "documentary": -1.5710890293121338, "horror": -2.0566515922546387, "romance": -1.4178026914596558, "sci_fi": -1.2243571281433105, "kids": -1.1792361736297607 }, "probabilities": { "comedy": 0.0384034286241374, "drama": 0.28461750793993584, "action": 0.05262727361847107, "documentary": 0.17206119740983045, "horror": 0.11338199835062478, "romance": 0.19500628337706422, "sci_fi": 0.2271705860328499, "kids": 0.23518956204296626 } }, { "domain": "screen_tags", "row": 30, "task": "format", "multi_label": false, "gold": [ "film" ], "selected": [ "film" ], "correct": true, "logits": { "film": -0.0920335128903389, "series": -0.12952928245067596 }, "probabilities": { "film": 0.509372844283432, "series": 0.490627155716568 } }, { "domain": "screen_tags", "row": 40, "task": "genres", "multi_label": true, "gold": [ "documentary", "drama" ], "selected": [ "documentary" ], "correct": false, "logits": { "comedy": -3.624598979949951, "drama": -1.016570806503296, "action": -2.419455051422119, "documentary": 0.8203496932983398, "horror": -3.1922175884246826, "romance": -2.882633924484253, "sci_fi": -1.8920269012451172, "kids": -2.3769617080688477 }, "probabilities": { "comedy": 0.025967498345130898, "drama": 0.2656959038587342, "action": 0.08170113148568522, "documentary": 0.694310565246883, "horror": 0.03945964133767026, "romance": 0.05301873717520621, "sci_fi": 0.13101353614422248, "kids": 0.08494643617386685 } }, { "domain": "screen_tags", "row": 40, "task": "format", "multi_label": false, "gold": [ "film" ], "selected": [ "film" ], "correct": true, "logits": { "film": 1.481339454650879, "series": -1.5633546113967896 }, "probabilities": { "film": 0.9545529006480344, "series": 0.045447099351965614 } }, { "domain": "screen_tags", "row": 50, "task": "genres", "multi_label": true, "gold": [ "comedy", "drama", "kids" ], "selected": [ "comedy" ], "correct": false, "logits": { "comedy": 2.1776421070098877, "drama": -1.5409821271896362, "action": -4.038027286529541, "documentary": -2.0654847621917725, "horror": -5.662467956542969, "romance": -2.362300395965576, "sci_fi": -2.132750988006592, "kids": -3.1231229305267334 }, "probabilities": { "comedy": 0.8982237208597911, "drama": 0.17639254774173407, "action": 0.01732671300939357, "documentary": 0.11249705838842965, "horror": 0.0034619063647477965, "romance": 0.08609302458380173, "sci_fi": 0.10595411389339347, "kids": 0.04216346958773319 } }, { "domain": "screen_tags", "row": 50, "task": "format", "multi_label": false, "gold": [ "film" ], "selected": [ "film" ], "correct": true, "logits": { "film": 1.838075041770935, "series": -2.1565794944763184 }, "probabilities": { "film": 0.9819191310102254, "series": 0.0180808689897746 } }, { "domain": "screen_tags", "row": 60, "task": "genres", "multi_label": true, "gold": [ "comedy" ], "selected": [ "comedy" ], "correct": true, "logits": { "comedy": 1.8940101861953735, "drama": -0.7889949083328247, "action": -1.4857261180877686, "documentary": -2.785794734954834, "horror": -2.057213068008423, "romance": -4.196743488311768, "sci_fi": -2.499542713165283, "kids": -4.326355457305908 }, "probabilities": { "comedy": 0.8692120936521984, "drama": 0.31238452282038465, "action": 0.18456408046248746, "documentary": 0.05809664619802072, "horror": 0.11332556739790674, "romance": 0.014821507671732589, "sci_fi": 0.0758902437462284, "kids": 0.013043250037821467 } }, { "domain": "screen_tags", "row": 60, "task": "format", "multi_label": false, "gold": [ "film" ], "selected": [ "series" ], "correct": false, "logits": { "film": -0.035419363528490067, "series": 0.0376308374106884 }, "probabilities": { "film": 0.4817455666861589, "series": 0.5182544333138411 } }, { "domain": "screen_tags", "row": 70, "task": "genres", "multi_label": true, "gold": [ "documentary", "drama" ], "selected": [], "correct": false, "logits": { "comedy": -2.780012369155884, "drama": -0.9037413001060486, "action": -2.3780908584594727, "documentary": -0.26711133122444153, "horror": -2.967707633972168, "romance": -1.9372559785842896, "sci_fi": -1.4683241844177246, "kids": -1.3590985536575317 }, "probabilities": { "comedy": 0.05841387521414841, "drama": 0.28828226625695047, "action": 0.0848587078291185, "documentary": 0.433616395983375, "horror": 0.048906240882317056, "romance": 0.12594962580229124, "sci_fi": 0.18719746332096188, "kids": 0.20438684999510603 } }, { "domain": "screen_tags", "row": 70, "task": "format", "multi_label": false, "gold": [ "film" ], "selected": [ "series" ], "correct": false, "logits": { "film": -0.18348264694213867, "series": 0.15380704402923584 }, "probabilities": { "film": 0.4164679933903713, "series": 0.5835320066096287 } }, { "domain": "screen_tags", "row": 80, "task": "genres", "multi_label": true, "gold": [ "drama", "sci_fi" ], "selected": [ "drama", "sci_fi" ], "correct": true, "logits": { "comedy": -6.5781145095825195, "drama": 0.3066234588623047, "action": -3.301103353500366, "documentary": -4.00963020324707, "horror": -4.827812194824219, "romance": -3.301034927368164, "sci_fi": 3.990722417831421, "kids": -5.211493015289307 }, "probabilities": { "comedy": 0.0013885378217294458, "drama": 0.576060872508525, "action": 0.03553335715023332, "documentary": 0.01781690173016259, "horror": 0.007940457815839625, "romance": 0.03553570223879035, "sci_fi": 0.9818491879224397, "kids": 0.0054239457834352155 } }, { "domain": "screen_tags", "row": 80, "task": "format", "multi_label": false, "gold": [ "series" ], "selected": [ "series" ], "correct": true, "logits": { "film": -2.4320244789123535, "series": 2.3908121585845947 }, "probabilities": { "film": 0.007979748418652476, "series": 0.9920202515813474 } }, { "domain": "screen_tags", "row": 90, "task": "genres", "multi_label": true, "gold": [ "drama" ], "selected": [ "action" ], "correct": false, "logits": { "comedy": -3.3116559982299805, "drama": -0.857361376285553, "action": 0.16020071506500244, "documentary": -2.6301140785217285, "horror": -2.9696078300476074, "romance": -3.026545763015747, "sci_fi": -0.7092880606651306, "kids": -3.775458335876465 }, "probabilities": { "comedy": 0.03517347746731564, "drama": 0.29789092440408005, "action": 0.539964743143364, "documentary": 0.06722529682046942, "horror": 0.048817930086695896, "romance": 0.04624092968019825, "sci_fi": 0.3297561718390959, "kids": 0.02241273284312032 } }, { "domain": "screen_tags", "row": 90, "task": "format", "multi_label": false, "gold": [ "film" ], "selected": [ "film" ], "correct": true, "logits": { "film": 0.0998937338590622, "series": -0.22861698269844055 }, "probabilities": { "film": 0.5813969671778477, "series": 0.4186030328221523 } }, { "domain": "sports_recap", "row": 0, "task": "sport", "multi_label": false, "gold": [ "motorsport" ], "selected": [ "motorsport" ], "correct": true, "logits": { "football": -3.40275239944458, "basketball": -3.5192811489105225, "baseball": -3.482391595840454, "soccer": -2.738308906555176, "tennis": -3.206150770187378, "cricket": -3.3841514587402344, "motorsport": 0.7256821393966675, "other": -1.9831194877624512 }, "probabilities": { "football": 0.013659492750723503, "basketball": 0.012157010145005412, "baseball": 0.012613851343637251, "soccer": 0.026545977043659134, "tennis": 0.01662714081029131, "cricket": 0.013915949943583826, "motorsport": 0.8479903520092403, "other": 0.05649022595385936 } }, { "domain": "sports_recap", "row": 0, "task": "result", "multi_label": false, "gold": [ "draw" ], "selected": [ "draw" ], "correct": true, "logits": { "win": -3.6908392906188965, "loss": -4.7196245193481445, "draw": 1.8601021766662598, "upcoming": -2.0428690910339355 }, "probabilities": { "win": 0.0037873951345046442, "loss": 0.0013537699478297146, "draw": 0.9751779337271308, "upcoming": 0.019680901190534825 } }, { "domain": "sports_recap", "row": 0, "task": "upset", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.6043936610221863, "no": 0.643641471862793 }, "probabilities": { "yes": 0.22304045204140244, "no": 0.7769595479585976 } }, { "domain": "sports_recap", "row": 10, "task": "sport", "multi_label": false, "gold": [ "tennis" ], "selected": [ "tennis" ], "correct": true, "logits": { "football": -5.411147594451904, "basketball": -5.23582649230957, "baseball": -5.42658805847168, "soccer": -4.774247169494629, "tennis": 2.6654679775238037, "cricket": -5.140115261077881, "motorsport": -5.185337066650391, "other": -3.455927848815918 }, "probabilities": { "football": 0.0003093083583589456, "basketball": 0.00036858074472891143, "baseball": 0.0003045691755262174, "soccer": 0.0005847819997333507, "tennis": 0.995454080917965, "cricket": 0.0004056014536291572, "motorsport": 0.00038766797184398686, "other": 0.002185409378214436 } }, { "domain": "sports_recap", "row": 10, "task": "result", "multi_label": false, "gold": [ "draw" ], "selected": [ "draw" ], "correct": true, "logits": { "win": -4.540075302124023, "loss": -3.7153735160827637, "draw": 3.0110177993774414, "upcoming": -3.843536138534546 }, "probabilities": { "win": 0.0005240789191627369, "loss": 0.0011955290285422114, "draw": 0.9972286728437305, "upcoming": 0.0010517192085644253 } }, { "domain": "sports_recap", "row": 10, "task": "upset", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 0.853242039680481, "no": -0.77530837059021 }, "probabilities": { "yes": 0.8359709627357182, "no": 0.16402903726428178 } }, { "domain": "sports_recap", "row": 20, "task": "sport", "multi_label": false, "gold": [ "cricket" ], "selected": [ "cricket" ], "correct": true, "logits": { "football": -4.66676664352417, "basketball": -4.864945888519287, "baseball": -4.280831813812256, "soccer": -4.602468013763428, "tennis": -4.9387006759643555, "cricket": 1.9596943855285645, "motorsport": -4.193851947784424, "other": -2.6121573448181152 }, "probabilities": { "football": 0.0012998252577641454, "basketball": 0.0010661463372937852, "baseball": 0.0019120283999325648, "soccer": 0.0013861477101776234, "tennis": 0.0009903427396389252, "cricket": 0.9811161649443573, "motorsport": 0.0020857834374850407, "other": 0.01014356117335074 } }, { "domain": "sports_recap", "row": 20, "task": "result", "multi_label": false, "gold": [ "win" ], "selected": [ "win" ], "correct": true, "logits": { "win": 3.914074659347534, "loss": -4.352173328399658, "draw": -6.251486778259277, "upcoming": -4.418915748596191 }, "probabilities": { "win": 0.9994643145536303, "loss": 0.0002569102343610504, "draw": 3.8452099251538944e-05, "upcoming": 0.00024032311275719648 } }, { "domain": "sports_recap", "row": 20, "task": "upset", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 1.9932836294174194, "no": -1.964835524559021 }, "probabilities": { "yes": 0.9812589328795374, "no": 0.018741067120462602 } }, { "domain": "sports_recap", "row": 30, "task": "sport", "multi_label": false, "gold": [ "tennis" ], "selected": [ "tennis" ], "correct": true, "logits": { "football": -3.258631944656372, "basketball": -2.6675801277160645, "baseball": -3.250924587249756, "soccer": -2.9458470344543457, "tennis": -0.19579628109931946, "cricket": -2.0203590393066406, "motorsport": -2.5737075805664062, "other": -1.7609295845031738 }, "probabilities": { "football": 0.02741705461441213, "basketball": 0.04951209919492819, "baseball": 0.027629184082074803, "soccer": 0.037485348919276164, "tennis": 0.5863992751699467, "cricket": 0.09457925462561861, "motorsport": 0.05438506713784295, "other": 0.12259271625590044 } }, { "domain": "sports_recap", "row": 30, "task": "result", "multi_label": false, "gold": [ "loss" ], "selected": [ "loss" ], "correct": true, "logits": { "win": -2.5975780487060547, "loss": 2.487283706665039, "draw": -4.777074337005615, "upcoming": -4.574151515960693 }, "probabilities": { "win": 0.006142157350379052, "loss": 0.9923122201446292, "draw": 0.0006946686929571316, "upcoming": 0.0008509538120347677 } }, { "domain": "sports_recap", "row": 30, "task": "upset", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.5849778652191162, "no": 0.566157341003418 }, "probabilities": { "yes": 0.2402817940874383, "no": 0.7597182059125618 } }, { "domain": "sports_recap", "row": 40, "task": "sport", "multi_label": false, "gold": [ "basketball" ], "selected": [ "basketball" ], "correct": true, "logits": { "football": -4.703919410705566, "basketball": 2.664344310760498, "baseball": -5.449629306793213, "soccer": -4.831054210662842, "tennis": -5.824774742126465, "cricket": -5.51491117477417, "motorsport": -4.998651504516602, "other": -3.25665020942688 }, "probabilities": { "football": 0.0006277458977375254, "basketball": 0.9949016674920101, "baseball": 0.0002978010264624541, "soccer": 0.0005528024272058809, "tennis": 0.00020464568782240427, "cricket": 0.0002789810055620955, "motorsport": 0.0004675018796389623, "other": 0.0026688545835604847 } }, { "domain": "sports_recap", "row": 40, "task": "result", "multi_label": false, "gold": [ "draw" ], "selected": [ "draw" ], "correct": true, "logits": { "win": -6.678597927093506, "loss": -6.311668872833252, "draw": 5.832929611206055, "upcoming": -6.301272869110107 }, "probabilities": { "win": 3.6838877422730625e-06, "loss": 5.316938673922259e-06, "draw": 0.9999856266716783, "upcoming": 5.3725019055291926e-06 } }, { "domain": "sports_recap", "row": 40, "task": "upset", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -3.9575252532958984, "no": 4.268951892852783 }, "probabilities": { "yes": 0.00026740542986847867, "no": 0.9997325945701316 } }, { "domain": "sports_recap", "row": 50, "task": "sport", "multi_label": false, "gold": [ "football" ], "selected": [ "football" ], "correct": true, "logits": { "football": 0.3298264443874359, "basketball": -2.941612958908081, "baseball": -3.4793291091918945, "soccer": -3.1606991291046143, "tennis": -3.828801155090332, "cricket": -3.271023750305176, "motorsport": -3.3008480072021484, "other": -1.7379943132400513 }, "probabilities": { "football": 0.7773073710727126, "basketball": 0.029500182678390188, "baseball": 0.01723048684036846, "soccer": 0.0236960954498704, "tennis": 0.012148531030432438, "cricket": 0.02122088131103267, "motorsport": 0.020597329006706185, "other": 0.09829912261048707 } }, { "domain": "sports_recap", "row": 50, "task": "result", "multi_label": false, "gold": [ "upcoming" ], "selected": [ "upcoming" ], "correct": true, "logits": { "win": -2.2609195709228516, "loss": -2.376054286956787, "draw": -1.5505235195159912, "upcoming": 0.8545271158218384 }, "probabilities": { "win": 0.037779272351956786, "loss": 0.03367062803695253, "draw": 0.07687314941265058, "upcoming": 0.85167695019844 } }, { "domain": "sports_recap", "row": 50, "task": "upset", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.1624870300292969, "no": 1.2415966987609863 }, "probabilities": { "yes": 0.08286182136557227, "no": 0.9171381786344278 } }, { "domain": "sports_recap", "row": 60, "task": "sport", "multi_label": false, "gold": [ "tennis" ], "selected": [ "tennis" ], "correct": true, "logits": { "football": -5.262640476226807, "basketball": -5.167787551879883, "baseball": -5.3518900871276855, "soccer": -4.882481098175049, "tennis": 2.933011054992676, "cricket": -5.024293899536133, "motorsport": -5.13757848739624, "other": -4.162101745605469 }, "probabilities": { "football": 0.00027510037052433075, "basketball": 0.00030247206894689005, "baseball": 0.00025161154343978953, "soccer": 0.00040233915128574173, "tennis": 0.9972806920904782, "cricket": 0.00034914335301529137, "motorsport": 0.0003117488836475307, "other": 0.0008268925386621547 } }, { "domain": "sports_recap", "row": 60, "task": "result", "multi_label": false, "gold": [ "draw" ], "selected": [ "win" ], "correct": false, "logits": { "win": 0.8680621385574341, "loss": -0.5558586716651917, "draw": -4.178221702575684, "upcoming": -4.164464473724365 }, "probabilities": { "win": 0.7976239332870076, "loss": 0.19204244486541835, "draw": 0.005131270984158244, "upcoming": 0.0052023508634156754 } }, { "domain": "sports_recap", "row": 60, "task": "upset", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 0.6553980112075806, "no": -0.5177114009857178 }, "probabilities": { "yes": 0.7637065966444465, "no": 0.23629340335555346 } }, { "domain": "sports_recap", "row": 70, "task": "sport", "multi_label": false, "gold": [ "football" ], "selected": [ "football" ], "correct": true, "logits": { "football": 1.2942062616348267, "basketball": -4.255070686340332, "baseball": -4.5513153076171875, "soccer": -4.233662128448486, "tennis": -4.787654399871826, "cricket": -4.312697410583496, "motorsport": -3.616319417953491, "other": -2.0903522968292236 }, "probabilities": { "football": 0.9452018231959857, "basketball": 0.0036770894401279516, "baseball": 0.0027343039465337633, "soccer": 0.003756659321427653, "tennis": 0.0021587682679058942, "cricket": 0.0034711807227104994, "motorsport": 0.006964827182318829, "other": 0.032035347922989746 } }, { "domain": "sports_recap", "row": 70, "task": "result", "multi_label": false, "gold": [ "win" ], "selected": [ "win" ], "correct": true, "logits": { "win": 3.390052318572998, "loss": -3.5722217559814453, "draw": -5.622653961181641, "upcoming": -4.753787994384766 }, "probabilities": { "win": 0.9986425334970668, "loss": 0.0009456552701033382, "draw": 0.00012168623523317905, "upcoming": 0.0002901249975967974 } }, { "domain": "sports_recap", "row": 70, "task": "upset", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 1.4320242404937744, "no": -1.4664568901062012 }, "probabilities": { "yes": 0.947771302693011, "no": 0.05222869730698893 } }, { "domain": "sports_recap", "row": 80, "task": "sport", "multi_label": false, "gold": [ "football" ], "selected": [ "basketball" ], "correct": false, "logits": { "football": -1.6629345417022705, "basketball": -0.9483256936073303, "baseball": -2.60512638092041, "soccer": -2.3981869220733643, "tennis": -3.283414125442505, "cricket": -2.579902410507202, "motorsport": -2.429736614227295, "other": -1.5109542608261108 }, "probabilities": { "football": 0.1629006549887037, "basketball": 0.3328691216002774, "baseball": 0.06349420851074951, "soccer": 0.07809204195186316, "tennis": 0.03222237067091347, "cricket": 0.065116154510754, "motorsport": 0.07566672228652631, "other": 0.18963872548021243 } }, { "domain": "sports_recap", "row": 80, "task": "result", "multi_label": false, "gold": [ "loss" ], "selected": [ "loss" ], "correct": true, "logits": { "win": -4.141717433929443, "loss": 3.851264476776123, "draw": -5.884843826293945, "upcoming": -5.311557292938232 }, "probabilities": { "win": 0.0003376557789918982, "loss": 0.9994984497662107, "draw": 5.9080479801194594e-05, "upcoming": 0.00010481397499626608 } }, { "domain": "sports_recap", "row": 80, "task": "upset", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 1.0740731954574585, "no": -0.9368605017662048 }, "probabilities": { "yes": 0.881940274939081, "no": 0.11805972506091911 } }, { "domain": "sports_recap", "row": 90, "task": "sport", "multi_label": false, "gold": [ "other" ], "selected": [ "other" ], "correct": true, "logits": { "football": -2.528750419616699, "basketball": -2.6186563968658447, "baseball": -1.6534979343414307, "soccer": -2.6076138019561768, "tennis": -2.16637921333313, "cricket": -2.0300567150115967, "motorsport": -2.2031657695770264, "other": -1.00301992893219 }, "probabilities": { "football": 0.06990901396025904, "basketball": 0.06389803557649215, "baseball": 0.1677453543004882, "soccer": 0.06460754590673956, "tennis": 0.1004405291195495, "cricket": 0.11511001239577, "motorsport": 0.09681280281746037, "other": 0.32147670592324123 } }, { "domain": "sports_recap", "row": 90, "task": "result", "multi_label": false, "gold": [ "draw" ], "selected": [ "draw" ], "correct": true, "logits": { "win": -3.6799890995025635, "loss": -2.641489028930664, "draw": 2.309356212615967, "upcoming": -3.6898207664489746 }, "probabilities": { "win": 0.0024754411687310874, "loss": 0.006993063300772425, "draw": 0.9880802728258625, "upcoming": 0.0024512227046339613 } }, { "domain": "sports_recap", "row": 90, "task": "upset", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.23471786081790924, "no": 0.31099414825439453 }, "probabilities": { "yes": 0.3668598266698583, "no": 0.6331401733301417 } }, { "domain": "support_intent", "row": 0, "task": "intent", "multi_label": false, "gold": [ "refund_request" ], "selected": [ "refund_request" ], "correct": true, "logits": { "order_status": -3.532860279083252, "refund_request": 1.5252102613449097, "cancel_subscription": -3.1222126483917236, "update_payment": -2.358572006225586, "login_problem": -4.430446147918701, "password_reset": -4.938834190368652, "shipping_delay": -5.399564266204834, "missing_item": -5.6522626876831055, "damaged_item": -5.860651016235352, "change_address": -6.041759014129639, "change_order": -5.504449844360352, "product_question": -3.899657726287842, "pricing_question": -3.1928601264953613, "upgrade_plan": -5.349855899810791, "downgrade_plan": -4.986571311950684, "bug_report": -3.8043453693389893, "feature_request": -5.2069292068481445, "speak_to_human": -2.1584556102752686, "complaint": -2.2726423740386963, "invoice_copy": -4.382680892944336, "tax_document": -5.349926471710205, "appointment": -4.820757865905762, "outage": -4.076328754425049, "data_export": -4.874078750610352, "close_account": -4.382353782653809, "promo_code": -4.894509315490723, "partner_inquiry": -4.44793701171875, "other": -3.2471506595611572 }, "probabilities": { "order_status": 0.005576566353598531, "refund_request": 0.8771199697259451, "cancel_subscription": 0.00840831308012896, "update_payment": 0.018044871392550135, "login_problem": 0.0022727427665657335, "password_reset": 0.0013669737116722828, "shipping_delay": 0.0008623183603502198, "missing_item": 0.0006697644668423252, "damaged_item": 0.0005437762080755494, "change_address": 0.0004536970942589796, "change_order": 0.0007764552044537134, "product_question": 0.0038642815977771626, "pricing_question": 0.007834784657599436, "upgrade_plan": 0.0009062660319369185, "downgrade_plan": 0.0013032513822764957, "bug_report": 0.0042507190119744406, "feature_request": 0.001045509483271168, "speak_to_human": 0.02204262121203863, "complaint": 0.01966403127349003, "invoice_copy": 0.0023839353351315746, "tax_document": 0.0009062020772783977, "appointment": 0.0015382965246287037, "outage": 0.0032384821986243414, "data_export": 0.0014584216169464482, "close_account": 0.0023847152724674574, "promo_code": 0.0014289275555601061, "partner_inquiry": 0.0022333361647699926, "other": 0.007420770239787125 } }, { "domain": "support_intent", "row": 10, "task": "intent", "multi_label": false, "gold": [ "complaint" ], "selected": [ "order_status" ], "correct": false, "logits": { "order_status": -1.4687130451202393, "refund_request": -3.193952798843384, "cancel_subscription": -4.715053558349609, "update_payment": -2.338899850845337, "login_problem": -3.491482973098755, "password_reset": -3.961684465408325, "shipping_delay": -1.7206521034240723, "missing_item": -3.470902919769287, "damaged_item": -4.844634056091309, "change_address": -5.507717132568359, "change_order": -4.205949783325195, "product_question": -3.3949267864227295, "pricing_question": -4.089383125305176, "upgrade_plan": -5.386978626251221, "downgrade_plan": -5.269582271575928, "bug_report": -2.6926543712615967, "feature_request": -4.187283515930176, "speak_to_human": -2.342346429824829, "complaint": -2.1933703422546387, "invoice_copy": -5.299068450927734, "tax_document": -4.88681173324585, "appointment": -4.259487152099609, "outage": -3.344449043273926, "data_export": -4.262869358062744, "close_account": -4.513934135437012, "promo_code": -5.005595684051514, "partner_inquiry": -3.832383871078491, "other": -2.3870341777801514 }, "probabilities": { "order_status": 0.18858650747208464, "refund_request": 0.03359297855224402, "cancel_subscription": 0.0073391006947282796, "update_payment": 0.07899385159052116, "login_problem": 0.02494783136588337, "password_reset": 0.015589309749907675, "shipping_delay": 0.1465868035797255, "missing_item": 0.025466578680551518, "damaged_item": 0.006447134807507151, "change_address": 0.003321952590601519, "change_order": 0.012210791244231117, "product_question": 0.027476829500353794, "pricing_question": 0.013720440758136497, "upgrade_plan": 0.003748258177458829, "downgrade_plan": 0.004215160212184285, "bug_report": 0.055457418999703655, "feature_request": 0.01244086173712993, "speak_to_human": 0.07872206168385691, "complaint": 0.0913683857850697, "invoice_copy": 0.004092685767393617, "tax_document": 0.00618086444222075, "appointment": 0.011574249041181078, "outage": 0.028899399866611886, "data_export": 0.011535168673236326, "close_account": 0.008974037954040254, "promo_code": 0.0054886053479319, "partner_inquiry": 0.01774113582842819, "other": 0.07528159589707638 } }, { "domain": "support_intent", "row": 20, "task": "intent", "multi_label": false, "gold": [ "pricing_question" ], "selected": [ "pricing_question" ], "correct": true, "logits": { "order_status": -4.535772323608398, "refund_request": -6.841989517211914, "cancel_subscription": -8.470584869384766, "update_payment": -2.077702283859253, "login_problem": -7.167312145233154, "password_reset": -8.209773063659668, "shipping_delay": -6.517019271850586, "missing_item": -8.440262794494629, "damaged_item": -9.195425987243652, "change_address": -8.094571113586426, "change_order": -5.519059181213379, "product_question": -4.684219837188721, "pricing_question": 4.850820541381836, "upgrade_plan": -8.037114143371582, "downgrade_plan": -9.194077491760254, "bug_report": -5.863413333892822, "feature_request": -9.013949394226074, "speak_to_human": -4.298465251922607, "complaint": -4.950105667114258, "invoice_copy": -7.4174017906188965, "tax_document": -5.114377975463867, "appointment": -8.3695650100708, "outage": -7.836897850036621, "data_export": -8.278581619262695, "close_account": -8.877793312072754, "promo_code": -7.899497032165527, "partner_inquiry": -7.269920349121094, "other": -5.506084442138672 }, "probabilities": { "order_status": 8.371593052318754e-05, "refund_request": 8.34124173697327e-06, "cancel_subscription": 1.6365931644746412e-06, "update_payment": 0.0009779898855087382, "login_problem": 6.024831225334992e-06, "password_reset": 2.1242707099745903e-06, "shipping_delay": 1.1544190691171636e-05, "missing_item": 1.6869780923663784e-06, "damaged_item": 7.927682256469643e-07, "change_address": 2.383644212252022e-06, "change_order": 3.1316415928281044e-05, "product_question": 7.216692039514148e-05, "pricing_question": 0.9985127023675575, "upgrade_plan": 2.5246122042407206e-06, "downgrade_plan": 7.938379911434926e-07, "bug_report": 2.2193251982784772e-05, "feature_request": 9.505183777649264e-07, "speak_to_human": 0.00010613759940597695, "complaint": 5.5317866786542775e-05, "invoice_copy": 4.6917226660231555e-06, "tax_document": 4.6937814803206346e-05, "appointment": 1.8105607460652554e-06, "outage": 3.084235336951884e-06, "data_export": 1.9830181307007294e-06, "close_account": 1.0891616803686317e-06, "promo_code": 2.8970836111534346e-06, "partner_inquiry": 5.4372926519004275e-06, "other": 3.1725385654168016e-05 } }, { "domain": "support_intent", "row": 30, "task": "intent", "multi_label": false, "gold": [ "password_reset" ], "selected": [ "password_reset" ], "correct": true, "logits": { "order_status": -4.734320163726807, "refund_request": -6.311790466308594, "cancel_subscription": -7.196364879608154, "update_payment": -1.786896824836731, "login_problem": -3.4832663536071777, "password_reset": 3.359421730041504, "shipping_delay": -6.739846706390381, "missing_item": -7.409097671508789, "damaged_item": -8.064614295959473, "change_address": -7.644317150115967, "change_order": -6.473245143890381, "product_question": -5.4415788650512695, "pricing_question": -6.741420745849609, "upgrade_plan": -6.307089805603027, "downgrade_plan": -7.770752429962158, "bug_report": -6.239121913909912, "feature_request": -7.0143280029296875, "speak_to_human": -3.6695990562438965, "complaint": -6.544839859008789, "invoice_copy": -7.524979591369629, "tax_document": -8.526406288146973, "appointment": -6.465123176574707, "outage": -7.096297740936279, "data_export": -6.9752278327941895, "close_account": -6.222803115844727, "promo_code": -7.042648792266846, "partner_inquiry": -6.82668399810791, "other": -5.311529159545898 }, "probabilities": { "order_status": 0.00030266620225407004, "refund_request": 6.2499606014055e-05, "cancel_subscription": 2.580545334743803e-05, "update_payment": 0.005767845508810235, "login_problem": 0.0010575226886186183, "password_reset": 0.9909035169585358, "shipping_delay": 4.073566557879714e-05, "missing_item": 2.0860408821927083e-05, "damaged_item": 1.0830177204981003e-05, "change_address": 1.648801196290301e-05, "change_order": 5.318123323460558e-05, "product_question": 0.00014921255816058697, "pricing_question": 4.067159647065847e-05, "upgrade_plan": 6.279408704161333e-05, "downgrade_plan": 1.4529750256316047e-05, "bug_report": 6.721045429840153e-05, "feature_request": 3.09577297122868e-05, "speak_to_human": 0.000877741089095214, "complaint": 4.950684091095425e-05, "invoice_copy": 1.8577870751835987e-05, "tax_document": 6.824673022341915e-06, "appointment": 5.361492831728322e-05, "outage": 2.85213513887187e-05, "data_export": 3.21921581269933e-05, "close_account": 6.831624618769e-05, "promo_code": 3.0093281082561457e-05, "partner_inquiry": 3.734752786922713e-05, "other": 0.00016993594292375456 } }, { "domain": "support_intent", "row": 40, "task": "intent", "multi_label": false, "gold": [ "change_address" ], "selected": [ "change_address" ], "correct": true, "logits": { "order_status": -4.203387260437012, "refund_request": -8.169947624206543, "cancel_subscription": -5.483940601348877, "update_payment": -5.151953220367432, "login_problem": -5.922311782836914, "password_reset": -7.611243724822998, "shipping_delay": -5.733709335327148, "missing_item": -8.607081413269043, "damaged_item": -11.711304664611816, "change_address": 5.51249885559082, "change_order": -4.431805610656738, "product_question": -4.799135684967041, "pricing_question": -7.172919273376465, "upgrade_plan": -8.207106590270996, "downgrade_plan": -9.761561393737793, "bug_report": -8.16679859161377, "feature_request": -6.314681053161621, "speak_to_human": -5.179069519042969, "complaint": -7.990290641784668, "invoice_copy": -10.016608238220215, "tax_document": -12.220879554748535, "appointment": -6.735589981079102, "outage": -8.549565315246582, "data_export": -8.490715980529785, "close_account": -6.196490287780762, "promo_code": -10.383454322814941, "partner_inquiry": -5.978036403656006, "other": -4.083138465881348 }, "probabilities": { "order_status": 6.0297150913325804e-05, "refund_request": 1.141935340737987e-06, "cancel_subscription": 1.675558294977465e-05, "update_payment": 2.3352846772870087e-05, "login_problem": 1.0808796914284229e-05, "password_reset": 1.996565370821655e-06, "shipping_delay": 1.3052279321798648e-05, "missing_item": 7.375589227168208e-07, "damaged_item": 3.3086413468537776e-08, "change_address": 0.9996604697986525, "change_order": 4.7983946346974015e-05, "product_question": 3.323276974238549e-05, "pricing_question": 3.0948914757452076e-06, "upgrade_plan": 1.100280914751176e-06, "downgrade_plan": 2.3249436769425168e-07, "bug_report": 1.1455370002400985e-06, "feature_request": 7.300852056101918e-06, "speak_to_human": 2.272811252487517e-05, "complaint": 1.366675943210354e-06, "invoice_copy": 1.801552817307072e-07, "tax_document": 1.987669298906305e-08, "appointment": 4.792643477625589e-06, "outage": 7.812241232438865e-07, "data_export": 8.285783600079386e-07, "close_account": 8.216808117121554e-06, "promo_code": 1.2483252895944727e-07, "partner_inquiry": 1.0222955295816264e-05, "other": 6.800176417824748e-05 } }, { "domain": "support_intent", "row": 50, "task": "intent", "multi_label": false, "gold": [ "order_status" ], "selected": [ "order_status" ], "correct": true, "logits": { "order_status": 2.9918785095214844, "refund_request": -0.5019248127937317, "cancel_subscription": -4.992936611175537, "update_payment": -2.024937391281128, "login_problem": -6.232428550720215, "password_reset": -6.906068325042725, "shipping_delay": -3.5551421642303467, "missing_item": -6.426534175872803, "damaged_item": -7.954030990600586, "change_address": -7.829706192016602, "change_order": -5.338606834411621, "product_question": -4.222372055053711, "pricing_question": -3.2597711086273193, "upgrade_plan": -6.594976902008057, "downgrade_plan": -6.711283206939697, "bug_report": -5.996662139892578, "feature_request": -7.109330177307129, "speak_to_human": -4.326732635498047, "complaint": -4.666415214538574, "invoice_copy": -6.48930025100708, "tax_document": -7.56387996673584, "appointment": -6.130380630493164, "outage": -5.99195671081543, "data_export": -7.000283241271973, "close_account": -6.146318435668945, "promo_code": -6.60166072845459, "partner_inquiry": -6.4084086418151855, "other": -4.190824031829834 }, "probabilities": { "order_status": 0.9571309285353793, "refund_request": 0.029082507343269726, "cancel_subscription": 0.00032599444841079577, "update_payment": 0.006341556814982584, "login_problem": 9.438558991201932e-05, "password_reset": 4.812244199957989e-05, "shipping_delay": 0.0013728920768091059, "missing_item": 7.773322616257524e-05, "damaged_item": 1.6874202432492045e-05, "change_address": 1.9108070230401976e-05, "change_order": 0.00023072121692991994, "product_question": 0.0007044694088718875, "pricing_question": 0.001844651877436397, "upgrade_plan": 6.568299480240729e-05, "downgrade_plan": 5.847116671355544e-05, "bug_report": 0.0001194806965396067, "feature_request": 3.927101777713481e-05, "speak_to_human": 0.0006346567664524163, "complaint": 0.0004518732627838659, "invoice_copy": 7.30041808900939e-05, "tax_document": 2.492663713770253e-05, "appointment": 0.00010452605102994685, "outage": 0.0001200442292751546, "data_export": 4.3795616028984134e-05, "close_account": 0.00010287334046642768, "promo_code": 6.524544494721341e-05, "partner_inquiry": 7.91550289423645e-05, "other": 0.0007270483133864057 } }, { "domain": "support_intent", "row": 60, "task": "intent", "multi_label": false, "gold": [ "speak_to_human" ], "selected": [ "speak_to_human" ], "correct": true, "logits": { "order_status": -6.513601303100586, "refund_request": -1.837632417678833, "cancel_subscription": -7.604344367980957, "update_payment": -8.496052742004395, "login_problem": -7.377538681030273, "password_reset": -7.700960159301758, "shipping_delay": -9.100801467895508, "missing_item": -8.00849437713623, "damaged_item": -3.371328592300415, "change_address": -9.338873863220215, "change_order": -8.49479866027832, "product_question": -3.7571523189544678, "pricing_question": -6.171403408050537, "upgrade_plan": -8.840527534484863, "downgrade_plan": -8.492977142333984, "bug_report": -6.279052734375, "feature_request": -7.754677772521973, "speak_to_human": 2.0725409984588623, "complaint": -5.459669589996338, "invoice_copy": -9.375932693481445, "tax_document": -9.033928871154785, "appointment": -6.549628734588623, "outage": -7.236505508422852, "data_export": -7.727665901184082, "close_account": -7.433069705963135, "promo_code": -8.22385311126709, "partner_inquiry": -5.528470516204834, "other": -4.83676290512085 }, "probabilities": { "order_status": 0.0001810807195039796, "refund_request": 0.019436573095618796, "cancel_subscription": 6.0837101876068556e-05, "update_payment": 2.494046181346897e-05, "login_problem": 7.632538005465433e-05, "password_reset": 5.5234294622531924e-05, "shipping_delay": 1.3622771004538513e-05, "missing_item": 4.061144087769826e-05, "damaged_item": 0.004193183907307224, "change_address": 1.0736727448924769e-05, "change_order": 2.497175881126382e-05, "product_question": 0.0028509053232753266, "pricing_question": 0.0002549686996142909, "upgrade_plan": 1.7672622038162105e-05, "downgrade_plan": 2.501728677044927e-05, "bug_report": 0.00022894721268493964, "feature_request": 5.2345523702361236e-05, "speak_to_human": 0.9700328321879778, "complaint": 0.0005195040513514764, "invoice_copy": 1.0346119331468622e-05, "tax_document": 1.4564911699928571e-05, "appointment": 0.0001746729668013346, "outage": 8.78858392158192e-05, "data_export": 5.377874409245201e-05, "close_account": 7.220248713085342e-05, "promo_code": 3.274306185223272e-05, "partner_inquiry": 0.000484963525453797, "other": 0.0009685317780682526 } }, { "domain": "support_intent", "row": 70, "task": "intent", "multi_label": false, "gold": [ "invoice_copy" ], "selected": [ "tax_document" ], "correct": false, "logits": { "order_status": -5.3596343994140625, "refund_request": -6.054774284362793, "cancel_subscription": -6.648933410644531, "update_payment": -2.598698377609253, "login_problem": -6.025968551635742, "password_reset": -7.240901947021484, "shipping_delay": -6.963342189788818, "missing_item": -7.060949325561523, "damaged_item": -7.338775634765625, "change_address": -8.389089584350586, "change_order": -7.116280555725098, "product_question": -3.839367628097534, "pricing_question": -3.62807297706604, "upgrade_plan": -1.4474129676818848, "downgrade_plan": -5.9136481285095215, "bug_report": -3.4885268211364746, "feature_request": -5.992020606994629, "speak_to_human": -3.640990972518921, "complaint": -4.936382293701172, "invoice_copy": -1.0706796646118164, "tax_document": 1.1603353023529053, "appointment": -7.0094218254089355, "outage": -4.934514045715332, "data_export": -6.466073989868164, "close_account": -7.390020847320557, "promo_code": -7.714328765869141, "partner_inquiry": -5.515594005584717, "other": -4.5615386962890625 }, "probabilities": { "order_status": 0.0011754475748458528, "refund_request": 0.0005865537937652413, "cancel_subscription": 0.00032379327031987836, "update_payment": 0.01858927890603412, "login_problem": 0.0006036956115920722, "password_reset": 0.00017913446432930603, "shipping_delay": 0.0002364404734668813, "missing_item": 0.00021445273033060167, "damaged_item": 0.00016243258163878906, "change_address": 5.68234495019286e-05, "change_order": 0.00020290910344830716, "product_question": 0.0053758449472387886, "pricing_question": 0.006640653731471411, "upgrade_plan": 0.058784041885284535, "downgrade_plan": 0.0006754577030783102, "bug_report": 0.00763510406145249, "feature_request": 0.0006245416753152326, "speak_to_human": 0.006555421496828557, "complaint": 0.00179481346180814, "invoice_copy": 0.08567865470532983, "tax_document": 0.7976091508760411, "appointment": 0.00022579259187205196, "outage": 0.0017981697526587132, "data_export": 0.0003887609686829371, "close_account": 0.0001543183722886174, "promo_code": 0.00011157643841639413, "partner_inquiry": 0.001005705598938556, "other": 0.002611029774021431 } }, { "domain": "support_intent", "row": 80, "task": "intent", "multi_label": false, "gold": [ "update_payment" ], "selected": [ "update_payment" ], "correct": true, "logits": { "order_status": -5.155436038970947, "refund_request": -5.385929584503174, "cancel_subscription": -6.939935684204102, "update_payment": 4.292902946472168, "login_problem": -5.162038803100586, "password_reset": -6.354181289672852, "shipping_delay": -6.0758771896362305, "missing_item": -8.406773567199707, "damaged_item": -8.897976875305176, "change_address": -7.190630912780762, "change_order": -6.77241325378418, "product_question": -5.421619415283203, "pricing_question": -6.293161392211914, "upgrade_plan": -6.729935646057129, "downgrade_plan": -7.562501430511475, "bug_report": -5.722991943359375, "feature_request": -6.305751323699951, "speak_to_human": -2.9883017539978027, "complaint": -4.648770332336426, "invoice_copy": -6.4809417724609375, "tax_document": -9.254229545593262, "appointment": -6.735681533813477, "outage": -6.762672424316406, "data_export": -7.91070556640625, "close_account": -7.200671195983887, "promo_code": -7.436043739318848, "partner_inquiry": -6.340688705444336, "other": -4.920166969299316 }, "probabilities": { "order_status": 7.870089116683865e-05, "refund_request": 6.24996485433974e-05, "cancel_subscription": 1.3212387463322589e-05, "update_payment": 0.9984840566366885, "login_problem": 7.818295951759491e-05, "password_reset": 2.3734014454206878e-05, "shipping_delay": 3.1349970568913366e-05, "missing_item": 3.047485885082606e-06, "damaged_item": 1.8647250997227116e-06, "change_address": 1.0282666413541247e-05, "change_order": 1.5621954492427176e-05, "product_question": 6.030838221668236e-05, "pricing_question": 2.5227360148098953e-05, "upgrade_plan": 1.6299833154937884e-05, "downgrade_plan": 7.08931759814032e-06, "bug_report": 4.461626927967451e-05, "feature_request": 2.4911740396701078e-05, "speak_to_human": 0.0006873122930193917, "complaint": 0.00013062363665682243, "invoice_copy": 2.090835302645775e-05, "tax_document": 1.3058588831929826e-06, "appointment": 1.620644469990919e-05, "outage": 1.5774868834266026e-05, "data_export": 5.004737438334186e-06, "close_account": 1.017994208429009e-05, "promo_code": 8.044967805268722e-06, "partner_inquiry": 2.405641777921249e-05, "other": 9.957623668507307e-05 } }, { "domain": "support_intent", "row": 90, "task": "intent", "multi_label": false, "gold": [ "upgrade_plan" ], "selected": [ "upgrade_plan" ], "correct": true, "logits": { "order_status": -5.234085559844971, "refund_request": -3.0334415435791016, "cancel_subscription": -5.812528610229492, "update_payment": -3.3869330883026123, "login_problem": -7.0417375564575195, "password_reset": -7.869025707244873, "shipping_delay": -7.173951625823975, "missing_item": -7.348437786102295, "damaged_item": -7.792149543762207, "change_address": -8.261539459228516, "change_order": -5.480127334594727, "product_question": -4.827376365661621, "pricing_question": -3.4508824348449707, "upgrade_plan": 2.8660898208618164, "downgrade_plan": -4.398944854736328, "bug_report": -6.535708427429199, "feature_request": -7.351415634155273, "speak_to_human": -3.172208547592163, "complaint": -5.496082782745361, "invoice_copy": -9.335244178771973, "tax_document": -8.273920059204102, "appointment": -7.025714874267578, "outage": -5.833596229553223, "data_export": -8.210705757141113, "close_account": -6.937388896942139, "promo_code": -8.026123046875, "partner_inquiry": -6.111848831176758, "other": -4.470645427703857 }, "probabilities": { "order_status": 0.0002997780165984308, "refund_request": 0.002707243594241239, "cancel_subscription": 0.00016810675192101838, "update_payment": 0.0019011128945109433, "login_problem": 4.917524072216986e-05, "password_reset": 2.1501057257461372e-05, "shipping_delay": 4.308505520998985e-05, "missing_item": 3.618664144363869e-05, "damaged_item": 2.321917096567105e-05, "change_address": 1.4520890790263706e-05, "change_order": 0.00023439330179633844, "product_question": 0.00045022679751224984, "pricing_question": 0.0017833437299618986, "upgrade_plan": 0.9877823281278885, "downgrade_plan": 0.0006910298086078177, "bug_report": 8.15665611786351e-05, "feature_request": 3.607904340868254e-05, "speak_to_human": 0.002356468239257898, "complaint": 0.00023068312906682317, "invoice_copy": 4.962370932759408e-06, "tax_document": 1.4342221747257354e-05, "appointment": 4.996950611452159e-05, "outage": 0.00016460218888821767, "data_export": 1.527812487791418e-05, "close_account": 5.458389735378383e-05, "promo_code": 1.8375252179517424e-05, "partner_inquiry": 0.00012462123095785822, "other": 0.0006432171546084802 } }, { "domain": "support_topic", "row": 0, "task": "topic", "multi_label": false, "gold": [ "technical_outage" ], "selected": [ "account_access" ], "correct": false, "logits": { "billing": -0.5522791147232056, "account_access": 0.7107518315315247, "shipping": -3.8379273414611816, "returns": -3.3245043754577637, "technical_outage": 0.3603755235671997, "onboarding": -3.3029062747955322, "integrations": -3.533291816711426, "security": -3.1000144481658936, "compliance": -4.554596424102783, "pricing": -0.6100595593452454, "data_privacy": -5.060029029846191, "other": -2.687840700149536 }, "probabilities": { "billing": 0.11888898557575246, "account_access": 0.42040608349490405, "shipping": 0.004448388681829587, "returns": 0.0074332628124516775, "technical_outage": 0.29614369768138576, "onboarding": 0.0075955534469528884, "integrations": 0.006032596176118621, "security": 0.009304096899006836, "compliance": 0.0021724875642901856, "pricing": 0.11221421952033159, "data_privacy": 0.0013105412872570985, "other": 0.014050086859719185 } }, { "domain": "support_topic", "row": 10, "task": "topic", "multi_label": false, "gold": [ "onboarding" ], "selected": [ "technical_outage" ], "correct": false, "logits": { "billing": -0.4797506332397461, "account_access": -2.2023563385009766, "shipping": -4.719625949859619, "returns": -5.13369607925415, "technical_outage": 0.36451780796051025, "onboarding": -4.250681400299072, "integrations": -3.9016969203948975, "security": -3.7198874950408936, "compliance": -3.52447509765625, "pricing": -1.186299204826355, "data_privacy": -4.67002534866333, "other": -0.9672078490257263 }, "probabilities": { "billing": 0.20859718366538957, "account_access": 0.037255490729218375, "shipping": 0.003005757826556632, "returns": 0.0019866694538175, "technical_outage": 0.4852544936793914, "onboarding": 0.004804121879380464, "integrations": 0.006810453792892225, "security": 0.00816836020602035, "compliance": 0.009931192653467886, "pricing": 0.10291016979301945, "data_privacy": 0.003158604530931056, "other": 0.12811750178991507 } }, { "domain": "support_topic", "row": 20, "task": "topic", "multi_label": false, "gold": [ "technical_outage" ], "selected": [ "billing" ], "correct": false, "logits": { "billing": 0.7118673324584961, "account_access": -1.6983413696289062, "shipping": -2.8258111476898193, "returns": -1.6837973594665527, "technical_outage": -1.5042935609817505, "onboarding": -2.2883946895599365, "integrations": -3.236422300338745, "security": -2.9071788787841797, "compliance": -2.8814303874969482, "pricing": -0.76356440782547, "data_privacy": -3.2160778045654297, "other": -2.121169090270996 }, "probabilities": { "billing": 0.5715619369294329, "account_access": 0.051324291127076205, "shipping": 0.016621455707608257, "returns": 0.05207620681930214, "technical_outage": 0.06231561023655366, "onboarding": 0.028448938005542063, "integrations": 0.011024093774660923, "security": 0.015322565976094648, "compliance": 0.01572222212379455, "pricing": 0.130704769660432, "data_privacy": 0.011250670382360228, "other": 0.03362723925714244 } }, { "domain": "support_topic", "row": 30, "task": "topic", "multi_label": false, "gold": [ "billing" ], "selected": [ "billing" ], "correct": true, "logits": { "billing": 0.5697722434997559, "account_access": -1.763974666595459, "shipping": -3.0975770950317383, "returns": -3.6154580116271973, "technical_outage": -2.5924692153930664, "onboarding": -2.7885332107543945, "integrations": -3.1140565872192383, "security": -2.476257562637329, "compliance": -1.7864997386932373, "pricing": -2.7152609825134277, "data_privacy": -2.8536484241485596, "other": -1.9511626958847046 }, "probabilities": { "billing": 0.6524472929670099, "account_access": 0.06324293670837612, "shipping": 0.016666171733692597, "returns": 0.009929400501180356, "technical_outage": 0.027618584566459218, "onboarding": 0.022701361589799397, "integrations": 0.016393772353223063, "security": 0.031022121685042123, "compliance": 0.06183430927507143, "pricing": 0.024427196759449456, "data_privacy": 0.021270256560535977, "other": 0.05244659530016031 } }, { "domain": "support_topic", "row": 40, "task": "topic", "multi_label": false, "gold": [ "pricing" ], "selected": [ "pricing" ], "correct": true, "logits": { "billing": -0.6356263756752014, "account_access": -5.363243579864502, "shipping": -7.363761901855469, "returns": -7.952587127685547, "technical_outage": -6.064155578613281, "onboarding": -8.086786270141602, "integrations": 1.137995719909668, "security": -7.9888176918029785, "compliance": -5.953073501586914, "pricing": 1.737048864364624, "data_privacy": -3.3543639183044434, "other": -5.888942718505859 }, "probabilities": { "billing": 0.056463303202925315, "account_access": 0.0004995606444828345, "shipping": 6.757314758796427e-05, "returns": 3.750166954832412e-05, "technical_outage": 0.0002478483339280705, "onboarding": 3.279205665202868e-05, "integrations": 0.3326906309215135, "security": 3.616728168042509e-05, "compliance": 0.00027696719771469025, "pricing": 0.6056281393846132, "data_privacy": 0.003724204915974828, "other": 0.0002953112433787621 } }, { "domain": "support_topic", "row": 50, "task": "topic", "multi_label": false, "gold": [ "account_access" ], "selected": [ "billing" ], "correct": false, "logits": { "billing": 0.5388683080673218, "account_access": -2.1461894512176514, "shipping": 0.23809751868247986, "returns": -0.6679723858833313, "technical_outage": -3.705113172531128, "onboarding": -4.220236301422119, "integrations": -3.6575586795806885, "security": -4.185859680175781, "compliance": -4.729445934295654, "pricing": -3.7095937728881836, "data_privacy": -4.525164604187012, "other": -3.384657621383667 }, "probabilities": { "billing": 0.4545493170655699, "account_access": 0.03100810608266385, "shipping": 0.33647896188905885, "returns": 0.13597427618882693, "technical_outage": 0.006522938310412574, "onboarding": 0.0038969796588042616, "integrations": 0.006840627235167167, "security": 0.004033273898362842, "compliance": 0.0023419693401126513, "pricing": 0.006493777009543734, "data_privacy": 0.002872760757653253, "other": 0.008987012563823922 } }, { "domain": "support_topic", "row": 60, "task": "topic", "multi_label": false, "gold": [ "billing" ], "selected": [ "billing" ], "correct": true, "logits": { "billing": 1.2040841579437256, "account_access": -2.543665647506714, "shipping": -4.310200214385986, "returns": -2.8558549880981445, "technical_outage": -2.9584784507751465, "onboarding": -3.462702512741089, "integrations": -1.0134044885635376, "security": -3.252946615219116, "compliance": -3.645857334136963, "pricing": -2.093966484069824, "data_privacy": -3.868567943572998, "other": -2.348576307296753 }, "probabilities": { "billing": 0.7874036357000267, "account_access": 0.018559674516853236, "shipping": 0.003172299196340765, "returns": 0.013582766903661987, "technical_outage": 0.012257995122111703, "onboarding": 0.0074035108372201, "integrations": 0.08573424602479086, "security": 0.009131319636707802, "compliance": 0.006164453602696022, "pricing": 0.02909860846126905, "data_privacy": 0.004933698424502855, "other": 0.022557791573818906 } }, { "domain": "support_topic", "row": 70, "task": "topic", "multi_label": false, "gold": [ "technical_outage" ], "selected": [ "shipping" ], "correct": false, "logits": { "billing": -0.8628971576690674, "account_access": -3.320862293243408, "shipping": 0.04671566188335419, "returns": -4.025796890258789, "technical_outage": -1.589821696281433, "onboarding": -3.922858715057373, "integrations": -4.264143943786621, "security": -3.8401713371276855, "compliance": -4.46016788482666, "pricing": -1.9680149555206299, "data_privacy": -4.093713760375977, "other": -1.6646575927734375 }, "probabilities": { "billing": 0.19714328126133224, "account_access": 0.01687723453438804, "shipping": 0.4895779042501265, "returns": 0.008339731718681895, "technical_outage": 0.09529775391474936, "onboarding": 0.009243949441178278, "integrations": 0.006571118058182527, "security": 0.01004079807815596, "compliance": 0.005401410122416812, "pricing": 0.06528830883913249, "data_privacy": 0.007792129436886291, "other": 0.08842638034476963 } }, { "domain": "support_topic", "row": 80, "task": "topic", "multi_label": false, "gold": [ "pricing" ], "selected": [ "billing" ], "correct": false, "logits": { "billing": 0.39374682307243347, "account_access": -2.5125651359558105, "shipping": -6.598797798156738, "returns": -0.25151219964027405, "technical_outage": -6.141294956207275, "onboarding": -5.590350151062012, "integrations": -7.132987976074219, "security": 0.368577241897583, "compliance": -5.174820899963379, "pricing": -3.0509116649627686, "data_privacy": -5.75956916809082, "other": -3.947506904602051 }, "probabilities": { "billing": 0.3830458144347056, "account_access": 0.02094379964573083, "shipping": 0.0003519064100527616, "returns": 0.20091774174545943, "technical_outage": 0.0005560554956907753, "onboarding": 0.0009646958852081631, "integrations": 0.00020626774993365285, "security": 0.3735250313942958, "compliance": 0.0014616806136186899, "pricing": 0.012225159886497427, "data_privacy": 0.0008145158517394384, "other": 0.004987330887067489 } }, { "domain": "support_topic", "row": 90, "task": "topic", "multi_label": false, "gold": [ "returns" ], "selected": [ "returns" ], "correct": true, "logits": { "billing": 1.2613896131515503, "account_access": -3.086107015609741, "shipping": -2.40873122215271, "returns": 1.8175461292266846, "technical_outage": -3.3264918327331543, "onboarding": -6.073574066162109, "integrations": -1.9713855981826782, "security": -5.190095901489258, "compliance": -4.681256294250488, "pricing": -4.493731498718262, "data_privacy": -6.409843444824219, "other": -2.9318580627441406 }, "probabilities": { "billing": 0.35019193886934985, "account_access": 0.0045311907885327634, "shipping": 0.008920576420480948, "returns": 0.6107195896064901, "technical_outage": 0.003562989554831652, "onboarding": 0.00022843986449178267, "integrations": 0.013814326275855262, "security": 0.0005526645213584741, "compliance": 0.000919280016151009, "pricing": 0.0011088908583676887, "data_privacy": 0.0001632044357407813, "other": 0.005286908788349732 } }, { "domain": "ticket_route", "row": 0, "task": "queue", "multi_label": false, "gold": [ "compliance" ], "selected": [ "compliance" ], "correct": true, "logits": { "billing_disputes": -4.767746925354004, "refunds": -6.231613636016846, "shipping": -7.0399980545043945, "returns": -6.818154335021973, "identity": -5.868091583251953, "payments": -3.8340349197387695, "outages": -5.614709854125977, "mobile_app": -6.311042785644531, "api": -6.522747039794922, "account_closure": -5.805496692657471, "compliance": 3.119619131088257, "fraud_review": -5.46289587020874, "onboarding": -5.618372440338135, "partnerships": -6.863076210021973, "product_feedback": -5.775279998779297, "general": -5.336216926574707 }, "probabilities": { "billing_disputes": 0.00037440431238158805, "refunds": 8.661470042145622e-05, "shipping": 3.8593548820474636e-05, "returns": 4.8179270357758195e-05, "identity": 0.0001245854224533324, "payments": 0.0009524601494382794, "outages": 0.00016051274281080592, "mobile_app": 8.000110142245488e-05, "api": 6.473720974744057e-05, "account_closure": 0.00013263307689791775, "compliance": 0.9971957005426185, "fraud_review": 0.00018682779641364787, "onboarding": 0.00015992592634105613, "partnerships": 4.6062859658914773e-05, "product_feedback": 0.00013670197471344614, "general": 0.00021205936550300163 } }, { "domain": "ticket_route", "row": 0, "task": "urgency", "multi_label": false, "gold": [ "low" ], "selected": [ "normal" ], "correct": false, "logits": { "low": -1.0406419038772583, "normal": -0.8218404054641724, "high": -0.8667811751365662, "critical": -1.5192376375198364 }, "probabilities": { "low": 0.24666222985412034, "normal": 0.3069919140692024, "high": 0.2935008804317572, "critical": 0.15284497564492003 } }, { "domain": "ticket_route", "row": 0, "task": "contains_pii", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.5450303554534912, "no": 0.6630094051361084 }, "probabilities": { "yes": 0.23004807649205938, "no": 0.7699519235079406 } }, { "domain": "ticket_route", "row": 10, "task": "queue", "multi_label": false, "gold": [ "payments" ], "selected": [ "payments" ], "correct": true, "logits": { "billing_disputes": -4.458004951477051, "refunds": -4.367234706878662, "shipping": -5.394424915313721, "returns": -4.719562530517578, "identity": -4.871467113494873, "payments": 2.023822546005249, "outages": -4.024564743041992, "mobile_app": -5.165653705596924, "api": -4.882457733154297, "account_closure": -4.302916526794434, "compliance": -5.159491539001465, "fraud_review": -5.169976711273193, "onboarding": -4.169389724731445, "partnerships": -5.785766124725342, "product_feedback": -4.69548225402832, "general": -4.0538225173950195 }, "probabilities": { "billing_disputes": 0.0015019257520735444, "refunds": 0.0016446348173012693, "shipping": 0.0005887981553770097, "returns": 0.0011562598634647472, "identity": 0.0009933084462205165, "payments": 0.9810030973453083, "outages": 0.0023168032542688647, "mobile_app": 0.0007401513144450311, "api": 0.0009824511444119033, "account_closure": 0.0017538906417642526, "compliance": 0.0007447263316827035, "fraud_review": 0.0007369585422698146, "onboarding": 0.002004437246216934, "partnerships": 0.0003981155242680451, "product_feedback": 0.001184440862050259, "general": 0.002250000758876637 } }, { "domain": "ticket_route", "row": 10, "task": "urgency", "multi_label": false, "gold": [ "critical" ], "selected": [ "critical" ], "correct": true, "logits": { "low": -2.210422992706299, "normal": -2.2345597743988037, "high": -0.48591113090515137, "critical": -0.018829286098480225 }, "probabilities": { "low": 0.06047631592952214, "normal": 0.059034087686101515, "high": 0.3392589492746644, "critical": 0.5412306471097119 } }, { "domain": "ticket_route", "row": 10, "task": "contains_pii", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.19479815661907196, "no": 0.3291495144367218 }, "probabilities": { "yes": 0.37192960046450885, "no": 0.6280703995354913 } }, { "domain": "ticket_route", "row": 20, "task": "queue", "multi_label": false, "gold": [ "account_closure" ], "selected": [ "account_closure" ], "correct": true, "logits": { "billing_disputes": -6.416181564331055, "refunds": -7.268327713012695, "shipping": -9.081807136535645, "returns": -7.9404497146606445, "identity": -7.189838409423828, "payments": -5.182271957397461, "outages": -6.880102157592773, "mobile_app": -7.788702964782715, "api": -7.278958320617676, "account_closure": 4.888782978057861, "compliance": -6.788288116455078, "fraud_review": -7.127537727355957, "onboarding": -6.193844795227051, "partnerships": -8.236096382141113, "product_feedback": -7.53516960144043, "general": -6.438440799713135 }, "probabilities": { "billing_disputes": 1.2310012260316478e-05, "refunds": 5.250203236025682e-06, "shipping": 8.562330921072501e-07, "returns": 2.6808791608747316e-06, "identity": 5.6788917034780705e-06, "payments": 4.228035439630941e-05, "outages": 7.740701703800875e-06, "mobile_app": 3.120182631071772e-06, "api": 5.194685999387849e-06, "account_closure": 0.9998669294391512, "compliance": 8.485055028369994e-06, "fraud_review": 6.043943964670282e-06, "onboarding": 1.5375105958273217e-05, "partnerships": 1.9947088869177356e-06, "product_feedback": 4.02057488670891e-06, "general": 1.2039027940443e-05 } }, { "domain": "ticket_route", "row": 20, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "normal" ], "correct": true, "logits": { "low": -0.8974087834358215, "normal": -0.7921150922775269, "high": -1.1538794040679932, "critical": -1.6003873348236084 }, "probabilities": { "low": 0.2958647290645891, "normal": 0.32871662079191066, "high": 0.2289335376515211, "critical": 0.14648511249197915 } }, { "domain": "ticket_route", "row": 20, "task": "contains_pii", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.588455080986023, "no": 0.7085678577423096 }, "probabilities": { "yes": 0.21466647784133255, "no": 0.7853335221586674 } }, { "domain": "ticket_route", "row": 30, "task": "queue", "multi_label": false, "gold": [ "returns" ], "selected": [ "refunds" ], "correct": false, "logits": { "billing_disputes": -3.8116486072540283, "refunds": 1.1502304077148438, "shipping": -4.96439266204834, "returns": -2.175548791885376, "identity": -4.740817546844482, "payments": -2.5749471187591553, "outages": -4.483358860015869, "mobile_app": -4.825428009033203, "api": -4.747352600097656, "account_closure": -4.673920631408691, "compliance": -4.61240816116333, "fraud_review": -4.343784332275391, "onboarding": -4.626736640930176, "partnerships": -5.30058479309082, "product_feedback": -4.3507232666015625, "general": -3.9982988834381104 }, "probabilities": { "billing_disputes": 0.006330807495816272, "refunds": 0.9044317170184304, "shipping": 0.0019990733330354114, "returns": 0.032509345760800605, "identity": 0.0024999203088582325, "payments": 0.02180478155206231, "outages": 0.00323399282241365, "mobile_app": 0.0022971021475956563, "api": 0.0024836364623658544, "account_closure": 0.002672877935399744, "compliance": 0.002842455355897729, "fraud_review": 0.003718394798865886, "onboarding": 0.0028020176888265, "partnerships": 0.0014283095450417106, "product_feedback": 0.0036926824130045217, "general": 0.005252885361585587 } }, { "domain": "ticket_route", "row": 30, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "high" ], "correct": false, "logits": { "low": -2.0601212978363037, "normal": -1.7897851467132568, "high": -0.17775557935237885, "critical": -0.6351694464683533 }, "probabilities": { "low": 0.07670423686666668, "normal": 0.10051360559909883, "high": 0.5038722001534138, "critical": 0.3189099573808207 } }, { "domain": "ticket_route", "row": 30, "task": "contains_pii", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.14962495863437653, "no": 0.23200754821300507 }, "probabilities": { "yes": 0.4057332166830156, "no": 0.5942667833169845 } }, { "domain": "ticket_route", "row": 40, "task": "queue", "multi_label": false, "gold": [ "identity" ], "selected": [ "identity" ], "correct": true, "logits": { "billing_disputes": -4.945812225341797, "refunds": -8.558162689208984, "shipping": -12.993834495544434, "returns": -9.883313179016113, "identity": 4.519521236419678, "payments": -5.854539394378662, "outages": -7.855839729309082, "mobile_app": -10.196314811706543, "api": -10.525528907775879, "account_closure": -7.643986701965332, "compliance": -7.184526443481445, "fraud_review": -6.305644989013672, "onboarding": -7.950765132904053, "partnerships": -7.902010917663574, "product_feedback": -7.103607177734375, "general": -6.708662986755371 }, "probabilities": { "billing_disputes": 7.747825411600655e-05, "refunds": 2.091009243538469e-06, "shipping": 2.477240454587133e-08, "returns": 5.55712809008396e-07, "identity": 0.9998202248931398, "payments": 3.122659501017495e-05, "outages": 4.220568357989601e-06, "mobile_app": 4.0636427953318565e-07, "api": 2.923746130647984e-07, "account_closure": 5.216479913305711e-06, "compliance": 8.258831286592337e-06, "fraud_review": 1.9888954776074737e-05, "onboarding": 3.8383569497792e-06, "partnerships": 4.030129915809992e-06, "product_feedback": 8.954913289416335e-06, "general": 1.3291789895395859e-05 } }, { "domain": "ticket_route", "row": 40, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "high" ], "correct": false, "logits": { "low": -1.7095357179641724, "normal": -0.9208477735519409, "high": -0.5495415925979614, "critical": -1.2134305238723755 }, "probabilities": { "low": 0.12449061257682298, "normal": 0.2739425071591547, "high": 0.3971144147142843, "critical": 0.20445246554973792 } }, { "domain": "ticket_route", "row": 40, "task": "contains_pii", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 2.2801156044006348, "no": -2.208752155303955 }, "probabilities": { "yes": 0.9888914310343306, "no": 0.011108568965669457 } }, { "domain": "ticket_route", "row": 50, "task": "queue", "multi_label": false, "gold": [ "shipping" ], "selected": [ "shipping" ], "correct": true, "logits": { "billing_disputes": -4.542666435241699, "refunds": -5.240078926086426, "shipping": 0.7392980456352234, "returns": -4.336411476135254, "identity": -5.202234268188477, "payments": -4.348140239715576, "outages": -2.336308717727661, "mobile_app": -4.555428504943848, "api": -3.342426300048828, "account_closure": -5.148918151855469, "compliance": -4.885896682739258, "fraud_review": -4.251340389251709, "onboarding": -4.452849388122559, "partnerships": -5.288628578186035, "product_feedback": -3.2607359886169434, "general": -2.7109320163726807 }, "probabilities": { "billing_disputes": 0.00437416249026108, "refunds": 0.0021777725313909405, "shipping": 0.8606427996999413, "returns": 0.00537613669499156, "identity": 0.002261768971223318, "payments": 0.005313449598409964, "outages": 0.03972865787868563, "mobile_app": 0.004318693824012704, "api": 0.014526218744262062, "account_closure": 0.002385630271177307, "compliance": 0.0031033582239306428, "fraud_review": 0.005853507865583259, "onboarding": 0.004785220559614934, "partnerships": 0.002074567976155137, "product_feedback": 0.015762686251125306, "general": 0.027315368419234976 } }, { "domain": "ticket_route", "row": 50, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "normal" ], "correct": true, "logits": { "low": -0.9948704838752747, "normal": -0.6922186017036438, "high": -0.8291046619415283, "critical": -1.6170654296875 }, "probabilities": { "low": 0.24566977129585527, "normal": 0.3325000874757832, "high": 0.28996321292258864, "critical": 0.13186692830577287 } }, { "domain": "ticket_route", "row": 50, "task": "contains_pii", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.2969661951065063, "no": 1.5306870937347412 }, "probabilities": { "yes": 0.05584800878588963, "no": 0.9441519912141103 } }, { "domain": "ticket_route", "row": 60, "task": "queue", "multi_label": false, "gold": [ "identity" ], "selected": [ "identity" ], "correct": true, "logits": { "billing_disputes": -5.000199317932129, "refunds": -5.40068244934082, "shipping": -5.232804298400879, "returns": -5.371731281280518, "identity": 1.436538815498352, "payments": -2.3755404949188232, "outages": -4.760354042053223, "mobile_app": -4.723570346832275, "api": -3.3615286350250244, "account_closure": -5.519169330596924, "compliance": -4.843202590942383, "fraud_review": -4.47158145904541, "onboarding": -4.552881240844727, "partnerships": -5.3187713623046875, "product_feedback": -4.874526500701904, "general": -4.503817558288574 }, "probabilities": { "billing_disputes": 0.0015207122165554908, "refunds": 0.0010188715152484105, "shipping": 0.0012051135660316287, "returns": 0.0010488001802092196, "identity": 0.9494823316956418, "payments": 0.020985623575762898, "outages": 0.0019329050230110693, "mobile_app": 0.0020053282433232155, "api": 0.007829114958914795, "account_closure": 0.0009050263460713761, "compliance": 0.001779220799958008, "fraud_review": 0.002580018701438025, "onboarding": 0.002378563813245125, "partnerships": 0.0011058416776638406, "product_feedback": 0.0017243524796039054, "general": 0.0024981752073212253 } }, { "domain": "ticket_route", "row": 60, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "normal" ], "correct": true, "logits": { "low": -0.9193361401557922, "normal": -0.591458797454834, "high": -0.9815137982368469, "critical": -1.6894752979278564 }, "probabilities": { "low": 0.2638046830069513, "normal": 0.36616583612153303, "high": 0.2479014614032445, "critical": 0.12212801946827119 } }, { "domain": "ticket_route", "row": 60, "task": "contains_pii", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 2.077592134475708, "no": -1.85466730594635 }, "probabilities": { "yes": 0.9807774110868888, "no": 0.01922258891311122 } }, { "domain": "ticket_route", "row": 70, "task": "queue", "multi_label": false, "gold": [ "account_closure" ], "selected": [ "account_closure" ], "correct": true, "logits": { "billing_disputes": -7.5145344734191895, "refunds": -8.024880409240723, "shipping": -9.59329605102539, "returns": -8.668187141418457, "identity": -7.374898910522461, "payments": -5.385883331298828, "outages": -7.280911445617676, "mobile_app": -8.874541282653809, "api": -8.168136596679688, "account_closure": 4.778309345245361, "compliance": -7.713844299316406, "fraud_review": -8.119589805603027, "onboarding": -6.876421928405762, "partnerships": -9.023036003112793, "product_feedback": -7.982568740844727, "general": -6.6864824295043945 }, "probabilities": { "billing_disputes": 4.584013492404578e-06, "refunds": 2.751727749534244e-06, "shipping": 5.733914396055794e-07, "returns": 1.446175166094824e-06, "identity": 5.270949334342584e-06, "payments": 3.8521867416582775e-05, "outages": 5.790380178055904e-06, "mobile_app": 1.1765284532405926e-06, "api": 2.3844602763747796e-06, "account_closure": 0.9999081876926275, "compliance": 3.755663986496974e-06, "fraud_review": 2.5030740351890075e-06, "onboarding": 8.677100835824313e-06, "partnerships": 1.0141728894829193e-06, "product_feedback": 2.870656230428065e-06, "general": 1.0492145888995627e-05 } }, { "domain": "ticket_route", "row": 70, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "normal" ], "correct": true, "logits": { "low": -0.46278196573257446, "normal": -0.4048696756362915, "high": -1.6588656902313232, "critical": -2.2147505283355713 }, "probabilities": { "low": 0.39441036401905105, "normal": 0.41792591936478857, "high": 0.11926026143350558, "critical": 0.06840345518265477 } }, { "domain": "ticket_route", "row": 70, "task": "contains_pii", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.1549756526947021, "no": 1.285603404045105 }, "probabilities": { "yes": 0.08013021988811539, "no": 0.9198697801118845 } }, { "domain": "ticket_route", "row": 80, "task": "queue", "multi_label": false, "gold": [ "payments" ], "selected": [ "payments" ], "correct": true, "logits": { "billing_disputes": -4.492818832397461, "refunds": -4.086909294128418, "shipping": -5.706459999084473, "returns": -3.412310838699341, "identity": -5.7881693840026855, "payments": 2.0154669284820557, "outages": -5.122769355773926, "mobile_app": -6.251284122467041, "api": -5.742589473724365, "account_closure": -5.77417516708374, "compliance": -5.463891506195068, "fraud_review": -5.384148597717285, "onboarding": -5.122934341430664, "partnerships": -6.347397327423096, "product_feedback": -4.979047775268555, "general": -4.930567741394043 }, "probabilities": { "billing_disputes": 0.0014690739789281712, "refunds": 0.0022045905372282976, "shipping": 0.0004364816643292688, "returns": 0.004328150219426764, "identity": 0.0004022351964620161, "payments": 0.9852722676781474, "outages": 0.0007824554685466888, "mobile_app": 0.00025313482322146216, "api": 0.000420993289295208, "account_closure": 0.0004079037339157756, "compliance": 0.0005563039742698046, "fraud_review": 0.0006024819880986872, "onboarding": 0.0007823263852660799, "partnerships": 0.00022993784653337537, "product_feedback": 0.000903393828782099, "general": 0.0009482693875490107 } }, { "domain": "ticket_route", "row": 80, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "high" ], "correct": false, "logits": { "low": -2.102153778076172, "normal": -1.882369041442871, "high": 0.1774630844593048, "critical": -0.9722520709037781 }, "probabilities": { "low": 0.06616340104870368, "normal": 0.08242692905515565, "high": 0.6466106386346, "critical": 0.2047990312615407 } }, { "domain": "ticket_route", "row": 80, "task": "contains_pii", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.44986188411712646, "no": 0.5327073335647583 }, "probabilities": { "yes": 0.2723822922762549, "no": 0.7276177077237451 } }, { "domain": "ticket_route", "row": 90, "task": "queue", "multi_label": false, "gold": [ "shipping" ], "selected": [ "shipping" ], "correct": true, "logits": { "billing_disputes": -4.571217060089111, "refunds": -4.765566825866699, "shipping": 1.411437749862671, "returns": -4.39089298248291, "identity": -4.464351654052734, "payments": -4.006231784820557, "outages": -3.6828432083129883, "mobile_app": -4.306344032287598, "api": -4.323209285736084, "account_closure": -4.73533296585083, "compliance": -4.505581855773926, "fraud_review": -4.6971635818481445, "onboarding": -4.167115211486816, "partnerships": -4.9548187255859375, "product_feedback": -3.7663984298706055, "general": -3.3449857234954834 }, "probabilities": { "billing_disputes": 0.0023922138689402585, "refunds": 0.0019696768158151395, "shipping": 0.9484926612565694, "returns": 0.002864928288867723, "identity": 0.002662018443057615, "payments": 0.004208913411927359, "outages": 0.005815885738407526, "mobile_app": 0.003117689772406081, "api": 0.0030655500548614884, "account_closure": 0.0020301371176681044, "compliance": 0.0025544947293313683, "fraud_review": 0.0021091240527673395, "onboarding": 0.003583432323855234, "partnerships": 0.0016300612269949373, "product_feedback": 0.005349686025707472, "general": 0.008153526872822942 } }, { "domain": "ticket_route", "row": 90, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "normal" ], "correct": true, "logits": { "low": -0.4591774344444275, "normal": -0.39930659532546997, "high": -1.8022913932800293, "critical": -2.0640554428100586 }, "probabilities": { "low": 0.3962522413774748, "normal": 0.4207007699536852, "high": 0.1034343407816771, "critical": 0.07961264788716296 } }, { "domain": "ticket_route", "row": 90, "task": "contains_pii", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.7764317989349365, "no": 0.905678391456604 }, "probabilities": { "yes": 0.1568162470431322, "no": 0.8431837529568679 } }, { "domain": "travel_request", "row": 0, "task": "intent", "multi_label": false, "gold": [ "hotel" ], "selected": [ "hotel" ], "correct": true, "logits": { "new_booking": -3.6798462867736816, "change_flight": -5.980119228363037, "cancel_trip": -6.719210624694824, "seat": -2.820553779602051, "baggage": -5.571332931518555, "refund": -6.051131248474121, "hotel": 0.5747595429420471, "car": -5.250791072845459, "visa_docs": -0.8198679685592651, "loyalty": -4.928055763244629, "disruption": -5.355851173400879, "other": -3.8183796405792236 }, "probabilities": { "new_booking": 0.010730003505350283, "change_flight": 0.0010754841601681706, "cancel_trip": 0.0005135949010120522, "seat": 0.025338789172438234, "baggage": 0.0016185929761087978, "refund": 0.0010017604694390076, "hotel": 0.7557039606480548, "car": 0.002230217461884341, "visa_docs": 0.18735818293428314, "loyalty": 0.003079706828362612, "disruption": 0.0020077987981853957, "other": 0.009341908144713258 } }, { "domain": "travel_request", "row": 10, "task": "intent", "multi_label": false, "gold": [ "cancel_trip" ], "selected": [ "cancel_trip" ], "correct": true, "logits": { "new_booking": -5.3618998527526855, "change_flight": -7.5974016189575195, "cancel_trip": 0.5474910736083984, "seat": -9.465726852416992, "baggage": -8.72346305847168, "refund": -0.3205936551094055, "hotel": -3.7107574939727783, "car": -9.091673851013184, "visa_docs": -11.487988471984863, "loyalty": -9.950817108154297, "disruption": -6.616144180297852, "other": -5.057530403137207 }, "probabilities": { "new_booking": 0.0018825228960689428, "change_flight": 0.00020131409619104793, "cancel_trip": 0.6936751515381915, "seat": 3.107927264397842e-05, "baggage": 6.528778221921903e-05, "refund": 0.2911734217947028, "hotel": 0.00981346283870387, "car": 4.517727179257153e-05, "visa_docs": 4.113521617846407e-06, "loyalty": 1.9133694040772615e-05, "disruption": 0.000537067505213759, "other": 0.002552267788613734 } }, { "domain": "travel_request", "row": 20, "task": "intent", "multi_label": false, "gold": [ "new_booking" ], "selected": [ "visa_docs" ], "correct": false, "logits": { "new_booking": -2.790898561477661, "change_flight": -4.422411918640137, "cancel_trip": -5.405909061431885, "seat": -6.7078351974487305, "baggage": -1.1307663917541504, "refund": -6.899700164794922, "hotel": -6.181307792663574, "car": -5.556093215942383, "visa_docs": -0.1538442224264145, "loyalty": -7.025164604187012, "disruption": -5.338569641113281, "other": -2.303650379180908 }, "probabilities": { "new_booking": 0.044733991897317464, "change_flight": 0.008751457873413136, "cancel_trip": 0.0032730528995846622, "seat": 0.0008902944945702768, "baggage": 0.23530106632271178, "refund": 0.0007348653854729737, "hotel": 0.0015073067161481101, "car": 0.00281662399615756, "visa_docs": 0.6250227173332354, "loyalty": 0.0006482152998546018, "disruption": 0.0035010487907997973, "other": 0.07281935899073412 } }, { "domain": "travel_request", "row": 30, "task": "intent", "multi_label": false, "gold": [ "car" ], "selected": [ "car" ], "correct": true, "logits": { "new_booking": -2.955901622772217, "change_flight": -3.7242538928985596, "cancel_trip": -6.538210391998291, "seat": -6.388094425201416, "baggage": -6.02665376663208, "refund": -4.033542633056641, "hotel": -5.135909080505371, "car": 2.925750494003296, "visa_docs": -6.991560459136963, "loyalty": -6.738690376281738, "disruption": -3.5819711685180664, "other": -4.179001808166504 }, "probabilities": { "new_booking": 0.0027678286621772583, "change_flight": 0.001283654215219604, "cancel_trip": 7.697722863421753e-05, "seat": 8.944515220393586e-05, "baggage": 0.0001283891994724109, "refund": 0.0009421621608479023, "hotel": 0.00031287725515924603, "car": 0.9919922086917491, "visa_docs": 4.891869229467006e-05, "loyalty": 6.299338127905034e-05, "disruption": 0.0014799282259070209, "other": 0.0008146171350555282 } }, { "domain": "travel_request", "row": 40, "task": "intent", "multi_label": false, "gold": [ "visa_docs" ], "selected": [ "visa_docs" ], "correct": true, "logits": { "new_booking": -4.464197158813477, "change_flight": -5.594268798828125, "cancel_trip": -7.317447185516357, "seat": -6.725396156311035, "baggage": -6.318550109863281, "refund": -7.477623462677002, "hotel": -5.031625270843506, "car": -5.209400177001953, "visa_docs": 3.780771255493164, "loyalty": -5.493616104125977, "disruption": -5.721653461456299, "other": -4.35848331451416 }, "probabilities": { "new_booking": 0.0002622675345156666, "change_flight": 8.471506652242839e-05, "cancel_trip": 1.51214623171368e-05, "seat": 2.733495016914924e-05, "baggage": 4.105908618839708e-05, "refund": 1.2883388946233177e-05, "hotel": 0.0001487009132257086, "car": 0.00012448211969529844, "visa_docs": 0.9988236556525707, "loyalty": 9.368575608888664e-05, "disruption": 7.458271696015255e-05, "other": 0.0002915113528001775 } }, { "domain": "travel_request", "row": 50, "task": "intent", "multi_label": false, "gold": [ "baggage" ], "selected": [ "baggage" ], "correct": true, "logits": { "new_booking": -3.663616418838501, "change_flight": -5.8715314865112305, "cancel_trip": -6.158261299133301, "seat": -5.894170761108398, "baggage": 2.231682777404785, "refund": -5.882806777954102, "hotel": -4.921265125274658, "car": -4.457190990447998, "visa_docs": -5.979581832885742, "loyalty": -5.54732608795166, "disruption": -5.434723377227783, "other": -2.950164794921875 }, "probabilities": { "new_booking": 0.0027178894005561167, "change_flight": 0.00029877650970702843, "cancel_trip": 0.00022429586856253565, "seat": 0.00029208841868049335, "baggage": 0.9874785940844859, "refund": 0.000295426638361291, "hotel": 0.0007727551117228639, "car": 0.00122909859584289, "visa_docs": 0.00026817653822187886, "loyalty": 0.0004131873959576466, "disruption": 0.0004624340462187319, "other": 0.0055472773916825785 } }, { "domain": "travel_request", "row": 60, "task": "intent", "multi_label": false, "gold": [ "disruption" ], "selected": [ "hotel" ], "correct": false, "logits": { "new_booking": -3.8691649436950684, "change_flight": -4.973991394042969, "cancel_trip": -5.473467826843262, "seat": -5.990563869476318, "baggage": -5.5420823097229, "refund": -5.981935501098633, "hotel": 0.5721499919891357, "car": -4.956390380859375, "visa_docs": -5.480414390563965, "loyalty": -0.7622740268707275, "disruption": -2.760852336883545, "other": -3.1689913272857666 }, "probabilities": { "new_booking": 0.008712408104909492, "change_flight": 0.0028861452209161726, "cancel_trip": 0.0017514523278450063, "seat": 0.0010443026089223643, "baggage": 0.0016353075039957574, "refund": 0.001053352222083651, "hotel": 0.7395657312079938, "car": 0.002937393993109671, "visa_docs": 0.001739327912911323, "loyalty": 0.19473489978823708, "disruption": 0.02639199739818789, "other": 0.01754768171088785 } }, { "domain": "travel_request", "row": 70, "task": "intent", "multi_label": false, "gold": [ "seat" ], "selected": [ "seat" ], "correct": true, "logits": { "new_booking": -4.399460792541504, "change_flight": -5.60944938659668, "cancel_trip": -8.5125150680542, "seat": 2.67915940284729, "baggage": -7.338000774383545, "refund": -8.099763870239258, "hotel": -6.284708023071289, "car": -3.224048137664795, "visa_docs": -9.978038787841797, "loyalty": -8.939416885375977, "disruption": -6.9565749168396, "other": -3.341486692428589 }, "probabilities": { "new_booking": 0.0008374602802347207, "change_flight": 0.0002497312256009205, "cancel_trip": 1.3698955108549042e-05, "seat": 0.9935046599272256, "baggage": 4.433763452227872e-05, "refund": 2.0698697142017902e-05, "hotel": 0.00012711937498184747, "car": 0.0027129353722451367, "visa_docs": 3.163869616725309e-06, "loyalty": 8.938946518560544e-06, "disruption": 6.49267500212693e-05, "other": 0.0024123289667824506 } }, { "domain": "travel_request", "row": 80, "task": "intent", "multi_label": false, "gold": [ "loyalty" ], "selected": [ "loyalty" ], "correct": true, "logits": { "new_booking": -3.2085533142089844, "change_flight": -4.201442718505859, "cancel_trip": -4.350618362426758, "seat": -6.9776411056518555, "baggage": -5.94756555557251, "refund": -5.619534969329834, "hotel": -5.641039848327637, "car": -6.163082599639893, "visa_docs": -7.801720142364502, "loyalty": 1.662906527519226, "disruption": -4.764927387237549, "other": -2.088224172592163 }, "probabilities": { "new_booking": 0.007363487926915163, "change_flight": 0.0027282061881754188, "cancel_trip": 0.0023501253693199555, "seat": 0.00016989850190639254, "baggage": 0.0004759328444624809, "refund": 0.0006607049344055613, "hotel": 0.0006466482402090027, "car": 0.000383661140791947, "visa_docs": 7.452407104124073e-05, "loyalty": 0.9610184094340214, "disruption": 0.0015529551288913724, "other": 0.022575446219860133 } }, { "domain": "travel_request", "row": 90, "task": "intent", "multi_label": false, "gold": [ "loyalty" ], "selected": [ "loyalty" ], "correct": true, "logits": { "new_booking": -4.26422119140625, "change_flight": -4.480396747589111, "cancel_trip": -5.198518753051758, "seat": -6.070472240447998, "baggage": -0.6916331648826599, "refund": -6.147486209869385, "hotel": -5.888052940368652, "car": -5.287261009216309, "visa_docs": -8.716522216796875, "loyalty": 1.50166916847229, "disruption": -3.7113447189331055, "other": -3.2196872234344482 }, "probabilities": { "new_booking": 0.002758657204159857, "change_flight": 0.002222357312343982, "cancel_trip": 0.0010837708410918034, "seat": 0.000453161362673589, "baggage": 0.09823200390699442, "refund": 0.0004195716465318982, "hotel": 0.0005438467899825057, "car": 0.0009917385382978206, "visa_docs": 3.21431156222944e-05, "loyalty": 0.8806272177100982, "disruption": 0.004795224436404517, "other": 0.007840307135799095 } } ], "q8_0": [ { "domain": "agent_handoff", "row": 0, "task": "should_handoff", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.5828102827072144, "no": 0.6424565315246582 }, "probabilities": { "yes": 0.2270109174425549, "no": 0.7729890825574451 } }, { "domain": "agent_handoff", "row": 10, "task": "should_handoff", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.2630865275859833, "no": 0.3196696639060974 }, "probabilities": { "yes": 0.3582986409267614, "no": 0.6417013590732386 } }, { "domain": "agent_handoff", "row": 20, "task": "should_handoff", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 0.6703857183456421, "no": -0.7047131657600403 }, "probabilities": { "yes": 0.7982027059810666, "no": 0.2017972940189334 } }, { "domain": "agent_handoff", "row": 30, "task": "should_handoff", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 2.0899603366851807, "no": -1.9880611896514893 }, "probabilities": { "yes": 0.9833412650505433, "no": 0.016658734949456706 } }, { "domain": "agent_handoff", "row": 40, "task": "should_handoff", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 2.0804128646850586, "no": -2.0459296703338623 }, "probabilities": { "yes": 0.9841146098445402, "no": 0.015885390155459773 } }, { "domain": "agent_handoff", "row": 50, "task": "should_handoff", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 1.5272674560546875, "no": -1.5791685581207275 }, "probabilities": { "yes": 0.9571574450511374, "no": 0.042842554948862674 } }, { "domain": "agent_handoff", "row": 60, "task": "should_handoff", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 1.9759256839752197, "no": -2.101545810699463 }, "probabilities": { "yes": 0.9833322524642608, "no": 0.016667747535739245 } }, { "domain": "agent_handoff", "row": 70, "task": "should_handoff", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 1.4008821249008179, "no": -1.4450359344482422 }, "probabilities": { "yes": 0.945107298130638, "no": 0.054892701869362036 } }, { "domain": "agent_handoff", "row": 80, "task": "should_handoff", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.3039378523826599, "no": 0.30777281522750854 }, "probabilities": { "yes": 0.3516690703384849, "no": 0.6483309296615151 } }, { "domain": "agent_handoff", "row": 90, "task": "should_handoff", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.0347436666488647, "no": 1.0889298915863037 }, "probabilities": { "yes": 0.10681708004368462, "no": 0.8931829199563154 } }, { "domain": "banking_intent", "row": 0, "task": "intent", "multi_label": false, "gold": [ "other" ], "selected": [ "wire" ], "correct": false, "logits": { "balance": -5.699422836303711, "transfer": -4.186276435852051, "card_lost": -8.742565155029297, "dispute_charge": -6.803252696990967, "loan_payment": -1.8487176895141602, "mortgage": -5.695123195648193, "wire": 1.3827797174453735, "statement": -6.8236284255981445, "fraud_alert": -6.932866096496582, "branch_appointment": -5.083902835845947, "fee_refund": -9.100729942321777, "direct_deposit": -7.259383201599121, "close_account": -6.73349142074585, "new_account": -4.046730041503906, "limit_increase": -7.990968227386475, "other": -3.5038955211639404 }, "probabilities": { "balance": 0.0007924486092566799, "transfer": 0.003598506069914651, "card_lost": 3.778776369924481e-05, "dispute_charge": 0.0002627749064227737, "loan_payment": 0.037265856238752136, "mortgage": 0.0007958631889869458, "wire": 0.9434797784108045, "statement": 0.00025747485580855856, "fraud_alert": 0.00023083066669253187, "branch_appointment": 0.0014665201928545238, "fee_refund": 2.641205543711929e-05, "direct_deposit": 0.0001665286375602651, "close_account": 0.0002817609659016954, "new_account": 0.004137390078013097, "limit_increase": 8.012454737206148e-05, "other": 0.007119942812523218 } }, { "domain": "banking_intent", "row": 10, "task": "intent", "multi_label": false, "gold": [ "branch_appointment" ], "selected": [ "branch_appointment" ], "correct": true, "logits": { "balance": -3.448636770248413, "transfer": -5.897235870361328, "card_lost": -7.4569292068481445, "dispute_charge": -6.076606750488281, "loan_payment": -7.3945770263671875, "mortgage": -8.93787670135498, "wire": -9.06495189666748, "statement": -7.0860595703125, "fraud_alert": -8.172863960266113, "branch_appointment": 5.098908424377441, "fee_refund": -8.696866035461426, "direct_deposit": -6.301844596862793, "close_account": -7.129538536071777, "new_account": -6.1064300537109375, "limit_increase": -8.669795036315918, "other": -4.5995588302612305 }, "probabilities": { "balance": 0.0001939561075870069, "transfer": 1.676063167529771e-05, "card_lost": 3.5230935296858937e-06, "dispute_charge": 1.400846659590183e-05, "loan_payment": 3.749759212027466e-06, "mortgage": 8.012293468889988e-07, "wire": 7.056166042221634e-07, "statement": 5.104941967878346e-06, "fraud_alert": 1.721859297495702e-06, "branch_appointment": 0.9996665761731897, "fee_refund": 1.0195920728250482e-06, "direct_deposit": 1.1183327546514137e-05, "close_account": 4.887740435619588e-06, "new_account": 1.3596856148273485e-05, "limit_increase": 1.0475704415392745e-06, "other": 6.135703434907743e-05 } }, { "domain": "banking_intent", "row": 20, "task": "intent", "multi_label": false, "gold": [ "other" ], "selected": [ "fraud_alert" ], "correct": false, "logits": { "balance": -3.2425498962402344, "transfer": -3.4533419609069824, "card_lost": -3.1002755165100098, "dispute_charge": -2.7327303886413574, "loan_payment": -3.5019071102142334, "mortgage": -4.413367748260498, "wire": -4.096543312072754, "statement": -3.6288199424743652, "fraud_alert": -0.877204954624176, "branch_appointment": -3.67885422706604, "fee_refund": -3.956519842147827, "direct_deposit": -1.3867130279541016, "close_account": -3.5584568977355957, "new_account": -2.833294153213501, "limit_increase": -4.207164287567139, "other": -1.6335917711257935 }, "probabilities": { "balance": 0.030666174842233374, "transfer": 0.024837837244731497, "card_lost": 0.03535481633737148, "dispute_charge": 0.05105889437326239, "loan_payment": 0.02366040643437521, "mortgage": 0.009509985943483876, "wire": 0.013054942982613754, "statement": 0.02084033317965769, "fraud_alert": 0.3265245678939295, "branch_appointment": 0.019823258492915236, "fee_refund": 0.015017111304814604, "direct_deposit": 0.19617303841118777, "close_account": 0.022359543876680075, "new_account": 0.04617395957078665, "limit_increase": 0.011687803464703736, "other": 0.1532573256472531 } }, { "domain": "banking_intent", "row": 30, "task": "intent", "multi_label": false, "gold": [ "limit_increase" ], "selected": [ "mortgage" ], "correct": false, "logits": { "balance": -5.641988754272461, "transfer": -8.82060718536377, "card_lost": -11.170132637023926, "dispute_charge": -8.355326652526855, "loan_payment": -5.703666687011719, "mortgage": 2.6413791179656982, "wire": -9.951775550842285, "statement": -7.696523666381836, "fraud_alert": -8.888798713684082, "branch_appointment": -6.357243537902832, "fee_refund": -9.976357460021973, "direct_deposit": -5.651732444763184, "close_account": -9.026737213134766, "new_account": -5.847254753112793, "limit_increase": -2.426055908203125, "other": -4.759380340576172 }, "probabilities": { "balance": 0.00025066316380454164, "transfer": 1.0438403297535645e-05, "card_lost": 9.959743031790652e-07, "dispute_charge": 1.6622748731828732e-05, "loan_payment": 0.00023566990734890747, "mortgage": 0.9919995505955622, "wire": 3.3680140380222495e-06, "statement": 3.2123092013982056e-05, "fraud_alert": 9.750319966892445e-06, "branch_appointment": 0.00012259120423970226, "fee_refund": 3.286231131026508e-06, "direct_deposit": 0.0002482326398847714, "close_account": 8.494013337685914e-06, "new_account": 0.00020414776342532015, "limit_increase": 0.006248163796902747, "other": 0.000605902132011605 } }, { "domain": "banking_intent", "row": 40, "task": "intent", "multi_label": false, "gold": [ "statement" ], "selected": [ "statement" ], "correct": true, "logits": { "balance": -6.843557357788086, "transfer": -6.033507347106934, "card_lost": -6.087825298309326, "dispute_charge": -7.45326566696167, "loan_payment": -5.250552177429199, "mortgage": -5.828832626342773, "wire": -6.591139793395996, "statement": 3.3224663734436035, "fraud_alert": -6.846717834472656, "branch_appointment": -6.986335754394531, "fee_refund": -7.745936393737793, "direct_deposit": -6.5021162033081055, "close_account": -8.2964506149292, "new_account": -6.051815986633301, "limit_increase": -8.227184295654297, "other": -3.8996431827545166 }, "probabilities": { "balance": 3.839550808913241e-05, "transfer": 8.631388579434663e-05, "card_lost": 8.175054982024106e-05, "dispute_charge": 2.0868318898283544e-05, "loan_payment": 0.0001888486038571793, "mortgage": 0.0001059179997368952, "wire": 4.9420140340025135e-05, "statement": 0.9984548441576668, "fraud_alert": 3.8274351538100644e-05, "branch_appointment": 3.328683852267678e-05, "fee_refund": 1.5573354981905658e-05, "direct_deposit": 5.402147379489768e-05, "close_account": 8.980425091387689e-06, "new_account": 8.47479745670093e-05, "limit_increase": 9.624515469154465e-06, "other": 0.0007291319018319316 } }, { "domain": "banking_intent", "row": 50, "task": "intent", "multi_label": false, "gold": [ "loan_payment" ], "selected": [ "loan_payment" ], "correct": true, "logits": { "balance": -4.788570404052734, "transfer": -6.145255088806152, "card_lost": -5.475679874420166, "dispute_charge": -4.302163124084473, "loan_payment": 2.15857195854187, "mortgage": -5.205526351928711, "wire": -5.831744194030762, "statement": -4.6732635498046875, "fraud_alert": -4.258639812469482, "branch_appointment": -5.868562698364258, "fee_refund": -6.009283065795898, "direct_deposit": -5.092615127563477, "close_account": -5.877417087554932, "new_account": -5.052148818969727, "limit_increase": -3.9548275470733643, "other": -3.109630584716797 }, "probabilities": { "balance": 0.0009455982245649131, "transfer": 0.00024350393055973872, "card_lost": 0.0004756623713678924, "dispute_charge": 0.0015379798507594864, "loan_payment": 0.9835857795536587, "mortgage": 0.0006231964643735605, "wire": 0.0003331670377560767, "statement": 0.0010611671044340563, "fraud_alert": 0.0016063958734219073, "branch_appointment": 0.0003211234005873966, "fee_refund": 0.00027897023940476817, "direct_deposit": 0.0006976887215048103, "close_account": 0.00031829260001992456, "new_account": 0.0007265006328889815, "limit_increase": 0.0021766899406947256, "other": 0.005068284054003067 } }, { "domain": "banking_intent", "row": 60, "task": "intent", "multi_label": false, "gold": [ "card_lost" ], "selected": [ "mortgage" ], "correct": false, "logits": { "balance": -3.4911372661590576, "transfer": -6.455164432525635, "card_lost": -1.9248316287994385, "dispute_charge": -1.0089256763458252, "loan_payment": -5.138460159301758, "mortgage": 1.2066130638122559, "wire": -6.683987617492676, "statement": -5.626028060913086, "fraud_alert": -3.403430938720703, "branch_appointment": -5.749159336090088, "fee_refund": -4.389988899230957, "direct_deposit": -6.272994041442871, "close_account": -5.86065673828125, "new_account": -5.4595513343811035, "limit_increase": -4.486268043518066, "other": -4.358551979064941 }, "probabilities": { "balance": 0.007659999037019754, "transfer": 0.00039533755542220617, "card_lost": 0.03668314904011597, "dispute_charge": 0.09167259208625482, "loan_payment": 0.0014750456889839007, "mortgage": 0.8403027033117956, "wire": 0.0003144788378639402, "statement": 0.0009058523668470542, "fraud_alert": 0.008362171862114243, "branch_appointment": 0.0008009071855136364, "fee_refund": 0.003117901642150993, "direct_deposit": 0.00047433335814743544, "close_account": 0.0007164064438395418, "new_account": 0.0010699348832165585, "limit_increase": 0.002831710881284556, "other": 0.003217475819429802 } }, { "domain": "banking_intent", "row": 70, "task": "intent", "multi_label": false, "gold": [ "wire" ], "selected": [ "mortgage" ], "correct": false, "logits": { "balance": -5.42426872253418, "transfer": -2.186994791030884, "card_lost": -4.08613395690918, "dispute_charge": -7.3150715827941895, "loan_payment": -1.4389780759811401, "mortgage": 0.06133613362908363, "wire": -3.0854389667510986, "statement": -5.866118431091309, "fraud_alert": -6.42620849609375, "branch_appointment": -4.527095794677734, "fee_refund": -6.681816101074219, "direct_deposit": -4.003491401672363, "close_account": -6.545990943908691, "new_account": -3.4132492542266846, "limit_increase": -7.40092658996582, "other": -3.2443127632141113 }, "probabilities": { "balance": 0.0027742302213705065, "transfer": 0.07064358056530909, "card_lost": 0.010575162345525182, "dispute_charge": 0.0004187716275701747, "loan_payment": 0.14925615001661796, "mortgage": 0.6691298702185654, "wire": 0.02876625695671045, "statement": 0.001783403473596559, "fraud_alert": 0.001018604483871055, "branch_appointment": 0.006804241984038628, "fee_refund": 0.0007888539622069595, "direct_deposit": 0.011486249586966804, "close_account": 0.0009036176962991693, "new_account": 0.020726079350838488, "limit_increase": 0.0003843181489929322, "other": 0.024540609361520707 } }, { "domain": "banking_intent", "row": 80, "task": "intent", "multi_label": false, "gold": [ "other" ], "selected": [ "wire" ], "correct": false, "logits": { "balance": -4.895227432250977, "transfer": -4.497491836547852, "card_lost": -6.719567775726318, "dispute_charge": -6.492221832275391, "loan_payment": -5.697565078735352, "mortgage": -6.487039566040039, "wire": 2.286149740219116, "statement": -4.16501522064209, "fraud_alert": -6.521812438964844, "branch_appointment": -6.391259670257568, "fee_refund": -8.229117393493652, "direct_deposit": -6.889995098114014, "close_account": -6.657087326049805, "new_account": -5.325785160064697, "limit_increase": -6.284188270568848, "other": -2.9700381755828857 }, "probabilities": { "balance": 0.0007525489532733377, "transfer": 0.001120131809473036, "card_lost": 0.00012140422802167326, "dispute_charge": 0.00015239436212937602, "loan_payment": 0.000337352508251485, "mortgage": 0.00015318616016878392, "wire": 0.989389357462639, "statement": 0.0015619311640209152, "fraud_alert": 0.00014795098581349368, "branch_appointment": 0.00016858394416320704, "fee_refund": 2.6831487040585587e-05, "direct_deposit": 0.00010238071684663616, "close_account": 0.00012923160147396075, "new_account": 0.0004892669844133559, "limit_increase": 0.00018763624290553377, "other": 0.005159811389365616 } }, { "domain": "banking_intent", "row": 90, "task": "intent", "multi_label": false, "gold": [ "statement" ], "selected": [ "statement" ], "correct": true, "logits": { "balance": -6.504275321960449, "transfer": -6.301660060882568, "card_lost": -5.099118232727051, "dispute_charge": -6.936932563781738, "loan_payment": -5.571768760681152, "mortgage": -1.599638819694519, "wire": -0.9978215098381042, "statement": 0.3697398602962494, "fraud_alert": -5.976150989532471, "branch_appointment": -6.044467926025391, "fee_refund": -6.878159046173096, "direct_deposit": -5.8629584312438965, "close_account": -7.384793281555176, "new_account": -5.621331691741943, "limit_increase": -7.451752662658691, "other": -4.553177833557129 }, "probabilities": { "balance": 0.0007279999214814251, "transfer": 0.0008915095962612227, "card_lost": 0.002967449265356455, "dispute_charge": 0.0004723138488032786, "loan_payment": 0.0018497531960288006, "mortgage": 0.09821727963972675, "wire": 0.1792890796842673, "statement": 0.703846887822955, "fraud_alert": 0.0012345048935482423, "branch_appointment": 0.0011529836459726241, "fee_refund": 0.0005009053775736936, "direct_deposit": 0.001382457278177209, "close_account": 0.0003018055635571297, "new_account": 0.0017603088763493424, "limit_increase": 0.00028225857957454945, "other": 0.005122502810366964 } }, { "domain": "benefits_request", "row": 0, "task": "program", "multi_label": false, "gold": [ "unemployment" ], "selected": [ "unemployment" ], "correct": true, "logits": { "unemployment": 6.173270225524902, "health_coverage": -6.025373458862305, "food_assistance": -6.997886657714844, "housing_aid": -6.618961334228516, "childcare": -8.192373275756836, "pension": -7.359552383422852, "disability": -5.957762718200684, "tax_credit": -7.677806854248047, "other": -5.639659881591797 }, "probabilities": { "unemployment": 0.9999746098977269, "health_coverage": 5.0371552245428406e-06, "food_assistance": 1.9047071777565144e-06, "housing_aid": 2.7822323464428526e-06, "childcare": 5.768584669895831e-07, "pension": 1.3266574171307215e-06, "disability": 5.389497885023371e-06, "tax_credit": 9.650340317205912e-07, "other": 7.407959723399385e-06 } }, { "domain": "benefits_request", "row": 0, "task": "asking_status", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.009840909391641617, "no": 0.23920944333076477 }, "probabilities": { "yes": 0.4380572534604289, "no": 0.5619427465395711 } }, { "domain": "benefits_request", "row": 10, "task": "program", "multi_label": false, "gold": [ "unemployment" ], "selected": [ "unemployment" ], "correct": true, "logits": { "unemployment": 5.386534214019775, "health_coverage": -6.727139949798584, "food_assistance": -6.0159196853637695, "housing_aid": -3.6689774990081787, "childcare": -7.192368030548096, "pension": -6.120594501495361, "disability": -1.6931670904159546, "tax_credit": -6.872840881347656, "other": -2.461669445037842 }, "probabilities": { "unemployment": 0.9986177954282848, "health_coverage": 5.476428930095079e-06, "food_assistance": 1.1152609405032053e-05, "housing_aid": 0.00011658442448397107, "childcare": 3.439152768067249e-06, "pension": 1.0044233429854589e-05, "disability": 0.0008408607695598884, "tax_credit": 4.733913798467317e-06, "other": 0.00038991303933980727 } }, { "domain": "benefits_request", "row": 10, "task": "asking_status", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 0.4226960837841034, "no": -0.3517155945301056 }, "probabilities": { "yes": 0.6844744547659586, "no": 0.3155255452340414 } }, { "domain": "benefits_request", "row": 20, "task": "program", "multi_label": false, "gold": [ "pension" ], "selected": [ "pension" ], "correct": true, "logits": { "unemployment": -5.46671199798584, "health_coverage": -6.425232410430908, "food_assistance": -6.312115669250488, "housing_aid": -5.893877983093262, "childcare": -8.184181213378906, "pension": 1.2296113967895508, "disability": 0.03149084374308586, "tax_credit": -6.906347274780273, "other": -3.594848155975342 }, "probabilities": { "unemployment": 0.0009407818764787777, "health_coverage": 0.0003607520573316894, "food_assistance": 0.0004039566788924356, "housing_aid": 0.0006137240074988212, "childcare": 6.213081599240142e-05, "pension": 0.7614918188458294, "disability": 0.22978839776996518, "tax_credit": 0.00022297865206278715, "other": 0.006115459295948514 } }, { "domain": "benefits_request", "row": 20, "task": "asking_status", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.6151318550109863, "no": 1.6758813858032227 }, "probabilities": { "yes": 0.03588077800271815, "no": 0.964119221997282 } }, { "domain": "benefits_request", "row": 30, "task": "program", "multi_label": false, "gold": [ "housing_aid" ], "selected": [ "housing_aid" ], "correct": true, "logits": { "unemployment": -5.42097282409668, "health_coverage": -5.817879676818848, "food_assistance": -5.766143321990967, "housing_aid": 3.3906102180480957, "childcare": -6.653801441192627, "pension": -6.090868949890137, "disability": -5.813676834106445, "tax_credit": -5.151769161224365, "other": -4.042332172393799 }, "probabilities": { "unemployment": 0.00014879462480348298, "health_coverage": 0.00010004900794534655, "food_assistance": 0.00010536141634131552, "housing_aid": 0.9986404073791463, "childcare": 4.33687172718867e-05, "pension": 7.61473951799318e-05, "disability": 0.00010047038305566853, "tax_credit": 0.00019476051215555797, "other": 0.0005906405641004405 } }, { "domain": "benefits_request", "row": 30, "task": "asking_status", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.08526597917079926, "no": 0.07124152034521103 }, "probabilities": { "yes": 0.4609527963957622, "no": 0.5390472036042379 } }, { "domain": "benefits_request", "row": 40, "task": "program", "multi_label": false, "gold": [ "housing_aid" ], "selected": [ "housing_aid" ], "correct": true, "logits": { "unemployment": -5.07028865814209, "health_coverage": -7.974592208862305, "food_assistance": -7.238130569458008, "housing_aid": 2.1830086708068848, "childcare": -8.48800277709961, "pension": -2.852874755859375, "disability": -4.863448619842529, "tax_credit": -7.924783229827881, "other": -3.2900326251983643 }, "probabilities": { "unemployment": 0.0006991247980708197, "health_coverage": 3.830290391664014e-05, "food_assistance": 7.999685752737032e-05, "housing_aid": 0.9876924010241345, "childcare": 2.292241253732744e-05, "pension": 0.006420447938442691, "disability": 0.0008597737752513409, "tax_credit": 4.0259044738679526e-05, "other": 0.004146771245380624 } }, { "domain": "benefits_request", "row": 40, "task": "asking_status", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.526228129863739, "no": 0.5993120670318604 }, "probabilities": { "yes": 0.24498508055793275, "no": 0.7550149194420672 } }, { "domain": "benefits_request", "row": 50, "task": "program", "multi_label": false, "gold": [ "tax_credit" ], "selected": [ "tax_credit" ], "correct": true, "logits": { "unemployment": -2.9596548080444336, "health_coverage": -7.4614739418029785, "food_assistance": -6.880287170410156, "housing_aid": -6.798680305480957, "childcare": -4.111812591552734, "pension": -6.2720417976379395, "disability": -7.109752655029297, "tax_credit": 3.5931572914123535, "other": -4.429104804992676 }, "probabilities": { "unemployment": 0.0014227505181901264, "health_coverage": 1.577660470695674e-05, "food_assistance": 2.8211082560361266e-05, "housing_aid": 3.060984729647581e-05, "childcare": 0.00044952410476036344, "pension": 5.182954078332708e-05, "disability": 2.2426637236597293e-05, "tax_credit": 0.9976515650911537, "other": 0.00032730657331211585 } }, { "domain": "benefits_request", "row": 50, "task": "asking_status", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 0.18115851283073425, "no": -0.16160376369953156 }, "probabilities": { "yes": 0.5848613564960394, "no": 0.4151386435039605 } }, { "domain": "benefits_request", "row": 60, "task": "program", "multi_label": false, "gold": [ "food_assistance" ], "selected": [ "food_assistance" ], "correct": true, "logits": { "unemployment": -3.890183448791504, "health_coverage": -5.131766319274902, "food_assistance": 2.308544158935547, "housing_aid": -5.646421909332275, "childcare": -4.638933181762695, "pension": -5.390480995178223, "disability": -5.544183731079102, "tax_credit": -2.676297426223755, "other": -3.550487995147705 }, "probabilities": { "unemployment": 0.002003034776085255, "health_coverage": 0.0005787298723688804, "food_assistance": 0.9857384209575102, "housing_aid": 0.0003459105573856758, "childcare": 0.0009473503329605834, "pension": 0.000446804505523965, "disability": 0.00038314688033287686, "tax_credit": 0.006743300146698777, "other": 0.0028133019711338737 } }, { "domain": "benefits_request", "row": 60, "task": "asking_status", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 0.8984341025352478, "no": -0.8296298384666443 }, "probabilities": { "yes": 0.84916460952563, "no": 0.15083539047437 } }, { "domain": "benefits_request", "row": 70, "task": "program", "multi_label": false, "gold": [ "unemployment" ], "selected": [ "unemployment" ], "correct": true, "logits": { "unemployment": 5.905807971954346, "health_coverage": -8.318185806274414, "food_assistance": -7.875299453735352, "housing_aid": -7.112399578094482, "childcare": -8.307280540466309, "pension": -7.250997066497803, "disability": -8.462526321411133, "tax_credit": -6.1771464347839355, "other": -5.175283432006836 }, "probabilities": { "unemployment": 0.9999718461425412, "health_coverage": 6.646389125010487e-07, "food_assistance": 1.0349726259610848e-06, "housing_aid": 2.2194842601823753e-06, "childcare": 6.719266415955584e-07, "pension": 1.932235002423522e-06, "disability": 5.753067572016123e-07, "tax_credit": 5.65493150368515e-06, "other": 1.5400361755326593e-05 } }, { "domain": "benefits_request", "row": 70, "task": "asking_status", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.8244362473487854, "no": 0.8829647302627563 }, "probabilities": { "yes": 0.15350112486595155, "no": 0.8464988751340485 } }, { "domain": "benefits_request", "row": 80, "task": "program", "multi_label": false, "gold": [ "disability" ], "selected": [ "disability" ], "correct": true, "logits": { "unemployment": -0.6635106801986694, "health_coverage": -7.320143222808838, "food_assistance": -7.13438606262207, "housing_aid": -6.935514450073242, "childcare": -8.70240592956543, "pension": -7.842189788818359, "disability": 1.1133906841278076, "tax_credit": -5.047186374664307, "other": -3.799956798553467 }, "probabilities": { "unemployment": 0.1434054883990228, "health_coverage": 0.00018434314460803497, "food_assistance": 0.00022197307646796123, "housing_aid": 0.0002708127736277241, "childcare": 4.627196316133017e-05, "pension": 0.00010937172169452781, "disability": 0.8477430350063857, "tax_credit": 0.0017896149881920238, "other": 0.006229088926839896 } }, { "domain": "benefits_request", "row": 80, "task": "asking_status", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.4479612112045288, "no": 0.45973044633865356 }, "probabilities": { "yes": 0.287472427665098, "no": 0.712527572334902 } }, { "domain": "benefits_request", "row": 90, "task": "program", "multi_label": false, "gold": [ "pension" ], "selected": [ "pension" ], "correct": true, "logits": { "unemployment": -5.824977397918701, "health_coverage": -6.691289901733398, "food_assistance": -7.558176040649414, "housing_aid": -5.053135395050049, "childcare": -6.6959333419799805, "pension": 4.885353088378906, "disability": -7.6067986488342285, "tax_credit": -7.217507839202881, "other": -5.113541126251221 }, "probabilities": { "unemployment": 2.230993453845492e-05, "health_coverage": 9.381311485348196e-06, "food_assistance": 3.942572509844873e-06, "housing_aid": 4.827308104966974e-05, "childcare": 9.337850907419467e-06, "pension": 0.9998520135450261, "disability": 3.7554601588672906e-06, "tax_credit": 5.5428102250011255e-06, "other": 4.544343409940917e-05 } }, { "domain": "benefits_request", "row": 90, "task": "asking_status", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 0.31566527485847473, "no": -0.3766179084777832 }, "probabilities": { "yes": 0.6664746396437681, "no": 0.33352536035623187 } }, { "domain": "clinic_request", "row": 0, "task": "request", "multi_label": false, "gold": [ "other" ], "selected": [ "provider_message" ], "correct": false, "logits": { "appointment": -1.5701828002929688, "refill": -5.875881195068359, "lab_results": -6.399631023406982, "referral": -1.7458059787750244, "billing": -8.221795082092285, "insurance": -7.5664567947387695, "records": -4.062856674194336, "symptom_question": -1.0146657228469849, "cancel_visit": -3.910578966140747, "provider_message": 0.29619497060775757, "other": -1.8462175130844116 }, "probabilities": { "appointment": 0.09082949303034779, "refill": 0.0012254224616550736, "lab_results": 0.0007258120535082992, "referral": 0.07619995464590995, "billing": 0.00011734602435052944, "insurance": 0.0002259842554088118, "records": 0.007510561065263057, "symptom_question": 0.15830144948620012, "cancel_visit": 0.00874592502206981, "provider_message": 0.5871978506034704, "other": 0.06892020135181609 } }, { "domain": "clinic_request", "row": 0, "task": "urgent", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -1.8227910995483398, "no": 1.9042013883590698 }, "probabilities": { "yes": 0.023499583301361603, "no": 0.9765004166986383 } }, { "domain": "clinic_request", "row": 10, "task": "request", "multi_label": false, "gold": [ "provider_message" ], "selected": [ "cancel_visit" ], "correct": false, "logits": { "appointment": -0.46373698115348816, "refill": -3.4559922218322754, "lab_results": -4.345073223114014, "referral": -3.6466097831726074, "billing": -4.510951519012451, "insurance": -5.043734073638916, "records": -3.898831844329834, "symptom_question": -1.9896382093429565, "cancel_visit": 0.8746070265769958, "provider_message": -2.1582014560699463, "other": -2.0825273990631104 }, "probabilities": { "appointment": 0.17907676846619955, "refill": 0.008985025401521103, "lab_results": 0.0036931448060426456, "referral": 0.00742566160666793, "billing": 0.00312864514605154, "insurance": 0.0018364190291233562, "records": 0.00577027491425827, "symptom_question": 0.03893576978351166, "cancel_visit": 0.6827703704164383, "provider_message": 0.03289596852723508, "other": 0.0354819519029506 } }, { "domain": "clinic_request", "row": 10, "task": "urgent", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.0349018573760986, "no": 1.1525840759277344 }, "probabilities": { "yes": 0.10087989861647077, "no": 0.8991201013835292 } }, { "domain": "clinic_request", "row": 20, "task": "request", "multi_label": false, "gold": [ "provider_message" ], "selected": [ "records" ], "correct": false, "logits": { "appointment": -2.6477441787719727, "refill": -0.9450633525848389, "lab_results": -6.308887958526611, "referral": -4.639873027801514, "billing": -6.02116584777832, "insurance": -2.0882246494293213, "records": 0.48505523800849915, "symptom_question": -2.611020088195801, "cancel_visit": -3.3988595008850098, "provider_message": -2.3705077171325684, "other": -1.2658452987670898 }, "probabilities": { "appointment": 0.026188884928141438, "refill": 0.14374140801422033, "lab_results": 0.0006731354555197882, "referral": 0.0035722878052205807, "billing": 0.0008975498763417267, "insurance": 0.04582613725245064, "records": 0.6007233414283859, "symptom_question": 0.027168526026892775, "cancel_visit": 0.012356963610332793, "provider_message": 0.03455566641564135, "other": 0.10429609918685258 } }, { "domain": "clinic_request", "row": 20, "task": "urgent", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -2.677983522415161, "no": 2.745427131652832 }, "probabilities": { "yes": 0.004392692024933548, "no": 0.9956073079750664 } }, { "domain": "clinic_request", "row": 30, "task": "request", "multi_label": false, "gold": [ "provider_message" ], "selected": [ "symptom_question" ], "correct": false, "logits": { "appointment": -4.4646501541137695, "refill": -5.399402141571045, "lab_results": -4.907787799835205, "referral": -5.098601818084717, "billing": -8.460128784179688, "insurance": -8.16673469543457, "records": -4.257126808166504, "symptom_question": 0.021676212549209595, "cancel_visit": -4.742191314697266, "provider_message": -0.47137317061424255, "other": -1.2102564573287964 }, "probabilities": { "appointment": 0.005762778575213485, "refill": 0.002262946584361694, "lab_results": 0.003699812357604008, "referral": 0.0030571040713596536, "billing": 0.00010602727781044447, "insurance": 0.00014217952043968696, "records": 0.007091827720816076, "symptom_question": 0.5117038893320964, "cancel_visit": 0.004366136794220136, "provider_message": 0.31252883390752384, "other": 0.14927846385855448 } }, { "domain": "clinic_request", "row": 30, "task": "urgent", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.4630933403968811, "no": 0.523311972618103 }, "probabilities": { "yes": 0.27162268005200035, "no": 0.7283773199479996 } }, { "domain": "clinic_request", "row": 40, "task": "request", "multi_label": false, "gold": [ "cancel_visit" ], "selected": [ "other" ], "correct": false, "logits": { "appointment": -0.7182997465133667, "refill": -3.9609923362731934, "lab_results": -7.272498607635498, "referral": -0.7530621886253357, "billing": -6.211994171142578, "insurance": -8.097643852233887, "records": -2.97365140914917, "symptom_question": -1.813455581665039, "cancel_visit": -2.5856101512908936, "provider_message": -2.962982654571533, "other": 0.2902568578720093 }, "probabilities": { "appointment": 0.18340146704744398, "refill": 0.007163401723524726, "lab_results": 0.000261186308102404, "referral": 0.1771355248739646, "billing": 0.0007542609633088133, "insurance": 0.00011444435613144365, "records": 0.019227199099164333, "symptom_question": 0.061345494170547514, "cancel_visit": 0.02834263347283912, "provider_message": 0.019433427511615642, "other": 0.5028209604733574 } }, { "domain": "clinic_request", "row": 40, "task": "urgent", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -3.67272686958313, "no": 3.7985541820526123 }, "probabilities": { "yes": 0.0005688748539510805, "no": 0.9994311251460489 } }, { "domain": "clinic_request", "row": 50, "task": "request", "multi_label": false, "gold": [ "referral" ], "selected": [ "referral" ], "correct": true, "logits": { "appointment": -1.2361781597137451, "refill": -4.147697925567627, "lab_results": -4.787657737731934, "referral": 4.358955383300781, "billing": -7.086928367614746, "insurance": -6.3467559814453125, "records": -4.826939582824707, "symptom_question": -3.470862865447998, "cancel_visit": -6.675673961639404, "provider_message": -3.0181217193603516, "other": -3.7604899406433105 }, "probabilities": { "appointment": 0.0036955879128361107, "refill": 0.0002010141221908272, "lab_results": 0.00010599748348074177, "referral": 0.9945329153511618, "billing": 1.0634939109987174e-05, "insurance": 2.2293989419308482e-05, "records": 0.00010191442677601308, "symptom_question": 0.00039552350849043287, "cancel_visit": 1.6045029757297043e-05, "provider_message": 0.0006220070154002514, "other": 0.00029606622137718735 } }, { "domain": "clinic_request", "row": 50, "task": "urgent", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.17934797704219818, "no": 0.08060139417648315 }, "probabilities": { "yes": 0.4353761539135609, "no": 0.5646238460864392 } }, { "domain": "clinic_request", "row": 60, "task": "request", "multi_label": false, "gold": [ "other" ], "selected": [ "symptom_question" ], "correct": false, "logits": { "appointment": -2.29267954826355, "refill": -5.228410720825195, "lab_results": -2.592191457748413, "referral": -4.654784679412842, "billing": -6.302794456481934, "insurance": -4.460928916931152, "records": -3.5160458087921143, "symptom_question": 1.3843411207199097, "cancel_visit": -4.880802631378174, "provider_message": -1.44903564453125, "other": -1.953242540359497 }, "probabilities": { "appointment": 0.021906845729165597, "refill": 0.0011630757512747524, "lab_results": 0.016236913623599702, "referral": 0.0020640994863317607, "billing": 0.000397199847849335, "insurance": 0.002505654078615571, "records": 0.006445826826844658, "symptom_question": 0.8659436588077801, "cancel_visit": 0.0016465399649919768, "provider_message": 0.0509295386366371, "other": 0.030760647246909436 } }, { "domain": "clinic_request", "row": 60, "task": "urgent", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -1.1575946807861328, "no": 1.2289191484451294 }, "probabilities": { "yes": 0.08420688233524667, "no": 0.9157931176647534 } }, { "domain": "clinic_request", "row": 70, "task": "request", "multi_label": false, "gold": [ "billing" ], "selected": [ "billing" ], "correct": true, "logits": { "appointment": -4.455710411071777, "refill": -6.868372917175293, "lab_results": -3.456378936767578, "referral": -4.9876179695129395, "billing": 4.650760650634766, "insurance": -2.097292184829712, "records": -2.852611541748047, "symptom_question": -2.69492506980896, "cancel_visit": -8.500468254089355, "provider_message": -3.3274502754211426, "other": -3.6134119033813477 }, "probabilities": { "appointment": 0.00011056304405064028, "refill": 9.903848178925877e-06, "lab_results": 0.0003003406609625505, "referral": 6.49539355113088e-05, "billing": 0.9965523532315648, "insurance": 0.0011691170898991314, "records": 0.0005493219843240425, "symptom_question": 0.0006431456622238298, "cancel_visit": 1.9363951019964336e-06, "provider_message": 0.00034167022869898816, "other": 0.0002566939194838889 } }, { "domain": "clinic_request", "row": 70, "task": "urgent", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -2.4503173828125, "no": 2.3287389278411865 }, "probabilities": { "yes": 0.008333888540039263, "no": 0.9916661114599608 } }, { "domain": "clinic_request", "row": 80, "task": "request", "multi_label": false, "gold": [ "records" ], "selected": [ "records" ], "correct": true, "logits": { "appointment": -1.4301544427871704, "refill": -5.622780799865723, "lab_results": -8.02865219116211, "referral": 0.531210720539093, "billing": -7.905220031738281, "insurance": -8.969460487365723, "records": 0.9037842750549316, "symptom_question": -4.9548845291137695, "cancel_visit": -5.276875019073486, "provider_message": -2.248868942260742, "other": -1.9233102798461914 }, "probabilities": { "appointment": 0.05115462507710464, "refill": 0.0007727702857581651, "lab_results": 6.969373585393124e-05, "referral": 0.3636595302550883, "billing": 7.884962836281521e-05, "insurance": 2.7202316756692817e-05, "records": 0.5278391662834148, "symptom_question": 0.001507002869599954, "cancel_visit": 0.0010921326385970585, "provider_message": 0.02255909725814387, "other": 0.03123992965131986 } }, { "domain": "clinic_request", "row": 80, "task": "urgent", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.3513615131378174, "no": 1.3655471801757812 }, "probabilities": { "yes": 0.061982954957924205, "no": 0.9380170450420757 } }, { "domain": "clinic_request", "row": 90, "task": "request", "multi_label": false, "gold": [ "symptom_question" ], "selected": [ "lab_results" ], "correct": false, "logits": { "appointment": -4.481597423553467, "refill": -0.7187599539756775, "lab_results": 0.22515612840652466, "referral": -5.424294471740723, "billing": -7.045811653137207, "insurance": -5.9566731452941895, "records": -4.183034420013428, "symptom_question": -0.47066277265548706, "cancel_visit": -5.503704071044922, "provider_message": -3.6692333221435547, "other": -1.4310364723205566 }, "probabilities": { "appointment": 0.004241878435236069, "refill": 0.18269967196398784, "lab_results": 0.4695429385019126, "referral": 0.0016525328146523022, "billing": 0.0003265383003686206, "insurance": 0.0009703780671954423, "records": 0.005717714724400601, "symptom_question": 0.23414506266752627, "cancel_visit": 0.0015263809645541513, "provider_message": 0.009557921620597716, "other": 0.0896189819395684 } }, { "domain": "clinic_request", "row": 90, "task": "urgent", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -2.8203988075256348, "no": 2.8934953212738037 }, "probabilities": { "yes": 0.003288944803494849, "no": 0.9967110551965053 } }, { "domain": "document_type", "row": 0, "task": "doc_type", "multi_label": false, "gold": [ "resume" ], "selected": [ "resume" ], "correct": true, "logits": { "invoice": -3.717766761779785, "receipt": -3.065006732940674, "contract": -3.672956943511963, "resume": -0.7693808078765869, "bank_statement": -2.4317662715911865, "clinical_note": -1.9325182437896729, "syllabus": -3.2396013736724854, "news_article": -2.551645517349243, "lease": -3.220419406890869, "research_abstract": -2.627199172973633, "recipe": -3.8168201446533203, "sports_recap": -2.7591872215270996 }, "probabilities": { "invoice": 0.021940513885949076, "receipt": 0.04214410873934493, "contract": 0.02294602445574448, "resume": 0.4185183896613727, "bank_statement": 0.07938705876261207, "clinical_note": 0.13078874591212297, "syllabus": 0.03539251174057499, "news_article": 0.07041850764755019, "lease": 0.03607796285798983, "research_abstract": 0.06529415130357928, "recipe": 0.019871399641759165, "sports_recap": 0.057220625391400194 } }, { "domain": "document_type", "row": 10, "task": "doc_type", "multi_label": false, "gold": [ "lease" ], "selected": [ "lease" ], "correct": true, "logits": { "invoice": -6.495243549346924, "receipt": -5.801093101501465, "contract": -5.529821395874023, "resume": -4.5417704582214355, "bank_statement": -6.54439115524292, "clinical_note": -4.90483283996582, "syllabus": -5.628314971923828, "news_article": -5.016096115112305, "lease": 2.6503190994262695, "research_abstract": -5.101913928985596, "recipe": -5.429075717926025, "sports_recap": -4.654085636138916 }, "probabilities": { "invoice": 0.00010625522724197656, "receipt": 0.00021272376625723605, "contract": 0.0002790151712655382, "resume": 0.0007494331309058345, "bank_statement": 0.00010115928950201305, "clinical_note": 0.0005212630003633401, "syllabus": 0.00025284397062672113, "news_article": 0.000466375656124424, "lease": 0.9959045044851793, "research_abstract": 0.00042802158413287656, "recipe": 0.00030858947557944606, "sports_recap": 0.0006698152428212919 } }, { "domain": "document_type", "row": 20, "task": "doc_type", "multi_label": false, "gold": [ "recipe" ], "selected": [ "research_abstract" ], "correct": false, "logits": { "invoice": -3.1631689071655273, "receipt": -3.1692326068878174, "contract": -3.6854875087738037, "resume": -3.0288262367248535, "bank_statement": -2.9296674728393555, "clinical_note": -2.483961582183838, "syllabus": -3.319735050201416, "news_article": -2.3201382160186768, "lease": -4.209490776062012, "research_abstract": -1.168447494506836, "recipe": -1.185955286026001, "sports_recap": -3.035522937774658 }, "probabilities": { "invoice": 0.03816093709602511, "receipt": 0.037930240775821814, "contract": 0.022634918985090773, "resume": 0.04364789574783513, "bank_statement": 0.04819782220796615, "clinical_note": 0.07526533936850564, "syllabus": 0.0326304617067104, "news_article": 0.08866303799867183, "lease": 0.013403160460462755, "research_abstract": 0.28048880574837964, "recipe": 0.27562080453452464, "sports_recap": 0.04335657537000611 } }, { "domain": "document_type", "row": 30, "task": "doc_type", "multi_label": false, "gold": [ "syllabus" ], "selected": [ "syllabus" ], "correct": true, "logits": { "invoice": -4.9594573974609375, "receipt": -4.926259994506836, "contract": -4.909808158874512, "resume": -4.481443405151367, "bank_statement": -4.677807807922363, "clinical_note": -4.706547260284424, "syllabus": 2.636038064956665, "news_article": -4.485772609710693, "lease": -5.0511064529418945, "research_abstract": -4.865212440490723, "recipe": -4.861274719238281, "sports_recap": -4.641040325164795 }, "probabilities": { "invoice": 0.0004993455915333799, "receipt": 0.0005162007949019411, "contract": 0.0005247634884133053, "resume": 0.0008053785509793018, "bank_statement": 0.0006617898204633142, "clinical_note": 0.0006430410486162287, "syllabus": 0.9933058351124509, "news_article": 0.0008018994388123416, "lease": 0.0004556155531674178, "research_abstract": 0.0005486953564512666, "recipe": 0.0005508602253452997, "sports_recap": 0.0006865750188653827 } }, { "domain": "document_type", "row": 40, "task": "doc_type", "multi_label": false, "gold": [ "research_abstract" ], "selected": [ "news_article" ], "correct": false, "logits": { "invoice": -5.500361442565918, "receipt": -5.288315773010254, "contract": -6.301980495452881, "resume": -3.663029193878174, "bank_statement": -5.313586235046387, "clinical_note": -3.569230556488037, "syllabus": -5.198582649230957, "news_article": 2.1963210105895996, "lease": -6.157444000244141, "research_abstract": -3.850562334060669, "recipe": -5.716029167175293, "sports_recap": -4.19045877456665 }, "probabilities": { "invoice": 0.000448492952791125, "receipt": 0.0005544289350102918, "contract": 0.0002011948649229485, "resume": 0.002816429472591316, "bank_statement": 0.0005405938059554468, "clinical_note": 0.0030933930893797842, "syllabus": 0.0006064800061387993, "news_article": 0.9871481630520272, "lease": 0.00023248144394587923, "research_abstract": 0.00233482466428205, "recipe": 0.00036148669788592196, "sports_recap": 0.001662031015069125 } }, { "domain": "document_type", "row": 50, "task": "doc_type", "multi_label": false, "gold": [ "clinical_note" ], "selected": [ "clinical_note" ], "correct": true, "logits": { "invoice": -3.83662748336792, "receipt": -3.6385903358459473, "contract": -4.884789943695068, "resume": -4.106767177581787, "bank_statement": -3.3216631412506104, "clinical_note": 1.1112792491912842, "syllabus": -4.603428363800049, "news_article": -2.9201622009277344, "lease": -5.052422523498535, "research_abstract": -3.9236631393432617, "recipe": -5.162744045257568, "sports_recap": -2.9223568439483643 }, "probabilities": { "invoice": 0.006543444460048777, "receipt": 0.007976509024900814, "contract": 0.002294009689046425, "resume": 0.0049944335809263786, "bank_statement": 0.01095097008734258, "clinical_note": 0.9218388592476211, "syllabus": 0.0030394081903521255, "news_article": 0.01636146680417843, "lease": 0.0019399625379270006, "research_abstract": 0.005998011794300888, "recipe": 0.00173732598475612, "sports_recap": 0.016325598598599368 } }, { "domain": "document_type", "row": 60, "task": "doc_type", "multi_label": false, "gold": [ "bank_statement" ], "selected": [ "bank_statement" ], "correct": true, "logits": { "invoice": -2.0683376789093018, "receipt": -1.2749611139297485, "contract": -3.4047317504882812, "resume": -3.211536169052124, "bank_statement": -1.0151761770248413, "clinical_note": -2.29103684425354, "syllabus": -3.439517021179199, "news_article": -3.2403934001922607, "lease": -3.560345411300659, "research_abstract": -2.573681592941284, "recipe": -3.7476160526275635, "sports_recap": -3.0538272857666016 }, "probabilities": { "invoice": 0.10625847175897486, "receipt": 0.23492142670949953, "contract": 0.02792383058954054, "resume": 0.03387495831634197, "bank_statement": 0.3046111486406086, "clinical_note": 0.08504456162117761, "syllabus": 0.026969192514281773, "news_article": 0.032911390627931386, "lease": 0.023899721506853877, "research_abstract": 0.06410552956021959, "recipe": 0.01981810997976248, "sports_recap": 0.0396616581748077 } }, { "domain": "document_type", "row": 70, "task": "doc_type", "multi_label": false, "gold": [ "receipt" ], "selected": [ "receipt" ], "correct": true, "logits": { "invoice": -7.049471378326416, "receipt": 4.923910140991211, "contract": -8.00204849243164, "resume": -7.241337776184082, "bank_statement": -6.929511070251465, "clinical_note": -6.935129642486572, "syllabus": -6.165859222412109, "news_article": -5.277427673339844, "lease": -8.388875007629395, "research_abstract": -7.049596309661865, "recipe": -6.826353073120117, "sports_recap": -6.937930107116699 }, "probabilities": { "invoice": 6.309305522287433e-06, "receipt": 0.9998965773385753, "contract": 2.433787039102624e-06, "resume": 5.207808910907276e-06, "bank_statement": 7.11343976031014e-06, "clinical_note": 7.073584454979871e-06, "syllabus": 1.5266147060754188e-05, "news_article": 3.7116785833183886e-05, "lease": 1.6530498593559487e-06, "research_abstract": 6.308517341558037e-06, "recipe": 7.886432798612531e-06, "sports_recap": 7.0538028436851365e-06 } }, { "domain": "document_type", "row": 80, "task": "doc_type", "multi_label": false, "gold": [ "receipt" ], "selected": [ "resume" ], "correct": false, "logits": { "invoice": -2.7465105056762695, "receipt": -2.549401044845581, "contract": -2.2703211307525635, "resume": -1.960444688796997, "bank_statement": -2.006202220916748, "clinical_note": -2.038778066635132, "syllabus": -2.688950777053833, "news_article": -2.2101147174835205, "lease": -2.878430128097534, "research_abstract": -2.5261082649230957, "recipe": -2.7428998947143555, "sports_recap": -2.7437353134155273 }, "probabilities": { "invoice": 0.05866140482522379, "receipt": 0.07144239586136635, "contract": 0.09444063034895386, "resume": 0.12874681852431039, "bank_statement": 0.12298843131851206, "clinical_note": 0.1190465330736127, "syllabus": 0.062137007193798555, "news_article": 0.10030121414820967, "lease": 0.051411526045204124, "research_abstract": 0.07312601988657846, "recipe": 0.05887359116717906, "sports_recap": 0.05882442760705099 } }, { "domain": "document_type", "row": 90, "task": "doc_type", "multi_label": false, "gold": [ "bank_statement" ], "selected": [ "bank_statement" ], "correct": true, "logits": { "invoice": -2.7346019744873047, "receipt": -3.089470386505127, "contract": -3.3269190788269043, "resume": -2.6517891883850098, "bank_statement": -0.5536752939224243, "clinical_note": -2.888329029083252, "syllabus": -2.8255655765533447, "news_article": -2.812869071960449, "lease": -3.3362436294555664, "research_abstract": -2.7412543296813965, "recipe": -3.4293816089630127, "sports_recap": -1.342448353767395 }, "probabilities": { "invoice": 0.04772266938786286, "receipt": 0.03346627188325208, "contract": 0.02639275207595236, "resume": 0.05184296887258697, "bank_statement": 0.4225607463416195, "clinical_note": 0.040922477312092496, "syllabus": 0.04357322830913267, "news_article": 0.04412998294225766, "lease": 0.02614779535352722, "research_abstract": 0.04740625485808362, "recipe": 0.02382241394148571, "sports_recap": 0.19201243872214688 } }, { "domain": "email_triage", "row": 0, "task": "category", "multi_label": false, "gold": [ "calendar" ], "selected": [ "support" ], "correct": false, "logits": { "billing": -2.9045255184173584, "support": 0.909726083278656, "sales": -5.464657783508301, "legal": -6.590215682983398, "recruiting": -5.902942657470703, "calendar": -0.862002432346344, "newsletter": -5.721218585968018, "security": -6.498226165771484 }, "probabilities": { "billing": 0.018418788606369065, "support": 0.8351596174741212, "sales": 0.0014236713580202853, "legal": 0.00046194063074997345, "recruiting": 0.0009184701481435751, "calendar": 0.1420095557866109, "newsletter": 0.0011015058475783962, "security": 0.0005064501484066864 } }, { "domain": "email_triage", "row": 0, "task": "action", "multi_label": false, "gold": [ "escalate" ], "selected": [ "reply" ], "correct": false, "logits": { "reply": 1.736121654510498, "approve": -2.969151020050049, "fyi": -1.3299572467803955, "escalate": -3.470463752746582 }, "probabilities": { "reply": 0.9423904149514091, "approve": 0.00852622721282834, "fyi": 0.043918723855367645, "escalate": 0.0051646339803950265 } }, { "domain": "email_triage", "row": 0, "task": "needs_reply", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 2.314236640930176, "no": -2.187143325805664 }, "probabilities": { "yes": 0.9890280422904387, "no": 0.01097195770956118 } }, { "domain": "email_triage", "row": 0, "task": "is_phishing", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.8854280710220337, "no": 0.9658798575401306 }, "probabilities": { "yes": 0.13571940433947255, "no": 0.8642805956605275 } }, { "domain": "email_triage", "row": 10, "task": "category", "multi_label": false, "gold": [ "calendar" ], "selected": [ "calendar" ], "correct": true, "logits": { "billing": -2.1153244972229004, "support": -2.538224458694458, "sales": -3.092930793762207, "legal": -3.023857355117798, "recruiting": -2.6072471141815186, "calendar": 0.4011651873588562, "newsletter": -3.0931832790374756, "security": -2.834486484527588 }, "probabilities": { "billing": 0.06137137883268413, "support": 0.04020710101593639, "sales": 0.023088560701515493, "legal": 0.024739736738740407, "recruiting": 0.03752550993843751, "calendar": 0.760087279600633, "newsletter": 0.023082731915783465, "security": 0.029897701256269615 } }, { "domain": "email_triage", "row": 10, "task": "action", "multi_label": false, "gold": [ "approve" ], "selected": [ "approve" ], "correct": true, "logits": { "reply": -2.9590327739715576, "approve": 2.8209052085876465, "fyi": -1.8972015380859375, "escalate": -4.2261433601379395 }, "probabilities": { "reply": 0.0030495947852549454, "approve": 0.9872731087245706, "fyi": 0.008818395702955213, "escalate": 0.0008589007872192498 } }, { "domain": "email_triage", "row": 10, "task": "needs_reply", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 0.2053561508655548, "no": -0.05516905337572098 }, "probabilities": { "yes": 0.5647653942530528, "no": 0.43523460574694717 } }, { "domain": "email_triage", "row": 10, "task": "is_phishing", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.9681211709976196, "no": 1.030969500541687 }, "probabilities": { "yes": 0.11929842874200985, "no": 0.8807015712579902 } }, { "domain": "email_triage", "row": 20, "task": "category", "multi_label": false, "gold": [ "newsletter" ], "selected": [ "newsletter" ], "correct": true, "logits": { "billing": -7.282729625701904, "support": -3.0082106590270996, "sales": -6.0646562576293945, "legal": -5.409533500671387, "recruiting": -6.14713191986084, "calendar": -5.42255973815918, "newsletter": 3.057353973388672, "security": -4.891972064971924 }, "probabilities": { "billing": 3.220423965090087e-05, "support": 0.0023137319825455357, "sales": 0.00010887184734791821, "legal": 0.00020961955295410367, "recruiting": 0.00010025288220031604, "calendar": 0.00020690670632764072, "newsletter": 0.9966766856608456, "security": 0.00035172712812800547 } }, { "domain": "email_triage", "row": 20, "task": "action", "multi_label": false, "gold": [ "reply" ], "selected": [ "reply" ], "correct": true, "logits": { "reply": 1.0721333026885986, "approve": -2.8331124782562256, "fyi": -1.21412992477417, "escalate": -2.041861057281494 }, "probabilities": { "reply": 0.8574823642612124, "approve": 0.01726626903419531, "fyi": 0.08715929181307619, "escalate": 0.03809207489151616 } }, { "domain": "email_triage", "row": 20, "task": "needs_reply", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 1.709019422531128, "no": -1.5769050121307373 }, "probabilities": { "yes": 0.9639427667829202, "no": 0.03605723321707978 } }, { "domain": "email_triage", "row": 20, "task": "is_phishing", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": 0.0177142471075058, "no": 0.021049663424491882 }, "probabilities": { "yes": 0.499166146693805, "no": 0.500833853306195 } }, { "domain": "email_triage", "row": 30, "task": "category", "multi_label": false, "gold": [ "sales" ], "selected": [ "billing" ], "correct": false, "logits": { "billing": -1.0092356204986572, "support": -1.189382553100586, "sales": -1.2097177505493164, "legal": -4.521008491516113, "recruiting": -3.1827971935272217, "calendar": -3.2149901390075684, "newsletter": -3.043745279312134, "security": -2.079066038131714 }, "probabilities": { "billing": 0.29576366849849717, "support": 0.24700628597175917, "sales": 0.2420340909994671, "legal": 0.008826758506548344, "recruiting": 0.033649531900298305, "calendar": 0.03258350568261982, "newsletter": 0.03866949550487197, "security": 0.10146666293593812 } }, { "domain": "email_triage", "row": 30, "task": "action", "multi_label": false, "gold": [ "fyi" ], "selected": [ "fyi" ], "correct": true, "logits": { "reply": -1.56499445438385, "approve": -2.239696979522705, "fyi": 2.0619428157806396, "escalate": -3.79347562789917 }, "probabilities": { "reply": 0.02550077860174533, "approve": 0.01298774810954137, "fyi": 0.9587652467227271, "escalate": 0.002746226565986284 } }, { "domain": "email_triage", "row": 30, "task": "needs_reply", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.5737261772155762, "no": 1.779862642288208 }, "probabilities": { "yes": 0.03377784004536682, "no": 0.9662221599546332 } }, { "domain": "email_triage", "row": 30, "task": "is_phishing", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.8650343418121338, "no": 1.8937525749206543 }, "probabilities": { "yes": 0.02278093344904527, "no": 0.9772190665509547 } }, { "domain": "email_triage", "row": 40, "task": "category", "multi_label": false, "gold": [ "calendar" ], "selected": [ "security" ], "correct": false, "logits": { "billing": -3.9978418350219727, "support": -1.869075894355774, "sales": -2.7192046642303467, "legal": -4.166479587554932, "recruiting": -2.4074440002441406, "calendar": -1.5354905128479004, "newsletter": -3.618589401245117, "security": 0.09942536801099777 }, "probabilities": { "billing": 0.01085588705818555, "support": 0.09123818088393808, "sales": 0.03899153961468192, "legal": 0.009171214922500727, "recruiting": 0.05325572729414859, "calendar": 0.1273652370140579, "newsletter": 0.015862533621939033, "security": 0.6532596795905482 } }, { "domain": "email_triage", "row": 40, "task": "action", "multi_label": false, "gold": [ "fyi" ], "selected": [ "fyi" ], "correct": true, "logits": { "reply": -0.8044319152832031, "approve": -1.4401228427886963, "fyi": 0.9063093066215515, "escalate": -2.499161958694458 }, "probabilities": { "reply": 0.13800185415046798, "approve": 0.07308156845428083, "fyi": 0.763572702546234, "escalate": 0.0253438748490172 } }, { "domain": "email_triage", "row": 40, "task": "needs_reply", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 0.5056554675102234, "no": -0.24813403189182281 }, "probabilities": { "yes": 0.6800038508605181, "no": 0.31999614913948193 } }, { "domain": "email_triage", "row": 40, "task": "is_phishing", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -1.0756397247314453, "no": 1.164656639099121 }, "probabilities": { "yes": 0.096189773555535, "no": 0.9038102264444651 } }, { "domain": "email_triage", "row": 50, "task": "category", "multi_label": false, "gold": [ "sales" ], "selected": [ "legal" ], "correct": false, "logits": { "billing": -5.03212308883667, "support": -3.161911964416504, "sales": -4.020991325378418, "legal": 1.2868590354919434, "recruiting": -4.614987373352051, "calendar": -2.70119047164917, "newsletter": -2.4758358001708984, "security": -3.697389602661133 }, "probabilities": { "billing": 0.001684244125985961, "support": 0.010930182483012323, "sales": 0.004629498915728371, "legal": 0.9347685864825619, "recruiting": 0.002556023137547825, "calendar": 0.017326714312998804, "newsletter": 0.021706335184043978, "security": 0.006398415358120832 } }, { "domain": "email_triage", "row": 50, "task": "action", "multi_label": false, "gold": [ "fyi" ], "selected": [ "fyi" ], "correct": true, "logits": { "reply": -2.0001986026763916, "approve": -2.446605682373047, "fyi": 1.426987648010254, "escalate": -2.565155506134033 }, "probabilities": { "reply": 0.030304689170919056, "approve": 0.01939267425360652, "fyi": 0.9330779166809744, "escalate": 0.017224719894500137 } }, { "domain": "email_triage", "row": 50, "task": "needs_reply", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.5471482276916504, "no": 1.7205866575241089 }, "probabilities": { "yes": 0.03669481226570259, "no": 0.9633051877342975 } }, { "domain": "email_triage", "row": 50, "task": "is_phishing", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.7069623470306396, "no": 1.7470277547836304 }, "probabilities": { "yes": 0.030650088525425567, "no": 0.9693499114745745 } }, { "domain": "email_triage", "row": 60, "task": "category", "multi_label": false, "gold": [ "billing" ], "selected": [ "billing" ], "correct": true, "logits": { "billing": 1.0598257780075073, "support": -2.7535247802734375, "sales": -2.153749942779541, "legal": -4.561210632324219, "recruiting": -4.563933372497559, "calendar": -4.1935882568359375, "newsletter": -3.972532272338867, "security": -3.43172550201416 }, "probabilities": { "billing": 0.9153528516978889, "support": 0.020205585234153466, "sales": 0.036808687858503, "legal": 0.003314388728067638, "recruiting": 0.0033053767828876185, "calendar": 0.00478696015296903, "newsletter": 0.005971221846469262, "security": 0.01025492769906108 } }, { "domain": "email_triage", "row": 60, "task": "action", "multi_label": false, "gold": [ "fyi" ], "selected": [ "fyi" ], "correct": true, "logits": { "reply": -0.4583055377006531, "approve": -2.30488920211792, "fyi": 0.34892305731773376, "escalate": -2.04378604888916 }, "probabilities": { "reply": 0.2774455101917192, "approve": 0.043774037470418414, "fyi": 0.621945923503246, "escalate": 0.056834528834616475 } }, { "domain": "email_triage", "row": 60, "task": "needs_reply", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.6437265872955322, "no": 0.8830789923667908 }, "probabilities": { "yes": 0.17846154845930134, "no": 0.8215384515406987 } }, { "domain": "email_triage", "row": 60, "task": "is_phishing", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -1.705591082572937, "no": 1.7788643836975098 }, "probabilities": { "yes": 0.029757770616122607, "no": 0.9702422293838774 } }, { "domain": "email_triage", "row": 70, "task": "category", "multi_label": false, "gold": [ "recruiting" ], "selected": [ "recruiting" ], "correct": true, "logits": { "billing": -3.6199278831481934, "support": -3.493718147277832, "sales": -3.898144245147705, "legal": -3.309880018234253, "recruiting": 1.7355766296386719, "calendar": -3.4369239807128906, "newsletter": -4.565179824829102, "security": -3.3334338665008545 }, "probabilities": { "billing": 0.004567313606901031, "support": 0.005181709057561929, "sales": 0.0034580638028203392, "legal": 0.0062274881468170696, "recruiting": 0.9672236128932749, "calendar": 0.0054845173883228985, "newsletter": 0.0017747742969531392, "security": 0.006082520807348872 } }, { "domain": "email_triage", "row": 70, "task": "action", "multi_label": false, "gold": [ "reply" ], "selected": [ "reply" ], "correct": true, "logits": { "reply": 0.5396403074264526, "approve": -1.7975094318389893, "fyi": -0.6892622709274292, "escalate": -2.326049327850342 }, "probabilities": { "reply": 0.6914864393715993, "approve": 0.06679938014238043, "fyi": 0.20233828240933294, "escalate": 0.03937589807668731 } }, { "domain": "email_triage", "row": 70, "task": "needs_reply", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 1.4190001487731934, "no": -1.2734938859939575 }, "probabilities": { "yes": 0.9365822785360409, "no": 0.06341772146395913 } }, { "domain": "email_triage", "row": 70, "task": "is_phishing", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.4190840721130371, "no": 0.4642097055912018 }, "probabilities": { "yes": 0.2924956925136776, "no": 0.7075043074863224 } }, { "domain": "email_triage", "row": 80, "task": "category", "multi_label": false, "gold": [ "billing" ], "selected": [ "billing" ], "correct": true, "logits": { "billing": -0.16560553014278412, "support": -2.917086124420166, "sales": -3.111635446548462, "legal": -2.746657371520996, "recruiting": -3.4830009937286377, "calendar": -0.7154465913772583, "newsletter": -3.6066505908966064, "security": -3.07230544090271 }, "probabilities": { "billing": 0.5285269336996236, "support": 0.033737607755217, "sales": 0.027772986791400507, "legal": 0.04000649937014001, "recruiting": 0.019157576996029182, "calendar": 0.3049819837870367, "newsletter": 0.016929348370875442, "security": 0.028887063229677542 } }, { "domain": "email_triage", "row": 80, "task": "action", "multi_label": false, "gold": [ "escalate" ], "selected": [ "escalate" ], "correct": true, "logits": { "reply": -5.24130392074585, "approve": -5.314145565032959, "fyi": -3.746983051300049, "escalate": 5.352441310882568 }, "probabilities": { "reply": 2.506832822702581e-05, "approve": 2.330722926789161e-05, "fyi": 0.0001117122194323758, "escalate": 0.9998399122230728 } }, { "domain": "email_triage", "row": 80, "task": "needs_reply", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.730243444442749, "no": 0.9943099021911621 }, "probabilities": { "yes": 0.15128559309317688, "no": 0.8487144069068231 } }, { "domain": "email_triage", "row": 80, "task": "is_phishing", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.897105872631073, "no": 1.0429555177688599 }, "probabilities": { "yes": 0.12564111229198707, "no": 0.874358887708013 } }, { "domain": "email_triage", "row": 90, "task": "category", "multi_label": false, "gold": [ "newsletter" ], "selected": [ "support" ], "correct": false, "logits": { "billing": -3.21710467338562, "support": -0.6221742033958435, "sales": -3.1950721740722656, "legal": -2.2697176933288574, "recruiting": -2.9424262046813965, "calendar": -1.4578075408935547, "newsletter": -2.7740440368652344, "security": -1.6621196269989014 }, "probabilities": { "billing": 0.03183310459678516, "support": 0.42642531632827824, "sales": 0.03254225089808469, "legal": 0.08209636608400621, "recruiting": 0.04189578570579657, "calendar": 0.18489792311325107, "newsletter": 0.04957900262230065, "security": 0.15073025065149734 } }, { "domain": "email_triage", "row": 90, "task": "action", "multi_label": false, "gold": [ "approve" ], "selected": [ "reply" ], "correct": false, "logits": { "reply": -0.10729897022247314, "approve": -1.698201298713684, "fyi": -0.3807440996170044, "escalate": -1.730592966079712 }, "probabilities": { "reply": 0.46258953774350736, "approve": 0.09424877273322979, "fyi": 0.35191687739301686, "escalate": 0.09124481213024609 } }, { "domain": "email_triage", "row": 90, "task": "needs_reply", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.21381953358650208, "no": 0.36092373728752136 }, "probabilities": { "yes": 0.3601430618070805, "no": 0.6398569381929196 } }, { "domain": "email_triage", "row": 90, "task": "is_phishing", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -1.3645211458206177, "no": 1.4000318050384521 }, "probabilities": { "yes": 0.059269996819364366, "no": 0.9407300031806357 } }, { "domain": "news_topic", "row": 0, "task": "topic", "multi_label": false, "gold": [ "world" ], "selected": [ "culture" ], "correct": false, "logits": { "politics": -4.489658832550049, "business": -4.202559471130371, "technology": -6.463160037994385, "science": -5.889232158660889, "health": -6.8161468505859375, "sports": -5.081418037414551, "entertainment": -1.3096625804901123, "world": -2.5794711112976074, "culture": 0.623578667640686, "weather": -1.5752997398376465 }, "probabilities": { "politics": 0.004570074505380987, "business": 0.006089882997717053, "technology": 0.0006351007079139204, "science": 0.0011274474477896105, "health": 0.0004462131572344206, "sports": 0.002528864285786505, "entertainment": 0.10989504346466163, "world": 0.030867912970177463, "culture": 0.759581032396362, "weather": 0.08425842806697646 } }, { "domain": "news_topic", "row": 10, "task": "topic", "multi_label": false, "gold": [ "sports" ], "selected": [ "sports" ], "correct": true, "logits": { "politics": -1.4312825202941895, "business": 0.2645479142665863, "technology": -6.792612075805664, "science": -6.237414360046387, "health": -6.45211124420166, "sports": 0.6271466016769409, "entertainment": -3.3869354724884033, "world": -4.341022491455078, "culture": -4.445141792297363, "weather": -5.9029011726379395 }, "probabilities": { "politics": 0.06867758721090714, "business": 0.3743732667299481, "technology": 0.0003224179315376574, "science": 0.0005617440638466389, "health": 0.00045320721973712076, "sports": 0.537996484901226, "entertainment": 0.009715960564724102, "world": 0.0037422346513965635, "culture": 0.003372194207116895, "weather": 0.0007849025195597972 } }, { "domain": "news_topic", "row": 20, "task": "topic", "multi_label": false, "gold": [ "entertainment" ], "selected": [ "entertainment" ], "correct": true, "logits": { "politics": -2.60714054107666, "business": -4.265990734100342, "technology": -4.25110387802124, "science": -2.0410659313201904, "health": -2.0449166297912598, "sports": -5.074434757232666, "entertainment": 1.6429940462112427, "world": -3.2678370475769043, "culture": -3.2552361488342285, "weather": -3.746854305267334 }, "probabilities": { "politics": 0.013078975947380056, "business": 0.002489684158500977, "technology": 0.002527024982370896, "science": 0.02303651720066668, "health": 0.022947981091433735, "sports": 0.0011092807533249053, "entertainment": 0.9170304141948105, "world": 0.006755179506830163, "culture": 0.0068408394021790345, "weather": 0.004184102762503205 } }, { "domain": "news_topic", "row": 30, "task": "topic", "multi_label": false, "gold": [ "politics" ], "selected": [ "health" ], "correct": false, "logits": { "politics": -1.4370054006576538, "business": -1.4615705013275146, "technology": -4.921696186065674, "science": -3.836298942565918, "health": 0.8058353662490845, "sports": -6.060693264007568, "entertainment": -5.363260746002197, "world": -1.4443620443344116, "culture": -4.45079231262207, "weather": -5.787358283996582 }, "probabilities": { "politics": 0.07935597967421683, "business": 0.07743034063683167, "technology": 0.0024333113298995145, "science": 0.007204099654517079, "health": 0.7475375732754779, "sports": 0.0007790001340719858, "entertainment": 0.0015646911511394104, "world": 0.0787743281329562, "culture": 0.0038968046250044274, "weather": 0.0010238713858850659 } }, { "domain": "news_topic", "row": 40, "task": "topic", "multi_label": false, "gold": [ "business" ], "selected": [ "business" ], "correct": true, "logits": { "politics": -1.7154409885406494, "business": 2.7486019134521484, "technology": -6.526081085205078, "science": -9.660402297973633, "health": -9.89986515045166, "sports": -9.839301109313965, "entertainment": -7.699788570404053, "world": -3.1381120681762695, "culture": -6.6294684410095215, "weather": -7.765586853027344 }, "probabilities": { "politics": 0.011350707224232471, "business": 0.9856713222767194, "technology": 9.242479019421905e-05, "science": 4.0231855815400275e-06, "health": 3.1664502628706337e-06, "sports": 3.3641495956607913e-06, "entertainment": 2.857944489914517e-05, "world": 0.0027363063755414174, "culture": 8.334660520760907e-05, "weather": 2.6759497765718772e-05 } }, { "domain": "news_topic", "row": 50, "task": "topic", "multi_label": false, "gold": [ "weather" ], "selected": [ "politics" ], "correct": false, "logits": { "politics": 0.47548604011535645, "business": 0.02219511568546295, "technology": -4.680123329162598, "science": -2.0590901374816895, "health": -0.2962869107723236, "sports": -5.971528053283691, "entertainment": -4.86473274230957, "world": -2.120131492614746, "culture": -5.243597507476807, "weather": -0.5970786213874817 }, "probabilities": { "politics": 0.38326203908165446, "business": 0.24357575572820891, "technology": 0.0022102586858701594, "science": 0.03039088575564515, "health": 0.1771409918305983, "sports": 0.000607565577548352, "entertainment": 0.001837673093040663, "world": 0.028591269176098215, "culture": 0.0012581411863645726, "weather": 0.13112541988497112 } }, { "domain": "news_topic", "row": 60, "task": "topic", "multi_label": false, "gold": [ "weather" ], "selected": [ "weather" ], "correct": true, "logits": { "politics": -4.51241397857666, "business": -7.74551248550415, "technology": -5.398658752441406, "science": 0.047842517495155334, "health": -6.300850868225098, "sports": -8.48996353149414, "entertainment": -6.487205505371094, "world": -1.7654693126678467, "culture": -6.845568656921387, "weather": 1.212571144104004 }, "probabilities": { "politics": 0.002383761895563746, "business": 9.400393473227063e-05, "technology": 0.0009825884580316503, "science": 0.2279067067429409, "health": 0.0003986158844261447, "sports": 4.465139698351877e-05, "entertainment": 0.0003308428933928095, "world": 0.037174555454931243, "culture": 0.00023119938316672084, "weather": 0.730453073955831 } }, { "domain": "news_topic", "row": 70, "task": "topic", "multi_label": false, "gold": [ "entertainment" ], "selected": [ "culture" ], "correct": false, "logits": { "politics": -0.6852070689201355, "business": -2.2987289428710938, "technology": -5.927213668823242, "science": -6.318424224853516, "health": -6.8929057121276855, "sports": -5.660313129425049, "entertainment": -0.7494106888771057, "world": -3.774480104446411, "culture": 0.313661128282547, "weather": -6.2109293937683105 }, "probabilities": { "politics": 0.20327218312057121, "business": 0.04048874404081399, "technology": 0.0010752350454540668, "science": 0.0007271145333544149, "health": 0.00040936309305182074, "sports": 0.001404160793052182, "entertainment": 0.19063150370187792, "world": 0.009256007951826909, "culture": 0.5519260565173169, "weather": 0.0008096312026806057 } }, { "domain": "news_topic", "row": 80, "task": "topic", "multi_label": false, "gold": [ "weather" ], "selected": [ "weather" ], "correct": true, "logits": { "politics": -1.4016790390014648, "business": -4.452223300933838, "technology": -1.391316294670105, "science": -1.46697998046875, "health": -0.3161798119544983, "sports": -5.512950897216797, "entertainment": -4.8097734451293945, "world": -1.0876777172088623, "culture": -4.631225109100342, "weather": 0.8386865854263306 }, "probabilities": { "politics": 0.05948777198228626, "business": 0.002815743976132019, "technology": 0.06010743370692929, "science": 0.055727282419530025, "health": 0.17613839223626082, "sports": 0.0009748213264630935, "entertainment": 0.0019692965000532997, "world": 0.08143230815317358, "culture": 0.002354255892745151, "weather": 0.5589926938064265 } }, { "domain": "news_topic", "row": 90, "task": "topic", "multi_label": false, "gold": [ "entertainment" ], "selected": [ "entertainment" ], "correct": true, "logits": { "politics": -3.737349271774292, "business": -5.164508819580078, "technology": -6.045992374420166, "science": -5.597693920135498, "health": -4.4790802001953125, "sports": -4.869915008544922, "entertainment": 2.079840898513794, "world": -3.37351393699646, "culture": -1.4536548852920532, "weather": -4.893987655639648 }, "probabilities": { "politics": 0.002858071382975066, "business": 0.0006859075054436156, "technology": 0.00028408094980758716, "science": 0.0004447701742370742, "health": 0.0013612673316243157, "sports": 0.0009208863203423103, "entertainment": 0.9603878622390354, "world": 0.004112299620976922, "culture": 0.02804587163156013, "weather": 0.0008989828439976228 } }, { "domain": "paper_field", "row": 0, "task": "field", "multi_label": false, "gold": [ "chemistry" ], "selected": [ "chemistry" ], "correct": true, "logits": { "biology": -2.4741079807281494, "medicine": -4.252721309661865, "chemistry": -0.6153420209884644, "physics": -1.7957077026367188, "computer_science": -2.9416005611419678, "mathematics": -2.532691478729248, "earth_science": -4.094973564147949, "engineering": -0.6269823312759399, "economics": -3.319430112838745, "other": -2.386392593383789 }, "probabilities": { "biology": 0.0521227799403504, "medicine": 0.008802086179192325, "chemistry": 0.3344100878861595, "physics": 0.10271954046146535, "computer_science": 0.0326586420314217, "mathematics": 0.04915696718901449, "earth_science": 0.010306105686651822, "engineering": 0.33054001880080336, "economics": 0.02238251236639635, "other": 0.056901259458544684 } }, { "domain": "paper_field", "row": 10, "task": "field", "multi_label": false, "gold": [ "earth_science" ], "selected": [ "economics" ], "correct": false, "logits": { "biology": -5.750082015991211, "medicine": -5.227787494659424, "chemistry": -8.478287696838379, "physics": -7.417190074920654, "computer_science": -6.901255130767822, "mathematics": -5.65446662902832, "earth_science": -4.532878398895264, "engineering": -8.648490905761719, "economics": 4.316717624664307, "other": -6.309660911560059 }, "probabilities": { "biology": 4.245123151680537e-05, "medicine": 7.156817161844002e-05, "chemistry": 2.7736114469406756e-06, "physics": 8.014463640859786e-06, "computer_science": 1.342586148520913e-05, "mathematics": 4.671060890534658e-05, "earth_science": 0.00014338875963915433, "engineering": 2.339522932449358e-06, "economics": 0.999645069027288, "other": 2.4258741526908267e-05 } }, { "domain": "paper_field", "row": 20, "task": "field", "multi_label": false, "gold": [ "physics" ], "selected": [ "biology" ], "correct": false, "logits": { "biology": -0.3540479242801666, "medicine": -2.962285041809082, "chemistry": -6.98345422744751, "physics": -1.1421189308166504, "computer_science": -3.2405495643615723, "mathematics": -3.4146435260772705, "earth_science": -3.1281957626342773, "engineering": -3.8709876537323, "economics": -2.1078052520751953, "other": -2.311972141265869 }, "probabilities": { "biology": 0.49050787698138326, "medicine": 0.03613291491520703, "chemistry": 0.0006479349553557274, "physics": 0.2230442832685493, "computer_science": 0.02735610435361687, "mathematics": 0.022985089800422836, "earth_science": 0.03060898213189349, "engineering": 0.014563255481085157, "economics": 0.08491782381121885, "other": 0.06923573430126748 } }, { "domain": "paper_field", "row": 30, "task": "field", "multi_label": false, "gold": [ "biology" ], "selected": [ "biology" ], "correct": true, "logits": { "biology": 0.713104248046875, "medicine": -1.0974675416946411, "chemistry": -3.6419014930725098, "physics": -4.421535491943359, "computer_science": -4.630498886108398, "mathematics": -3.8554770946502686, "earth_science": -4.497893810272217, "engineering": -3.4265458583831787, "economics": -5.696075439453125, "other": -2.8155200481414795 }, "probabilities": { "biology": 0.8001154120459397, "medicine": 0.13086734709715572, "chemistry": 0.010275374847247066, "physics": 0.004712017886694674, "computer_science": 0.0038234488215054814, "mathematics": 0.00829932876315333, "earth_science": 0.004365609984623952, "engineering": 0.012744577358108255, "economics": 0.0013172896033383844, "other": 0.023479593592233422 } }, { "domain": "paper_field", "row": 40, "task": "field", "multi_label": false, "gold": [ "computer_science" ], "selected": [ "chemistry" ], "correct": false, "logits": { "biology": -4.985596656799316, "medicine": -4.66553258895874, "chemistry": 0.7519052624702454, "physics": -2.7937872409820557, "computer_science": -1.4860426187515259, "mathematics": -2.55881929397583, "earth_science": -4.959569454193115, "engineering": -2.965700149536133, "economics": -5.001509666442871, "other": -3.2474348545074463 }, "probabilities": { "biology": 0.00262277912800145, "medicine": 0.0036121333710634226, "chemistry": 0.8138177208267774, "physics": 0.023477532671059707, "computer_science": 0.08681579103760391, "mathematics": 0.029695985083356465, "earth_science": 0.0026919388446256798, "engineering": 0.019769315209689654, "economics": 0.002581373139139739, "other": 0.014915430688682653 } }, { "domain": "paper_field", "row": 50, "task": "field", "multi_label": false, "gold": [ "economics" ], "selected": [ "computer_science" ], "correct": false, "logits": { "biology": -2.804840564727783, "medicine": -5.030385494232178, "chemistry": -5.3623046875, "physics": -2.207162618637085, "computer_science": 0.1498289704322815, "mathematics": -2.6692841053009033, "earth_science": -3.718735694885254, "engineering": -2.1997084617614746, "economics": -3.3998918533325195, "other": -2.25473952293396 }, "probabilities": { "biology": 0.03589198912088011, "medicine": 0.0038766415419233735, "chemistry": 0.0027816659298332057, "physics": 0.0652477832482868, "computer_science": 0.6889603081806515, "mathematics": 0.04110256736099642, "earth_science": 0.01439123004173129, "engineering": 0.06573596770168211, "economics": 0.019795662295676992, "other": 0.06221618457833817 } }, { "domain": "paper_field", "row": 60, "task": "field", "multi_label": false, "gold": [ "computer_science" ], "selected": [ "medicine" ], "correct": false, "logits": { "biology": -2.0927672386169434, "medicine": -1.0265430212020874, "chemistry": -3.1064069271087646, "physics": -1.8162269592285156, "computer_science": -2.217827558517456, "mathematics": -1.8284733295440674, "earth_science": -2.4868857860565186, "engineering": -2.3808016777038574, "economics": -3.390261650085449, "other": -2.281038284301758 }, "probabilities": { "biology": 0.09712115046784761, "medicine": 0.2820779328815979, "chemistry": 0.03524485230127299, "physics": 0.12806007022523255, "computer_science": 0.08570394465399324, "mathematics": 0.126501362916439, "earth_science": 0.06548627807396483, "engineering": 0.0728152006826542, "economics": 0.026535003856366087, "other": 0.0804542039406316 } }, { "domain": "paper_field", "row": 70, "task": "field", "multi_label": false, "gold": [ "other" ], "selected": [ "other" ], "correct": true, "logits": { "biology": -1.3174813985824585, "medicine": -3.3366734981536865, "chemistry": -3.804945468902588, "physics": -3.0500805377960205, "computer_science": -2.8739778995513916, "mathematics": -2.3579328060150146, "earth_science": -1.5896140336990356, "engineering": -3.065368413925171, "economics": -2.27363920211792, "other": -1.0420821905136108 }, "probabilities": { "biology": 0.21766603310794952, "medicine": 0.028897926072601113, "chemistry": 0.018092506727563074, "physics": 0.03848862683260895, "computer_science": 0.045900014315542984, "mathematics": 0.07690035727643799, "earth_science": 0.16580780142222593, "engineering": 0.03790469240482156, "economics": 0.08366361087458893, "other": 0.28667843096565987 } }, { "domain": "paper_field", "row": 80, "task": "field", "multi_label": false, "gold": [ "medicine" ], "selected": [ "medicine" ], "correct": true, "logits": { "biology": -3.103888511657715, "medicine": -0.8226505517959595, "chemistry": -3.365847587585449, "physics": -2.8128154277801514, "computer_science": -3.0287084579467773, "mathematics": -1.0951499938964844, "earth_science": -3.6634016036987305, "engineering": -3.858569860458374, "economics": -3.391070604324341, "other": -1.6387321949005127 }, "probabilities": { "biology": 0.036299715724307284, "medicine": 0.35533033211465365, "chemistry": 0.02793417456409275, "physics": 0.048564023220866344, "computer_science": 0.03913393383654926, "mathematics": 0.27057475743878295, "earth_science": 0.02074482499857336, "engineering": 0.017066689156722287, "economics": 0.0272384020396726, "other": 0.15711314690577957 } }, { "domain": "paper_field", "row": 90, "task": "field", "multi_label": false, "gold": [ "medicine" ], "selected": [ "medicine" ], "correct": true, "logits": { "biology": -0.8351287245750427, "medicine": -0.22011327743530273, "chemistry": -5.70932149887085, "physics": -4.202416896820068, "computer_science": -3.3654603958129883, "mathematics": -2.4256856441497803, "earth_science": -4.107569217681885, "engineering": -5.477057933807373, "economics": -4.721790313720703, "other": -1.7734700441360474 }, "probabilities": { "biology": 0.27513366560283736, "medicine": 0.5089106614963491, "chemistry": 0.002102367860253241, "physics": 0.009487440431495842, "computer_science": 0.021909610256807675, "mathematics": 0.05607556279035637, "earth_science": 0.01043135884838763, "engineering": 0.002652036477876785, "economics": 0.005644013620007859, "other": 0.10765328261562816 } }, { "domain": "product_feedback", "row": 0, "task": "feedback_type", "multi_label": false, "gold": [ "bug" ], "selected": [ "bug" ], "correct": true, "logits": { "bug": 0.9487800002098083, "feature_request": -1.456834077835083, "how_to": -1.6310778856277466, "praise": -3.2061145305633545 }, "probabilities": { "bug": 0.8462511444333995, "feature_request": 0.07634038561127035, "how_to": 0.06413295117567765, "praise": 0.013275518779652407 } }, { "domain": "product_feedback", "row": 0, "task": "product_area", "multi_label": true, "gold": [ "notifications" ], "selected": [ "notifications" ], "correct": true, "logits": { "billing": -3.938328266143799, "mobile": -2.336786985397339, "search": -3.6480700969696045, "checkout": -3.0517566204071045, "shipping": -3.5468578338623047, "notifications": 1.3766472339630127, "admin": -4.12984561920166, "integrations": -4.176385879516602 }, "probabilities": { "billing": 0.019108506066138266, "mobile": 0.08812175939675483, "search": 0.025380398056942453, "checkout": 0.04514169572450645, "shipping": 0.02800798792524719, "notifications": 0.7984519915101754, "admin": 0.015830719056730398, "integrations": 0.015121720886437987 } }, { "domain": "product_feedback", "row": 10, "task": "feedback_type", "multi_label": false, "gold": [ "feature_request" ], "selected": [ "praise" ], "correct": false, "logits": { "bug": -2.9951541423797607, "feature_request": -2.3752951622009277, "how_to": -2.1190972328186035, "praise": 2.7358672618865967 }, "probabilities": { "bug": 0.0031893443429183055, "feature_request": 0.005927925620198098, "how_to": 0.0076589298681592494, "praise": 0.9832238001687242 } }, { "domain": "product_feedback", "row": 10, "task": "product_area", "multi_label": true, "gold": [ "integrations" ], "selected": [ "integrations" ], "correct": true, "logits": { "billing": -7.592564582824707, "mobile": -7.841796398162842, "search": -6.639666557312012, "checkout": -5.627225875854492, "shipping": -7.7289276123046875, "notifications": -8.614835739135742, "admin": -7.315835952758789, "integrations": 5.6769585609436035 }, "probabilities": { "billing": 0.000503932291019885, "mobile": 0.00039280813026699757, "search": 0.0013057559074696353, "checkout": 0.003585641213232031, "shipping": 0.00043972218368092646, "notifications": 0.0001813617110435852, "admin": 0.0006644834028461781, "integrations": 0.9965877270958797 } }, { "domain": "product_feedback", "row": 20, "task": "feedback_type", "multi_label": false, "gold": [ "praise" ], "selected": [ "praise" ], "correct": true, "logits": { "bug": -1.0203558206558228, "feature_request": -4.260862350463867, "how_to": -3.540372371673584, "praise": 6.347087860107422 }, "probabilities": { "bug": 0.000631034237448254, "feature_request": 2.4701243593752363e-05, "how_to": 5.077192625657017e-05, "praise": 0.9992934925927014 } }, { "domain": "product_feedback", "row": 20, "task": "product_area", "multi_label": true, "gold": [ "billing", "checkout", "mobile", "notifications", "search", "shipping" ], "selected": [ "notifications" ], "correct": false, "logits": { "billing": -4.632621765136719, "mobile": -4.6806640625, "search": -5.235381603240967, "checkout": -5.570741176605225, "shipping": -6.59496545791626, "notifications": 3.3331375122070312, "admin": -6.150056838989258, "integrations": -6.943282127380371 }, "probabilities": { "billing": 0.009635472301446154, "mobile": 0.009187658272809264, "search": 0.005296588903182227, "checkout": 0.0037932139813331846, "shipping": 0.0013653673601486073, "notifications": 0.9655482909791919, "admin": 0.0021288189702156415, "integrations": 0.0009641663087373325 } }, { "domain": "product_feedback", "row": 30, "task": "feedback_type", "multi_label": false, "gold": [ "how_to" ], "selected": [ "how_to" ], "correct": true, "logits": { "bug": -3.177748680114746, "feature_request": -2.3071296215057373, "how_to": 2.580815553665161, "praise": -5.458286285400391 }, "probabilities": { "bug": 0.003121258040517101, "feature_request": 0.007454778210023471, "how_to": 0.9891048799340952, "praise": 0.00031908381536410366 } }, { "domain": "product_feedback", "row": 30, "task": "product_area", "multi_label": true, "gold": [ "checkout" ], "selected": [ "checkout" ], "correct": true, "logits": { "billing": -4.624454021453857, "mobile": -5.659699440002441, "search": -3.9202587604522705, "checkout": 1.8389825820922852, "shipping": -5.767838478088379, "notifications": -6.1522626876831055, "admin": -2.3312084674835205, "integrations": -6.021799564361572 }, "probabilities": { "billing": 0.009713727045791353, "mobile": 0.0034714706713692652, "search": 0.019450148916380236, "checkout": 0.8628283347638543, "shipping": 0.0031167636444836733, "notifications": 0.002124138256681648, "admin": 0.08857105971776429, "integrations": 0.0024194333255125565 } }, { "domain": "product_feedback", "row": 40, "task": "feedback_type", "multi_label": false, "gold": [ "feature_request" ], "selected": [ "feature_request" ], "correct": true, "logits": { "bug": -0.3710581362247467, "feature_request": -0.01749209314584732, "how_to": -1.7095649242401123, "praise": -2.868849277496338 }, "probabilities": { "bug": 0.36118809894572795, "feature_request": 0.5143813494802479, "how_to": 0.09471686606955457, "praise": 0.029713685504469567 } }, { "domain": "product_feedback", "row": 40, "task": "product_area", "multi_label": true, "gold": [ "mobile" ], "selected": [], "correct": false, "logits": { "billing": -1.830418586730957, "mobile": -0.14603063464164734, "search": -4.927522659301758, "checkout": -5.299370288848877, "shipping": -5.974586486816406, "notifications": -5.135750770568848, "admin": -0.7326897382736206, "integrations": -6.189521312713623 }, "probabilities": { "billing": 0.13818841513345162, "mobile": 0.46355708027589065, "search": 0.007192323714494382, "checkout": 0.004969914736355389, "shipping": 0.0025361050665315753, "notifications": 0.0058482303441302445, "admin": 0.32460476061440874, "integrations": 0.0020466110075045884 } }, { "domain": "product_feedback", "row": 50, "task": "feedback_type", "multi_label": false, "gold": [ "bug" ], "selected": [ "bug" ], "correct": true, "logits": { "bug": 1.9767060279846191, "feature_request": -3.1850509643554688, "how_to": -1.8374392986297607, "praise": -4.792257308959961 }, "probabilities": { "bug": 0.9718767414898614, "feature_request": 0.0055704285777091475, "how_to": 0.02143625528296592, "praise": 0.0011165746494636078 } }, { "domain": "product_feedback", "row": 50, "task": "product_area", "multi_label": true, "gold": [ "checkout" ], "selected": [ "checkout" ], "correct": true, "logits": { "billing": -4.466841697692871, "mobile": -4.9172258377075195, "search": -4.6771931648254395, "checkout": 2.4128456115722656, "shipping": -5.4546217918396, "notifications": -5.858590126037598, "admin": -5.180074691772461, "integrations": -3.1403322219848633 }, "probabilities": { "billing": 0.011353152849210985, "mobile": 0.00726622347953042, "search": 0.009219308591858008, "checkout": 0.9178016153498716, "shipping": 0.004258283358053612, "notifications": 0.002847137077145862, "admin": 0.005596093566160492, "integrations": 0.041473910195004966 } }, { "domain": "product_feedback", "row": 60, "task": "feedback_type", "multi_label": false, "gold": [ "feature_request" ], "selected": [ "feature_request" ], "correct": true, "logits": { "bug": -1.519136667251587, "feature_request": 3.0301737785339355, "how_to": -2.0772366523742676, "praise": -5.538629055023193 }, "probabilities": { "bug": 0.010399611940754725, "feature_request": 0.9834619393047771, "how_to": 0.005951650088686579, "praise": 0.00018679866578165242 } }, { "domain": "product_feedback", "row": 60, "task": "product_area", "multi_label": true, "gold": [ "billing", "integrations", "mobile", "notifications", "shipping" ], "selected": [ "mobile" ], "correct": false, "logits": { "billing": -0.39826154708862305, "mobile": 0.029278241097927094, "search": -5.347942352294922, "checkout": -5.472879886627197, "shipping": -3.623492479324341, "notifications": -2.061436653137207, "admin": -4.732295989990234, "integrations": -4.173546314239502 }, "probabilities": { "billing": 0.40173009344666905, "mobile": 0.5073190374493156, "search": 0.004735400371345425, "checkout": 0.004181562842105667, "shipping": 0.025995499957256717, "notifications": 0.11290186199219945, "admin": 0.008729356090468475, "integrations": 0.015164068964929583 } }, { "domain": "product_feedback", "row": 70, "task": "feedback_type", "multi_label": false, "gold": [ "feature_request" ], "selected": [ "bug" ], "correct": false, "logits": { "bug": 0.0006977617740631104, "feature_request": -1.1037142276763916, "how_to": -0.5864273309707642, "praise": -4.275925159454346 }, "probabilities": { "bug": 0.5259784985297821, "feature_request": 0.17431226986306994, "how_to": 0.2924036591053699, "praise": 0.00730557250177814 } }, { "domain": "product_feedback", "row": 70, "task": "product_area", "multi_label": true, "gold": [ "mobile", "shipping" ], "selected": [ "mobile" ], "correct": false, "logits": { "billing": -7.2205939292907715, "mobile": 1.521146297454834, "search": -5.180817604064941, "checkout": -6.014914035797119, "shipping": -1.4053372144699097, "notifications": -5.866874694824219, "admin": -5.789588451385498, "integrations": -6.2499847412109375 }, "probabilities": { "billing": 0.0007308334009393783, "mobile": 0.8207072167013102, "search": 0.005591960942794632, "checkout": 0.002436109164553263, "shipping": 0.19697054186928437, "notifications": 0.0028237135388998664, "admin": 0.003049910532579338, "integrations": 0.001926764006542989 } }, { "domain": "product_feedback", "row": 80, "task": "feedback_type", "multi_label": false, "gold": [ "bug" ], "selected": [ "bug" ], "correct": true, "logits": { "bug": 1.2689235210418701, "feature_request": -1.862947940826416, "how_to": -1.119667887687683, "praise": -3.174497604370117 }, "probabilities": { "bug": 0.8717251603662276, "feature_request": 0.03803864928507757, "how_to": 0.07998849264839263, "praise": 0.010247697700302224 } }, { "domain": "product_feedback", "row": 80, "task": "product_area", "multi_label": true, "gold": [ "checkout", "integrations", "mobile" ], "selected": [ "integrations" ], "correct": false, "logits": { "billing": -6.899574279785156, "mobile": -5.709765911102295, "search": -6.767151355743408, "checkout": -5.7771711349487305, "shipping": -6.63620662689209, "notifications": -6.661238193511963, "admin": -4.5836663246154785, "integrations": 4.268526077270508 }, "probabilities": { "billing": 0.001007199082240687, "mobile": 0.0033025054152560677, "search": 0.0011496454797963551, "checkout": 0.003087900672359363, "shipping": 0.001310275626907988, "notifications": 0.001277925860783142, "admin": 0.01011402861059297, "integrations": 0.9861909534656521 } }, { "domain": "product_feedback", "row": 90, "task": "feedback_type", "multi_label": false, "gold": [ "bug" ], "selected": [ "bug" ], "correct": true, "logits": { "bug": 1.0513887405395508, "feature_request": -2.2786924839019775, "how_to": -0.9083734750747681, "praise": -4.456528663635254 }, "probabilities": { "bug": 0.8469288999446242, "feature_request": 0.030311752945354688, "how_to": 0.11932543783138122, "praise": 0.0034339092786399667 } }, { "domain": "product_feedback", "row": 90, "task": "product_area", "multi_label": true, "gold": [ "admin", "billing", "checkout", "shipping" ], "selected": [ "checkout" ], "correct": false, "logits": { "billing": -2.7726492881774902, "mobile": -4.43357515335083, "search": -3.1206469535827637, "checkout": 0.14160507917404175, "shipping": -1.4004404544830322, "notifications": -4.956902503967285, "admin": -2.5374815464019775, "integrations": -4.974191665649414 }, "probabilities": { "billing": 0.05882017636986045, "mobile": 0.011732679582980222, "search": 0.042263577108086585, "checkout": 0.5353422326547007, "shipping": 0.1977462272695175, "notifications": 0.006985542938865009, "admin": 0.07327200084023455, "integrations": 0.006866628992359319 } }, { "domain": "restaurant_review", "row": 0, "task": "sentiment", "multi_label": false, "gold": [ "negative" ], "selected": [ "negative" ], "correct": true, "logits": { "negative": 3.423316478729248, "neutral": -3.9247703552246094, "positive": -3.9390969276428223 }, "probabilities": { "negative": 0.9987231446156553, "neutral": 0.0006430008542350951, "positive": 0.0006338545301096285 } }, { "domain": "restaurant_review", "row": 0, "task": "aspects", "multi_label": true, "gold": [ "food" ], "selected": [ "food", "service" ], "correct": false, "logits": { "food": 2.094191551208496, "service": 0.6947423219680786, "price": -4.668088912963867, "ambiance": -2.27437162399292, "wait": -2.2577474117279053, "cleanliness": -3.6740212440490723 }, "probabilities": { "food": 0.8903373451810958, "service": 0.6670210482452095, "price": 0.009302842345231252, "ambiance": 0.09326785060626341, "wait": 0.09468328090389837, "cleanliness": 0.024746310023779184 } }, { "domain": "restaurant_review", "row": 10, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "neutral" ], "correct": true, "logits": { "negative": -2.3603765964508057, "neutral": 2.174669027328491, "positive": -3.784548759460449 }, "probabilities": { "negative": 0.010585541755422428, "neutral": 0.9868664373916954, "positive": 0.002548020852882191 } }, { "domain": "restaurant_review", "row": 10, "task": "aspects", "multi_label": true, "gold": [ "cleanliness", "food", "price", "service", "wait" ], "selected": [ "food", "price", "service" ], "correct": false, "logits": { "food": 2.528733968734741, "service": 1.5568277835845947, "price": 1.4166327714920044, "ambiance": -0.43591636419296265, "wait": -0.2075594663619995, "cleanliness": -0.9548231363296509 }, "probabilities": { "food": 0.9261317887610405, "service": 0.8258976892297685, "price": 0.8048099984188691, "ambiance": 0.3927144469335244, "wait": 0.448295622995542, "cleanliness": 0.2779158855184076 } }, { "domain": "restaurant_review", "row": 20, "task": "sentiment", "multi_label": false, "gold": [ "negative" ], "selected": [ "negative" ], "correct": true, "logits": { "negative": 4.293211460113525, "neutral": -4.809658527374268, "positive": -4.801661968231201 }, "probabilities": { "negative": 0.9997764644496957, "neutral": 0.00011132089872123443, "positive": 0.00011221465158294973 } }, { "domain": "restaurant_review", "row": 20, "task": "aspects", "multi_label": true, "gold": [ "ambiance", "food", "service", "wait" ], "selected": [ "ambiance", "wait" ], "correct": false, "logits": { "food": -1.530649185180664, "service": -0.005783353000879288, "price": -8.00161361694336, "ambiance": 4.15154504776001, "wait": 0.5168853402137756, "cleanliness": -5.078949928283691 }, "probabilities": { "food": 0.1778987221019026, "service": 0.4985541657797001, "price": 0.0003348096211931496, "ambiance": 0.9845038323800382, "wait": 0.6264191667312847, "cleanliness": 0.0061879150913360555 } }, { "domain": "restaurant_review", "row": 30, "task": "sentiment", "multi_label": false, "gold": [ "positive" ], "selected": [ "positive" ], "correct": true, "logits": { "negative": -3.5561859607696533, "neutral": -2.911916971206665, "positive": 2.655693769454956 }, "probabilities": { "negative": 0.0019938498924455296, "neutral": 0.003797475024551628, "positive": 0.9942086750830028 } }, { "domain": "restaurant_review", "row": 30, "task": "aspects", "multi_label": true, "gold": [ "cleanliness", "food", "price", "service", "wait" ], "selected": [ "ambiance", "cleanliness", "food", "price", "service" ], "correct": false, "logits": { "food": 3.5561165809631348, "service": 0.966407299041748, "price": 1.2713878154754639, "ambiance": 0.4639521539211273, "wait": -2.5229225158691406, "cleanliness": 1.2452175617218018 }, "probabilities": { "food": 0.9722429694615173, "service": 0.7244028174105751, "price": 0.7809802264714436, "ambiance": 0.6139513171101859, "wait": 0.07426676848418605, "cleanliness": 0.7764709004528161 } }, { "domain": "restaurant_review", "row": 40, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "neutral" ], "correct": true, "logits": { "negative": -3.8937125205993652, "neutral": 0.9480120539665222, "positive": -0.671927809715271 }, "probabilities": { "negative": 0.0065461959850894235, "neutral": 0.8293221578742545, "positive": 0.1641316461406562 } }, { "domain": "restaurant_review", "row": 40, "task": "aspects", "multi_label": true, "gold": [ "ambiance" ], "selected": [ "ambiance", "food" ], "correct": false, "logits": { "food": 0.5076600313186646, "service": -0.27414703369140625, "price": -8.342477798461914, "ambiance": 2.836632490158081, "wait": -3.0455846786499023, "cleanliness": -1.3163286447525024 }, "probabilities": { "food": 0.6242577711907402, "service": 0.43188928895876627, "price": 0.00023812472866541684, "ambiance": 0.9446235717056272, "wait": 0.04540847873576515, "cleanliness": 0.2114297598177913 } }, { "domain": "restaurant_review", "row": 50, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "positive" ], "correct": false, "logits": { "negative": -1.1578432321548462, "neutral": -3.2507057189941406, "positive": 1.1417968273162842 }, "probabilities": { "negative": 0.09013941033446364, "neutral": 0.011117217011371623, "positive": 0.8987433726541646 } }, { "domain": "restaurant_review", "row": 50, "task": "aspects", "multi_label": true, "gold": [ "ambiance", "food", "price", "service", "wait" ], "selected": [ "ambiance", "food", "price", "service", "wait" ], "correct": true, "logits": { "food": 2.9156322479248047, "service": 1.7063283920288086, "price": 1.1759191751480103, "ambiance": 0.006085701286792755, "wait": 1.1059508323669434, "cleanliness": -4.99036169052124 }, "probabilities": { "food": 0.9486138068612203, "service": 0.8463594531378558, "price": 0.7642132674297323, "ambiance": 0.5015214206261203, "wait": 0.7513734509971518, "cleanliness": 0.006757232565481168 } }, { "domain": "restaurant_review", "row": 60, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "negative" ], "correct": false, "logits": { "negative": 1.1243791580200195, "neutral": -2.6600379943847656, "positive": -1.678848147392273 }, "probabilities": { "negative": 0.9230744559701766, "neutral": 0.020974192349011635, "positive": 0.055951351680811695 } }, { "domain": "restaurant_review", "row": 60, "task": "aspects", "multi_label": true, "gold": [ "cleanliness", "food", "service", "wait" ], "selected": [ "cleanliness", "food", "service", "wait" ], "correct": true, "logits": { "food": 0.8706992864608765, "service": 0.27247780561447144, "price": -5.5386223793029785, "ambiance": -0.18236279487609863, "wait": 2.420330762863159, "cleanliness": 0.3251728415489197 }, "probabilities": { "food": 0.704891184128812, "service": 0.5677011012117771, "price": 0.003916540230085642, "ambiance": 0.45453523024673403, "wait": 0.9183645456281333, "cleanliness": 0.5805843933878738 } }, { "domain": "restaurant_review", "row": 70, "task": "sentiment", "multi_label": false, "gold": [ "negative" ], "selected": [ "negative" ], "correct": true, "logits": { "negative": 4.124152183532715, "neutral": -4.54883337020874, "positive": -4.822336196899414 }, "probabilities": { "negative": 0.9996987498595091, "neutral": 0.00017109581285768477, "positive": 0.00013015432763313102 } }, { "domain": "restaurant_review", "row": 70, "task": "aspects", "multi_label": true, "gold": [ "cleanliness", "food", "service", "wait" ], "selected": [ "cleanliness", "food" ], "correct": false, "logits": { "food": 1.6059578657150269, "service": -0.4236510396003723, "price": -7.1243438720703125, "ambiance": -2.3435654640197754, "wait": -1.375187635421753, "cleanliness": 3.1485133171081543 }, "probabilities": { "food": 0.8328494326672969, "service": 0.3956434193592203, "price": 0.0008046132673486125, "ambiance": 0.08757858440822724, "wait": 0.20178299876607259, "cleanliness": 0.9588501023706898 } }, { "domain": "restaurant_review", "row": 80, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "neutral" ], "correct": true, "logits": { "negative": -1.2157318592071533, "neutral": 1.0542280673980713, "positive": -2.488887071609497 }, "probabilities": { "negative": 0.09124953570832756, "neutral": 0.883205436469662, "positive": 0.025545027822010468 } }, { "domain": "restaurant_review", "row": 80, "task": "aspects", "multi_label": true, "gold": [ "ambiance", "food", "price", "service", "wait" ], "selected": [ "ambiance", "food", "price", "service", "wait" ], "correct": true, "logits": { "food": 2.901947021484375, "service": 1.3886240720748901, "price": 1.1969270706176758, "ambiance": 0.2900168299674988, "wait": 0.5667166709899902, "cleanliness": -4.672945499420166 }, "probabilities": { "food": 0.9479426012598778, "service": 0.8003724932438455, "price": 0.7679776754254874, "ambiance": 0.5720002531812208, "wait": 0.6380052194671394, "cleanliness": 0.00925818909157475 } }, { "domain": "restaurant_review", "row": 90, "task": "sentiment", "multi_label": false, "gold": [ "positive" ], "selected": [ "positive" ], "correct": true, "logits": { "negative": -2.999330759048462, "neutral": -1.5007925033569336, "positive": 1.321890115737915 }, "probabilities": { "negative": 0.012383038636317617, "neutral": 0.05541586585514736, "positive": 0.9322010955085351 } }, { "domain": "restaurant_review", "row": 90, "task": "aspects", "multi_label": true, "gold": [ "ambiance", "cleanliness", "food", "service" ], "selected": [ "cleanliness" ], "correct": false, "logits": { "food": -1.206081509590149, "service": -2.1147825717926025, "price": -8.12697982788086, "ambiance": -2.4394311904907227, "wait": -5.96270227432251, "cleanliness": 7.259783744812012 }, "probabilities": { "food": 0.23039511692462927, "service": 0.1076683142127565, "price": 0.00029537192192011783, "ambiance": 0.0802148691713914, "wait": 0.0025663466722626775, "cleanliness": 0.999297234124393 } }, { "domain": "review_sentiment", "row": 0, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "neutral" ], "correct": true, "logits": { "negative": -1.0924460887908936, "neutral": 1.3669936656951904, "positive": -4.202434539794922 }, "probabilities": { "negative": 0.07847533548540574, "neutral": 0.9180245491587119, "positive": 0.003500115355882367 } }, { "domain": "review_sentiment", "row": 10, "task": "sentiment", "multi_label": false, "gold": [ "negative" ], "selected": [ "negative" ], "correct": true, "logits": { "negative": 2.0733914375305176, "neutral": -2.5241527557373047, "positive": -3.550973415374756 }, "probabilities": { "negative": 0.9864993557890699, "neutral": 0.00994051151064431, "positive": 0.003560132700285735 } }, { "domain": "review_sentiment", "row": 20, "task": "sentiment", "multi_label": false, "gold": [ "negative" ], "selected": [ "negative" ], "correct": true, "logits": { "negative": 3.213555335998535, "neutral": -2.827216625213623, "positive": -4.799509048461914 }, "probabilities": { "negative": 0.9972964990596564, "neutral": 0.002373287573570685, "positive": 0.00033021336677276525 } }, { "domain": "review_sentiment", "row": 30, "task": "sentiment", "multi_label": false, "gold": [ "positive" ], "selected": [ "positive" ], "correct": true, "logits": { "negative": -3.8303146362304688, "neutral": -2.7649245262145996, "positive": 2.895199775695801 }, "probabilities": { "negative": 0.0011943114253928908, "neutral": 0.0034658569437019766, "positive": 0.9953398316309052 } }, { "domain": "review_sentiment", "row": 40, "task": "sentiment", "multi_label": false, "gold": [ "negative" ], "selected": [ "negative" ], "correct": true, "logits": { "negative": 2.4277799129486084, "neutral": -2.9149601459503174, "positive": -3.5497279167175293 }, "probabilities": { "negative": 0.9927352782452193, "neutral": 0.004748002397684069, "positive": 0.002516719357096526 } }, { "domain": "review_sentiment", "row": 50, "task": "sentiment", "multi_label": false, "gold": [ "positive" ], "selected": [ "negative" ], "correct": false, "logits": { "negative": 1.2247308492660522, "neutral": -1.9256622791290283, "positive": -2.036797046661377 }, "probabilities": { "negative": 0.924928139898042, "neutral": 0.03961955935802076, "positive": 0.035452300743937334 } }, { "domain": "review_sentiment", "row": 60, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "neutral" ], "correct": true, "logits": { "negative": -2.301821708679199, "neutral": 1.3338183164596558, "positive": -1.5475858449935913 }, "probabilities": { "negative": 0.024359286719028553, "neutral": 0.9238532019981811, "positive": 0.05178751128279036 } }, { "domain": "review_sentiment", "row": 70, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "neutral" ], "correct": true, "logits": { "negative": -0.8353257179260254, "neutral": -0.44879165291786194, "positive": -0.904882550239563 }, "probabilities": { "negative": 0.29371356166070417, "neutral": 0.43230833728345086, "positive": 0.2739781010558449 } }, { "domain": "review_sentiment", "row": 80, "task": "sentiment", "multi_label": false, "gold": [ "negative" ], "selected": [ "negative" ], "correct": true, "logits": { "negative": 2.338960886001587, "neutral": -2.752769947052002, "positive": -3.4504411220550537 }, "probabilities": { "negative": 0.9908768168340613, "neutral": 0.006091287013543232, "positive": 0.003031896152395408 } }, { "domain": "review_sentiment", "row": 90, "task": "sentiment", "multi_label": false, "gold": [ "neutral" ], "selected": [ "neutral" ], "correct": true, "logits": { "negative": -2.5136914253234863, "neutral": 2.1333789825439453, "positive": -3.4700658321380615 }, "probabilities": { "negative": 0.009464021585189036, "neutral": 0.9868991099459119, "positive": 0.003636868468899136 } }, { "domain": "screen_tags", "row": 0, "task": "genres", "multi_label": true, "gold": [ "drama", "kids" ], "selected": [ "kids" ], "correct": false, "logits": { "comedy": -2.98821759223938, "drama": -2.455623149871826, "action": -3.7516510486602783, "documentary": -1.5046939849853516, "horror": -6.145459175109863, "romance": -3.1467390060424805, "sci_fi": -3.863232135772705, "kids": 2.433427095413208 }, "probabilities": { "comedy": 0.047961010385471924, "drama": 0.07902831035513339, "action": 0.022940334018511466, "documentary": 0.18172647594678829, "horror": 0.002138608118836111, "romance": 0.041219962906885135, "sci_fi": 0.020568084731137012, "kids": 0.919341028032549 } }, { "domain": "screen_tags", "row": 0, "task": "format", "multi_label": false, "gold": [ "series" ], "selected": [ "series" ], "correct": true, "logits": { "film": -2.6189968585968018, "series": 2.2031679153442383 }, "probabilities": { "film": 0.007985068697484922, "series": 0.992014931302515 } }, { "domain": "screen_tags", "row": 10, "task": "genres", "multi_label": true, "gold": [ "action", "drama" ], "selected": [], "correct": false, "logits": { "comedy": -3.556943416595459, "drama": -0.44812482595443726, "action": -2.1006920337677, "documentary": -0.44421085715293884, "horror": -3.9275920391082764, "romance": -0.8118070363998413, "sci_fi": -2.345543384552002, "kids": -2.0463998317718506 }, "probabilities": { "comedy": 0.027734725784915898, "drama": 0.38980669786733685, "action": 0.10902957737121122, "documentary": 0.39073806495495417, "horror": 0.019310781550596002, "romance": 0.30750556092943077, "sci_fi": 0.08742066047098285, "kids": 0.11441666401526969 } }, { "domain": "screen_tags", "row": 10, "task": "format", "multi_label": false, "gold": [ "series" ], "selected": [ "series" ], "correct": true, "logits": { "film": -1.8547301292419434, "series": 1.735437273979187 }, "probabilities": { "film": 0.02685274391189149, "series": 0.9731472560881085 } }, { "domain": "screen_tags", "row": 20, "task": "genres", "multi_label": true, "gold": [ "drama", "romance" ], "selected": [ "romance" ], "correct": false, "logits": { "comedy": -5.572000026702881, "drama": -0.8210218548774719, "action": -4.958508014678955, "documentary": -2.2191216945648193, "horror": -5.622750282287598, "romance": 4.8438215255737305, "sci_fi": -3.387235164642334, "kids": -5.298741817474365 }, "probabilities": { "comedy": 0.0037884599766887456, "drama": 0.3055467915270979, "action": 0.0069744147310686405, "documentary": 0.09804644836647554, "horror": 0.003601667122861362, "romance": 0.9921846658577527, "sci_fi": 0.03269678782113664, "kids": 0.00497302362933734 } }, { "domain": "screen_tags", "row": 20, "task": "format", "multi_label": false, "gold": [ "series" ], "selected": [ "series" ], "correct": true, "logits": { "film": -1.0660053491592407, "series": 1.0035345554351807 }, "probabilities": { "film": 0.11209282277045801, "series": 0.887907177229542 } }, { "domain": "screen_tags", "row": 30, "task": "genres", "multi_label": true, "gold": [ "drama", "romance" ], "selected": [], "correct": false, "logits": { "comedy": -3.204225778579712, "drama": -0.9203282594680786, "action": -2.883401393890381, "documentary": -1.5630481243133545, "horror": -2.047767162322998, "romance": -1.421515703201294, "sci_fi": -1.2221946716308594, "kids": -1.1848385334014893 }, "probabilities": { "comedy": 0.03900700858723356, "drama": 0.2848910138914326, "action": 0.05298021747846474, "documentary": 0.17320969602850333, "horror": 0.11427819160278246, "romance": 0.19442407911094686, "sci_fi": 0.2275504597582944, "kids": 0.2341833305990129 } }, { "domain": "screen_tags", "row": 30, "task": "format", "multi_label": false, "gold": [ "film" ], "selected": [ "film" ], "correct": true, "logits": { "film": -0.09152739495038986, "series": -0.13151687383651733 }, "probabilities": { "film": 0.5099960376530517, "series": 0.4900039623469483 } }, { "domain": "screen_tags", "row": 40, "task": "genres", "multi_label": true, "gold": [ "documentary", "drama" ], "selected": [ "documentary" ], "correct": false, "logits": { "comedy": -3.613978147506714, "drama": -1.0142854452133179, "action": -2.410386085510254, "documentary": 0.8154875040054321, "horror": -3.1783998012542725, "romance": -2.875196695327759, "sci_fi": -1.8965204954147339, "kids": -2.367314577102661 }, "probabilities": { "comedy": 0.02623748981663333, "drama": 0.26614202016886296, "action": 0.0823841265221003, "documentary": 0.6932776237772804, "horror": 0.0399867168313009, "romance": 0.05339338744624914, "sci_fi": 0.13050279269940834, "kids": 0.08569932163678341 } }, { "domain": "screen_tags", "row": 40, "task": "format", "multi_label": false, "gold": [ "film" ], "selected": [ "film" ], "correct": true, "logits": { "film": 1.4788554906845093, "series": -1.5655066967010498 }, "probabilities": { "film": 0.9545385010284407, "series": 0.045461498971559294 } }, { "domain": "screen_tags", "row": 50, "task": "genres", "multi_label": true, "gold": [ "comedy", "drama", "kids" ], "selected": [ "comedy" ], "correct": false, "logits": { "comedy": 2.1845204830169678, "drama": -1.537508249282837, "action": -4.044243812561035, "documentary": -2.0570130348205566, "horror": -5.662056922912598, "romance": -2.364374876022339, "sci_fi": -2.1326537132263184, "kids": -3.125180721282959 }, "probabilities": { "comedy": 0.8988508071857344, "drama": 0.1768977940055583, "action": 0.01722118432524394, "documentary": 0.1133456688640891, "horror": 0.0034633246879859155, "romance": 0.08592994247347856, "sci_fi": 0.1059633288763902, "kids": 0.04208044249584948 } }, { "domain": "screen_tags", "row": 50, "task": "format", "multi_label": false, "gold": [ "film" ], "selected": [ "film" ], "correct": true, "logits": { "film": 1.8330962657928467, "series": -2.1542279720306396 }, "probabilities": { "film": 0.9817885284667955, "series": 0.018211471533204533 } }, { "domain": "screen_tags", "row": 60, "task": "genres", "multi_label": true, "gold": [ "comedy" ], "selected": [ "comedy" ], "correct": true, "logits": { "comedy": 1.8893413543701172, "drama": -0.785805881023407, "action": -1.4932132959365845, "documentary": -2.7813451290130615, "horror": -2.0630042552948, "romance": -4.170058250427246, "sci_fi": -2.5013813972473145, "kids": -4.316948413848877 }, "probabilities": { "comedy": 0.8686804139676157, "drama": 0.313069936776157, "action": 0.1834399190549736, "documentary": 0.05834061429074781, "horror": 0.11274495399504214, "romance": 0.015216248318008323, "sci_fi": 0.07576139566233574, "kids": 0.0131649044518295 } }, { "domain": "screen_tags", "row": 60, "task": "format", "multi_label": false, "gold": [ "film" ], "selected": [ "series" ], "correct": false, "logits": { "film": -0.0370049811899662, "series": 0.036289580166339874 }, "probabilities": { "film": 0.48168455828115603, "series": 0.518315441718844 } }, { "domain": "screen_tags", "row": 70, "task": "genres", "multi_label": true, "gold": [ "documentary", "drama" ], "selected": [], "correct": false, "logits": { "comedy": -2.78193736076355, "drama": -0.9019675254821777, "action": -2.3747398853302, "documentary": -0.2657352387905121, "horror": -2.96401047706604, "romance": -1.9330910444259644, "sci_fi": -1.4639135599136353, "kids": -1.3651063442230225 }, "probabilities": { "comedy": 0.05830808737160632, "drama": 0.2886463381597699, "action": 0.08511929899198424, "documentary": 0.4339543857732985, "horror": 0.04907849908312815, "romance": 0.12640884281481846, "sci_fi": 0.18786948607905896, "kids": 0.20341164084268754 } }, { "domain": "screen_tags", "row": 70, "task": "format", "multi_label": false, "gold": [ "film" ], "selected": [ "series" ], "correct": false, "logits": { "film": -0.18295902013778687, "series": 0.14941996335983276 }, "probabilities": { "film": 0.41766189266508424, "series": 0.5823381073349158 } }, { "domain": "screen_tags", "row": 80, "task": "genres", "multi_label": true, "gold": [ "drama", "sci_fi" ], "selected": [ "drama", "sci_fi" ], "correct": true, "logits": { "comedy": -6.569530487060547, "drama": 0.30838778614997864, "action": -3.2965269088745117, "documentary": -3.990859031677246, "horror": -4.803251266479492, "romance": -3.2902917861938477, "sci_fi": 3.979696750640869, "kids": -5.195054531097412 }, "probabilities": { "comedy": 0.0014004916012230213, "drama": 0.5764916893190741, "action": 0.035690529094805336, "documentary": 0.018148377593278645, "horror": 0.008136291130551792, "romance": 0.0359057438918218, "sci_fi": 0.9816516480665461, "kids": 0.005513348461362822 } }, { "domain": "screen_tags", "row": 80, "task": "format", "multi_label": false, "gold": [ "series" ], "selected": [ "series" ], "correct": true, "logits": { "film": -2.416459083557129, "series": 2.375802755355835 }, "probabilities": { "film": 0.008225457896717008, "series": 0.991774542103283 } }, { "domain": "screen_tags", "row": 90, "task": "genres", "multi_label": true, "gold": [ "drama" ], "selected": [ "action" ], "correct": false, "logits": { "comedy": -3.3035330772399902, "drama": -0.8576457500457764, "action": 0.16837501525878906, "documentary": -2.6231608390808105, "horror": -2.961287498474121, "romance": -3.023698568344116, "sci_fi": -0.7172685265541077, "kids": -3.7781009674072266 }, "probabilities": { "comedy": 0.03545018263081371, "drama": 0.2978314505043221, "action": 0.5419945879372221, "documentary": 0.06766262127295165, "horror": 0.049205736098747066, "romance": 0.046366661022482436, "sci_fi": 0.3279947553516656, "kids": 0.022354904741043093 } }, { "domain": "screen_tags", "row": 90, "task": "format", "multi_label": false, "gold": [ "film" ], "selected": [ "film" ], "correct": true, "logits": { "film": 0.10117527097463608, "series": -0.22933827340602875 }, "probabilities": { "film": 0.5818843248514962, "series": 0.4181156751485038 } }, { "domain": "sports_recap", "row": 0, "task": "sport", "multi_label": false, "gold": [ "motorsport" ], "selected": [ "motorsport" ], "correct": true, "logits": { "football": -3.400987148284912, "basketball": -3.510934352874756, "baseball": -3.4788882732391357, "soccer": -2.733582019805908, "tennis": -3.1987273693084717, "cricket": -3.378969669342041, "motorsport": 0.7192809581756592, "other": -1.979027509689331 }, "probabilities": { "football": 0.013748062442744487, "basketball": 0.012316633874970742, "baseball": 0.012717726097023997, "soccer": 0.026797350733621543, "tennis": 0.016829910324986163, "cricket": 0.014054117029568838, "motorsport": 0.8465472409412304, "other": 0.056988958555853876 } }, { "domain": "sports_recap", "row": 0, "task": "result", "multi_label": false, "gold": [ "draw" ], "selected": [ "draw" ], "correct": true, "logits": { "win": -3.676682710647583, "loss": -4.701770782470703, "draw": 1.828000783920288, "upcoming": -2.0109682083129883 }, "probabilities": { "win": 0.003960570939043005, "loss": 0.0014209137059466575, "draw": 0.9736692685491443, "upcoming": 0.020949246805866073 } }, { "domain": "sports_recap", "row": 0, "task": "upset", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.6043593287467957, "no": 0.6432046294212341 }, "probabilities": { "yes": 0.2231221141693382, "no": 0.7768778858306619 } }, { "domain": "sports_recap", "row": 10, "task": "sport", "multi_label": false, "gold": [ "tennis" ], "selected": [ "tennis" ], "correct": true, "logits": { "football": -5.414678573608398, "basketball": -5.232568740844727, "baseball": -5.431250095367432, "soccer": -4.777647972106934, "tennis": 2.662224054336548, "cricket": -5.146073341369629, "motorsport": -5.1765546798706055, "other": -3.4512674808502197 }, "probabilities": { "football": 0.00030921255523955036, "basketball": 0.0003709765133294371, "baseball": 0.00030413065634800866, "soccer": 0.0005846769795103129, "tennis": 0.995431459107294, "cricket": 0.00040449288781604106, "motorsport": 0.0003923494179941239, "other": 0.0022027018824683988 } }, { "domain": "sports_recap", "row": 10, "task": "result", "multi_label": false, "gold": [ "draw" ], "selected": [ "draw" ], "correct": true, "logits": { "win": -4.533146858215332, "loss": -3.7268972396850586, "draw": 2.998802900314331, "upcoming": -3.8267040252685547 }, "probabilities": { "win": 0.0005341857755446905, "loss": 0.0011963054456787068, "draw": 0.9971868376806695, "upcoming": 0.0010826710981070304 } }, { "domain": "sports_recap", "row": 10, "task": "upset", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 0.8553258180618286, "no": -0.7763783931732178 }, "probabilities": { "yes": 0.8364029648994025, "no": 0.16359703510059748 } }, { "domain": "sports_recap", "row": 20, "task": "sport", "multi_label": false, "gold": [ "cricket" ], "selected": [ "cricket" ], "correct": true, "logits": { "football": -4.672085285186768, "basketball": -4.861149311065674, "baseball": -4.279763221740723, "soccer": -4.603326320648193, "tennis": -4.935629367828369, "cricket": 1.961493968963623, "motorsport": -4.188022613525391, "other": -2.6175880432128906 }, "probabilities": { "football": 0.0012907031854850876, "basketball": 0.0010683582763250227, "baseball": 0.0019107756103172893, "soccer": 0.001382572838943997, "tennis": 0.00099167791412842, "cricket": 0.9811903025059421, "motorsport": 0.0020943638308052797, "other": 0.010071245838052889 } }, { "domain": "sports_recap", "row": 20, "task": "result", "multi_label": false, "gold": [ "win" ], "selected": [ "win" ], "correct": true, "logits": { "win": 3.8963406085968018, "loss": -4.344655990600586, "draw": -6.239899635314941, "upcoming": -4.4029221534729 }, "probabilities": { "win": 0.999448365489062, "loss": 0.0002634759709676722, "draw": 3.959562084151667e-05, "upcoming": 0.00024856291912878493 } }, { "domain": "sports_recap", "row": 20, "task": "upset", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 2.00181245803833, "no": -1.975730538368225 }, "probabilities": { "yes": 0.9816128150443403, "no": 0.018387184955659684 } }, { "domain": "sports_recap", "row": 30, "task": "sport", "multi_label": false, "gold": [ "tennis" ], "selected": [ "tennis" ], "correct": true, "logits": { "football": -3.2752814292907715, "basketball": -2.6770524978637695, "baseball": -3.264598846435547, "soccer": -2.9583659172058105, "tennis": -0.15984034538269043, "cricket": -2.0688295364379883, "motorsport": -2.5587387084960938, "other": -1.7713016271591187 }, "probabilities": { "football": 0.026571432214184364, "basketball": 0.04833063348096818, "baseball": 0.026856805287308393, "soccer": 0.03647956255743309, "tennis": 0.5990096724566316, "cricket": 0.08879128955620237, "motorsport": 0.054400828446696955, "other": 0.11955977600057496 } }, { "domain": "sports_recap", "row": 30, "task": "result", "multi_label": false, "gold": [ "loss" ], "selected": [ "loss" ], "correct": true, "logits": { "win": -2.5868449211120605, "loss": 2.4740378856658936, "draw": -4.7552385330200195, "upcoming": -4.556288242340088 }, "probabilities": { "win": 0.006289957132946432, "loss": 0.9921130498709704, "draw": 0.0007193269064794409, "upcoming": 0.0008776660896038151 } }, { "domain": "sports_recap", "row": 30, "task": "upset", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.590677797794342, "no": 0.5717441439628601 }, "probabilities": { "yes": 0.23822748429556656, "no": 0.7617725157044334 } }, { "domain": "sports_recap", "row": 40, "task": "sport", "multi_label": false, "gold": [ "basketball" ], "selected": [ "basketball" ], "correct": true, "logits": { "football": -4.723170280456543, "basketball": 2.677088737487793, "baseball": -5.457226276397705, "soccer": -4.841002941131592, "tennis": -5.834390163421631, "cricket": -5.524484634399414, "motorsport": -5.0005998611450195, "other": -3.267233371734619 }, "probabilities": { "football": 0.0006080501602969047, "basketball": 0.9950182891189338, "baseball": 0.00029183873815205407, "soccer": 0.0005404622102622606, "tennis": 0.00020014407777878463, "cricket": 0.0002728556839275397, "motorsport": 0.00046073719216340847, "other": 0.0026076228184852865 } }, { "domain": "sports_recap", "row": 40, "task": "result", "multi_label": false, "gold": [ "draw" ], "selected": [ "draw" ], "correct": true, "logits": { "win": -6.668080806732178, "loss": -6.308037757873535, "draw": 5.8125319480896, "upcoming": -6.287997722625732 }, "probabilities": { "win": 3.7995513757562284e-06, "loss": 5.446243198511768e-06, "draw": 0.9999851977183656, "upcoming": 5.556487060116706e-06 } }, { "domain": "sports_recap", "row": 40, "task": "upset", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -3.944629430770874, "no": 4.244594573974609 }, "probabilities": { "yes": 0.0002775521830433724, "no": 0.9997224478169566 } }, { "domain": "sports_recap", "row": 50, "task": "sport", "multi_label": false, "gold": [ "football" ], "selected": [ "football" ], "correct": true, "logits": { "football": 0.32519882917404175, "basketball": -2.939699649810791, "baseball": -3.4772419929504395, "soccer": -3.1595776081085205, "tennis": -3.828579902648926, "cricket": -3.2690088748931885, "motorsport": -3.2935585975646973, "other": -1.7325758934020996 }, "probabilities": { "football": 0.7758439727808626, "basketball": 0.029637870577356923, "baseball": 0.017313916703319155, "soccer": 0.023787851202619083, "tennis": 0.012184598085165893, "cricket": 0.021322092247000626, "motorsport": 0.020805013830940848, "other": 0.09910468457273489 } }, { "domain": "sports_recap", "row": 50, "task": "result", "multi_label": false, "gold": [ "upcoming" ], "selected": [ "upcoming" ], "correct": true, "logits": { "win": -2.2632012367248535, "loss": -2.377628803253174, "draw": -1.5546538829803467, "upcoming": 0.8554320335388184 }, "probabilities": { "win": 0.03768129709503393, "loss": 0.03360706479264796, "draw": 0.07653217394237909, "upcoming": 0.8521794641699391 } }, { "domain": "sports_recap", "row": 50, "task": "upset", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.1575095653533936, "no": 1.2352163791656494 }, "probabilities": { "yes": 0.0837290640548393, "no": 0.9162709359451606 } }, { "domain": "sports_recap", "row": 60, "task": "sport", "multi_label": false, "gold": [ "tennis" ], "selected": [ "tennis" ], "correct": true, "logits": { "football": -5.259875297546387, "basketball": -5.162533760070801, "baseball": -5.3495001792907715, "soccer": -4.878846168518066, "tennis": 2.9278595447540283, "cricket": -5.017453193664551, "motorsport": -5.132083892822266, "other": -4.155551910400391 }, "probabilities": { "football": 0.00027727911022463, "basketball": 0.00030562722931843397, "baseball": 0.0002535091039175191, "soccer": 0.00040587845629429806, "tennis": 0.9972526821604895, "cricket": 0.00035334563748492186, "motorsport": 0.0003150766749497552, "other": 0.0008366016273210079 } }, { "domain": "sports_recap", "row": 60, "task": "result", "multi_label": false, "gold": [ "draw" ], "selected": [ "win" ], "correct": false, "logits": { "win": 0.818681538105011, "loss": -0.5058917999267578, "draw": -4.174739360809326, "upcoming": -4.1662116050720215 }, "probabilities": { "win": 0.7815312761461007, "loss": 0.20782197664046217, "draw": 0.005300675529365443, "upcoming": 0.005346071684071686 } }, { "domain": "sports_recap", "row": 60, "task": "upset", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 0.6863738298416138, "no": -0.5484247207641602 }, "probabilities": { "yes": 0.7746573291792215, "no": 0.22534267082077844 } }, { "domain": "sports_recap", "row": 70, "task": "sport", "multi_label": false, "gold": [ "football" ], "selected": [ "football" ], "correct": true, "logits": { "football": 1.2898449897766113, "basketball": -4.258035659790039, "baseball": -4.554379463195801, "soccer": -4.230870723724365, "tennis": -4.788171291351318, "cricket": -4.311444282531738, "motorsport": -3.599846363067627, "other": -2.0887200832366943 }, "probabilities": { "football": 0.9448213233465573, "basketball": 0.003680745025333091, "baseball": 0.0027367508095799357, "soccer": 0.0037821026829044408, "tennis": 0.002166210987526019, "cricket": 0.0034893189066613397, "motorsport": 0.007108594045046744, "other": 0.03221495419639105 } }, { "domain": "sports_recap", "row": 70, "task": "result", "multi_label": false, "gold": [ "win" ], "selected": [ "win" ], "correct": true, "logits": { "win": 3.388063669204712, "loss": -3.5750937461853027, "draw": -5.62938117980957, "upcoming": -4.755593299865723 }, "probabilities": { "win": 0.9986438886809865, "loss": 0.0009448215851671875, "draw": 0.00012111114494869964, "upcoming": 0.0002901785888976055 } }, { "domain": "sports_recap", "row": 70, "task": "upset", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 1.4496431350708008, "no": -1.4851633310317993 }, "probabilities": { "yes": 0.9495404672778921, "no": 0.050459532722107854 } }, { "domain": "sports_recap", "row": 80, "task": "sport", "multi_label": false, "gold": [ "football" ], "selected": [ "basketball" ], "correct": false, "logits": { "football": -1.665309190750122, "basketball": -0.9555305242538452, "baseball": -2.5965094566345215, "soccer": -2.39086651802063, "tennis": -3.277282476425171, "cricket": -2.580820083618164, "motorsport": -2.4128594398498535, "other": -1.5063152313232422 }, "probabilities": { "football": 0.16240784832068247, "basketball": 0.3302630374545199, "baseball": 0.06400175390650192, "soccer": 0.07861428503637523, "tennis": 0.03239932107108564, "cricket": 0.06501381987704669, "motorsport": 0.07690420101150824, "other": 0.19039573332227983 } }, { "domain": "sports_recap", "row": 80, "task": "result", "multi_label": false, "gold": [ "loss" ], "selected": [ "loss" ], "correct": true, "logits": { "win": -4.127989292144775, "loss": 3.834480047225952, "draw": -5.878139972686768, "upcoming": -5.301133155822754 }, "probabilities": { "win": 0.0003481121806428606, "loss": 0.9994837007321965, "draw": 6.048371198827499e-05, "upcoming": 0.00010770337517240105 } }, { "domain": "sports_recap", "row": 80, "task": "upset", "multi_label": false, "gold": [ "no" ], "selected": [ "yes" ], "correct": false, "logits": { "yes": 1.061614990234375, "no": -0.9275360703468323 }, "probabilities": { "yes": 0.8796532949015496, "no": 0.12034670509845036 } }, { "domain": "sports_recap", "row": 90, "task": "sport", "multi_label": false, "gold": [ "other" ], "selected": [ "other" ], "correct": true, "logits": { "football": -2.5329437255859375, "basketball": -2.619809865951538, "baseball": -1.6608694791793823, "soccer": -2.611584186553955, "tennis": -2.1519112586975098, "cricket": -2.049041748046875, "motorsport": -2.1915359497070312, "other": -0.997367262840271 }, "probabilities": { "football": 0.06958866947773984, "basketball": 0.06379887894887967, "baseball": 0.16644684440644866, "soccer": 0.06432583237680128, "tennis": 0.10186355545069387, "cricket": 0.11290014288759928, "motorsport": 0.09790616648152356, "other": 0.3231699099703138 } }, { "domain": "sports_recap", "row": 90, "task": "result", "multi_label": false, "gold": [ "draw" ], "selected": [ "draw" ], "correct": true, "logits": { "win": -3.6668667793273926, "loss": -2.636640787124634, "draw": 2.2980666160583496, "upcoming": -3.6755192279815674 }, "probabilities": { "win": 0.0025360102891794295, "loss": 0.007105137301670732, "draw": 0.9878446901629448, "upcoming": 0.0025141622462049283 } }, { "domain": "sports_recap", "row": 90, "task": "upset", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.231247216463089, "no": 0.30887070298194885 }, "probabilities": { "yes": 0.36816015162201804, "no": 0.631839848377982 } }, { "domain": "support_intent", "row": 0, "task": "intent", "multi_label": false, "gold": [ "refund_request" ], "selected": [ "refund_request" ], "correct": true, "logits": { "order_status": -3.5186607837677, "refund_request": 1.5239100456237793, "cancel_subscription": -3.1116373538970947, "update_payment": -2.3455512523651123, "login_problem": -4.424450874328613, "password_reset": -4.934874057769775, "shipping_delay": -5.396086692810059, "missing_item": -5.648200988769531, "damaged_item": -5.854650974273682, "change_address": -6.035045623779297, "change_order": -5.494908332824707, "product_question": -3.886061906814575, "pricing_question": -3.176896095275879, "upgrade_plan": -5.345699310302734, "downgrade_plan": -4.983253002166748, "bug_report": -3.7983171939849854, "feature_request": -5.202879905700684, "speak_to_human": -2.149244546890259, "complaint": -2.2623345851898193, "invoice_copy": -4.35495138168335, "tax_document": -5.339015960693359, "appointment": -4.816526412963867, "outage": -4.068275451660156, "data_export": -4.866534233093262, "close_account": -4.3677167892456055, "promo_code": -4.8877973556518555, "partner_inquiry": -4.436179161071777, "other": -3.2363131046295166 }, "probabilities": { "order_status": 0.005655173013817806, "refund_request": 0.8758033096227752, "cancel_subscription": 0.008495988694470977, "update_payment": 0.01827767254328219, "login_problem": 0.002285947532344102, "password_reset": 0.0013721206049702314, "shipping_delay": 0.0008651475529931024, "missing_item": 0.0006723545314189979, "damaged_item": 0.0005469381870194292, "change_address": 0.00045666091852814147, "change_order": 0.0007837408630312162, "product_question": 0.003916387140110467, "pricing_question": 0.007959254919771477, "upgrade_plan": 0.000909857010992192, "downgrade_plan": 0.0013073190134505353, "bug_report": 0.004275556604183041, "feature_request": 0.0010495395897003467, "speak_to_human": 0.022242101143955376, "complaint": 0.01986375873951202, "invoice_copy": 0.002450470644794663, "tax_document": 0.000915958269201555, "appointment": 0.0015445074775094036, "outage": 0.003264008482968584, "data_export": 0.0014691694698280397, "close_account": 0.0024193882004169016, "promo_code": 0.0014382601185475151, "partner_inquiry": 0.002259293890504751, "other": 0.0075001152199017825 } }, { "domain": "support_intent", "row": 10, "task": "intent", "multi_label": false, "gold": [ "complaint" ], "selected": [ "order_status" ], "correct": false, "logits": { "order_status": -1.4516880512237549, "refund_request": -3.1880106925964355, "cancel_subscription": -4.704874515533447, "update_payment": -2.303356885910034, "login_problem": -3.4879207611083984, "password_reset": -3.9582409858703613, "shipping_delay": -1.7185343503952026, "missing_item": -3.4692652225494385, "damaged_item": -4.839423656463623, "change_address": -5.498114109039307, "change_order": -4.1989593505859375, "product_question": -3.3880226612091064, "pricing_question": -4.069026947021484, "upgrade_plan": -5.375584602355957, "downgrade_plan": -5.262137413024902, "bug_report": -2.692237615585327, "feature_request": -4.178234100341797, "speak_to_human": -2.3340399265289307, "complaint": -2.1910200119018555, "invoice_copy": -5.27106237411499, "tax_document": -4.861237525939941, "appointment": -4.249571800231934, "outage": -3.337674379348755, "data_export": -4.253271579742432, "close_account": -4.509016036987305, "promo_code": -4.991664886474609, "partner_inquiry": -3.830404758453369, "other": -2.385430335998535 }, "probabilities": { "order_status": 0.19001013167155384, "refund_request": 0.03347352252564259, "cancel_subscription": 0.0073440534476087805, "update_payment": 0.0810777491738892, "login_problem": 0.024800025600317306, "password_reset": 0.01549510954135362, "shipping_delay": 0.14550800410864279, "missing_item": 0.025267025961726752, "damaged_item": 0.006419509995407041, "change_address": 0.0033222801438441375, "change_order": 0.012180131954961208, "product_question": 0.02740547429812232, "pricing_question": 0.013870142379074699, "upgrade_plan": 0.0037553475750462585, "downgrade_plan": 0.004206487740241036, "bug_report": 0.05495572334684141, "feature_request": 0.012435202306069943, "speak_to_human": 0.07862781521853406, "complaint": 0.09071706023594624, "invoice_copy": 0.004169112037277043, "tax_document": 0.006280991942345656, "appointment": 0.011579006179570273, "outage": 0.02882061896310966, "data_export": 0.011536245560962374, "close_account": 0.008932974316256356, "promo_code": 0.005512953803830908, "partner_inquiry": 0.017608129311375315, "other": 0.07468917066044928 } }, { "domain": "support_intent", "row": 20, "task": "intent", "multi_label": false, "gold": [ "pricing_question" ], "selected": [ "pricing_question" ], "correct": true, "logits": { "order_status": -4.491642951965332, "refund_request": -6.815442085266113, "cancel_subscription": -8.441645622253418, "update_payment": -2.058032274246216, "login_problem": -7.128695487976074, "password_reset": -8.183091163635254, "shipping_delay": -6.486687660217285, "missing_item": -8.398539543151855, "damaged_item": -9.144052505493164, "change_address": -8.046749114990234, "change_order": -5.480678558349609, "product_question": -4.636953353881836, "pricing_question": 4.835758209228516, "upgrade_plan": -8.003410339355469, "downgrade_plan": -9.1685209274292, "bug_report": -5.824034214019775, "feature_request": -8.981409072875977, "speak_to_human": -4.24939489364624, "complaint": -4.907851696014404, "invoice_copy": -7.373313903808594, "tax_document": -5.077184677124023, "appointment": -8.324347496032715, "outage": -7.798393249511719, "data_export": -8.237954139709473, "close_account": -8.843329429626465, "promo_code": -7.863395690917969, "partner_inquiry": -7.232419967651367, "other": -5.47210693359375 }, "probabilities": { "order_status": 8.881506035089204e-05, "refund_request": 8.695077847178845e-06, "cancel_subscription": 1.710102933745831e-06, "update_payment": 0.0010124889768737056, "login_problem": 6.356663915559381e-06, "password_reset": 2.2146802191639996e-06, "shipping_delay": 1.207952083704039e-05, "missing_item": 1.7854306434658702e-06, "damaged_item": 8.471704885825267e-07, "change_address": 2.538187062180762e-06, "change_order": 3.303344812806217e-05, "product_question": 7.680316361796663e-05, "pricing_question": 0.9984480429264027, "upgrade_plan": 2.6506074675708686e-06, "downgrade_plan": 8.266931098344351e-07, "bug_report": 2.343346132563671e-05, "feature_request": 9.967952594321274e-07, "speak_to_human": 0.00011316017495112197, "complaint": 5.857731409332115e-05, "invoice_copy": 4.977288586991786e-06, "tax_document": 4.9452593766836324e-05, "appointment": 1.9229331259340474e-06, "outage": 3.2537427093011733e-06, "data_export": 2.096449178921688e-06, "close_account": 1.1443877255850997e-06, "promo_code": 3.048968983306552e-06, "partner_inquiry": 5.7303649424360935e-06, "other": 3.3317815453710206e-05 } }, { "domain": "support_intent", "row": 30, "task": "intent", "multi_label": false, "gold": [ "password_reset" ], "selected": [ "password_reset" ], "correct": true, "logits": { "order_status": -4.687734127044678, "refund_request": -6.288403034210205, "cancel_subscription": -7.17341423034668, "update_payment": -1.739346981048584, "login_problem": -3.42808198928833, "password_reset": 3.3442718982696533, "shipping_delay": -6.720631122589111, "missing_item": -7.3936357498168945, "damaged_item": -8.05437183380127, "change_address": -7.623082160949707, "change_order": -6.441295623779297, "product_question": -5.382472991943359, "pricing_question": -6.704890251159668, "upgrade_plan": -6.258548736572266, "downgrade_plan": -7.749270915985107, "bug_report": -6.214292526245117, "feature_request": -6.98673677444458, "speak_to_human": -3.631901741027832, "complaint": -6.529404640197754, "invoice_copy": -7.505431175231934, "tax_document": -8.514178276062012, "appointment": -6.438354969024658, "outage": -7.079923629760742, "data_export": -6.959606170654297, "close_account": -6.211109161376953, "promo_code": -7.015597343444824, "partner_inquiry": -6.80777645111084, "other": -5.281686305999756 }, "probabilities": { "order_status": 0.000321757810929504, "refund_request": 6.491834279291473e-05, "cancel_subscription": 2.6792420203151402e-05, "update_payment": 0.006137582835131718, "login_problem": 0.0011339373777623488, "password_reset": 0.9903415354193184, "shipping_delay": 4.2135985035369484e-05, "missing_item": 2.149665840008964e-05, "damaged_item": 1.1102401272945798e-05, "change_address": 1.7089275254824225e-05, "change_order": 5.5714343421598595e-05, "product_question": 0.0001606230407124688, "pricing_question": 4.280448977879723e-05, "upgrade_plan": 6.688565455985202e-05, "downgrade_plan": 1.5063315240461752e-05, "bug_report": 6.991223875331766e-05, "feature_request": 3.2291255795585444e-05, "speak_to_human": 0.0009248499504197469, "complaint": 5.101545415257028e-05, "invoice_copy": 1.9222896111821797e-05, "tax_document": 7.010121081347175e-06, "appointment": 5.5878421200295056e-05, "outage": 2.9418084958174917e-05, "data_export": 3.3179329578423e-05, "close_account": 7.013514953294735e-05, "promo_code": 3.137263155057289e-05, "partner_inquiry": 3.861947946415523e-05, "other": 0.00017765561758658146 } }, { "domain": "support_intent", "row": 40, "task": "intent", "multi_label": false, "gold": [ "change_address" ], "selected": [ "change_address" ], "correct": true, "logits": { "order_status": -4.1927690505981445, "refund_request": -8.167494773864746, "cancel_subscription": -5.4688239097595215, "update_payment": -5.118967056274414, "login_problem": -5.900670051574707, "password_reset": -7.59567928314209, "shipping_delay": -5.723016738891602, "missing_item": -8.587674140930176, "damaged_item": -11.691245079040527, "change_address": 5.517696857452393, "change_order": -4.430087566375732, "product_question": -4.768746376037598, "pricing_question": -7.144040107727051, "upgrade_plan": -8.183262825012207, "downgrade_plan": -9.760907173156738, "bug_report": -8.161932945251465, "feature_request": -6.281453609466553, "speak_to_human": -5.164578914642334, "complaint": -7.989273548126221, "invoice_copy": -9.97068977355957, "tax_document": -12.190185546875, "appointment": -6.725860595703125, "outage": -8.550196647644043, "data_export": -8.45056438446045, "close_account": -6.169419288635254, "promo_code": -10.344805717468262, "partner_inquiry": -5.962368965148926, "other": -4.057329177856445 }, "probabilities": { "order_status": 6.062459546483338e-05, "refund_request": 1.13879985970215e-06, "cancel_subscription": 1.6922529105512796e-05, "update_payment": 2.4010774589042567e-05, "login_problem": 1.0987955029293217e-05, "password_reset": 2.0173614484753607e-06, "shipping_delay": 1.3124136137244967e-05, "missing_item": 7.481106277801783e-07, "damaged_item": 3.358165470805268e-08, "change_address": 0.9996560857287067, "change_order": 4.7817044752322046e-05, "product_question": 3.408043108370679e-05, "pricing_question": 3.1690429042807697e-06, "upgrade_plan": 1.1209840348163115e-06, "downgrade_plan": 2.3143934554554692e-07, "bug_report": 1.1451513158288994e-06, "feature_request": 7.50835287601624e-06, "speak_to_human": 2.294019959595756e-05, "complaint": 1.360967956030957e-06, "invoice_copy": 1.876418762312765e-07, "tax_document": 2.038989549355116e-08, "appointment": 4.814388948517736e-06, "outage": 7.766799481957204e-07, "data_export": 8.580484973514541e-07, "close_account": 8.398477491868234e-06, "promo_code": 1.2907831182775583e-07, "partner_inquiry": 1.0330500798226001e-05, "other": 6.941760774460704e-05 } }, { "domain": "support_intent", "row": 50, "task": "intent", "multi_label": false, "gold": [ "order_status" ], "selected": [ "order_status" ], "correct": true, "logits": { "order_status": 2.9859704971313477, "refund_request": -0.49388206005096436, "cancel_subscription": -4.99761438369751, "update_payment": -2.006225109100342, "login_problem": -6.223259925842285, "password_reset": -6.900552749633789, "shipping_delay": -3.5494110584259033, "missing_item": -6.416808128356934, "damaged_item": -7.941070079803467, "change_address": -7.82003116607666, "change_order": -5.33168888092041, "product_question": -4.183624744415283, "pricing_question": -3.234800100326538, "upgrade_plan": -6.592967987060547, "downgrade_plan": -6.7244367599487305, "bug_report": -5.988765716552734, "feature_request": -7.10624885559082, "speak_to_human": -4.311764717102051, "complaint": -4.647653579711914, "invoice_copy": -6.462778091430664, "tax_document": -7.546663284301758, "appointment": -6.124322891235352, "outage": -5.983854293823242, "data_export": -6.989690780639648, "close_account": -6.129400253295898, "promo_code": -6.60182523727417, "partner_inquiry": -6.403005599975586, "other": -4.17519474029541 }, "probabilities": { "order_status": 0.9564299788261601, "refund_request": 0.02946947622404479, "cancel_subscription": 0.00032615671239705544, "update_payment": 0.006494865712250155, "login_problem": 9.574921566997536e-05, "password_reset": 4.863967776794999e-05, "shipping_delay": 0.0013879474831550295, "missing_item": 7.890023719146481e-05, "damaged_item": 1.718303023635248e-05, "change_address": 1.9393950670923123e-05, "change_order": 0.00023352834964914787, "product_question": 0.0007361012064344821, "pricing_question": 0.001901108207273793, "upgrade_plan": 6.615658124765498e-05, "downgrade_plan": 5.80065307000522e-05, "bug_report": 0.00012105277964210286, "feature_request": 3.959660993296138e-05, "speak_to_human": 0.0006475704853491117, "complaint": 0.00046282026525321557, "invoice_copy": 7.5355300752269e-05, "tax_document": 2.5491092518272134e-05, "appointment": 0.00010570682614900753, "outage": 0.00012164878343071231, "data_export": 4.4491652254724256e-05, "close_account": 0.00010517147455740344, "promo_code": 6.557320323067104e-05, "partner_inquiry": 7.999681028628964e-05, "other": 0.0007423327717944072 } }, { "domain": "support_intent", "row": 60, "task": "intent", "multi_label": false, "gold": [ "speak_to_human" ], "selected": [ "speak_to_human" ], "correct": true, "logits": { "order_status": -6.4910407066345215, "refund_request": -1.8264200687408447, "cancel_subscription": -7.577895641326904, "update_payment": -8.464179039001465, "login_problem": -7.352111339569092, "password_reset": -7.676997184753418, "shipping_delay": -9.0587739944458, "missing_item": -7.966598033905029, "damaged_item": -3.3628649711608887, "change_address": -9.298197746276855, "change_order": -8.462986946105957, "product_question": -3.73840069770813, "pricing_question": -6.148716449737549, "upgrade_plan": -8.807968139648438, "downgrade_plan": -8.4667329788208, "bug_report": -6.261174201965332, "feature_request": -7.729066848754883, "speak_to_human": 2.063673496246338, "complaint": -5.440621376037598, "invoice_copy": -9.335543632507324, "tax_document": -9.001171112060547, "appointment": -6.523494720458984, "outage": -7.208323955535889, "data_export": -7.699976921081543, "close_account": -7.40717887878418, "promo_code": -8.19556999206543, "partner_inquiry": -5.505152225494385, "other": -4.822411060333252 }, "probabilities": { "order_status": 0.0001867401842293622, "refund_request": 0.019817861371306922, "cancel_subscription": 6.298290415319612e-05, "update_payment": 2.596059805799783e-05, "login_problem": 7.89368076079218e-05, "password_reset": 5.704051378359953e-05, "shipping_delay": 1.4324695041166733e-05, "missing_item": 4.269837803311708e-05, "damaged_item": 0.004263705927342769, "change_address": 1.127469937394347e-05, "change_order": 2.5991563955952283e-05, "product_question": 0.002928829899484248, "pricing_question": 0.00026297067353995456, "upgrade_plan": 1.840810079997587e-05, "downgrade_plan": 2.589438084626303e-05, "bug_report": 0.00023499982425196934, "feature_request": 5.4146434125663165e-05, "speak_to_human": 0.9693998663179813, "complaint": 0.00053386211403921, "invoice_copy": 1.0861401260975278e-05, "tax_document": 1.5174067058133266e-05, "appointment": 0.00018077700367153632, "outage": 9.114348406385427e-05, "data_export": 5.574468375639997e-05, "close_account": 7.470747017346823e-05, "promo_code": 3.396019067699278e-05, "partner_inquiry": 0.0005004995715699767, "other": 0.0009906367398142323 } }, { "domain": "support_intent", "row": 70, "task": "intent", "multi_label": false, "gold": [ "invoice_copy" ], "selected": [ "tax_document" ], "correct": false, "logits": { "order_status": -5.348315238952637, "refund_request": -6.043322563171387, "cancel_subscription": -6.631126403808594, "update_payment": -2.5838325023651123, "login_problem": -6.010106086730957, "password_reset": -7.231306076049805, "shipping_delay": -6.949633598327637, "missing_item": -7.053769111633301, "damaged_item": -7.319991111755371, "change_address": -8.365715980529785, "change_order": -7.0954508781433105, "product_question": -3.811724901199341, "pricing_question": -3.594036817550659, "upgrade_plan": -1.420294165611267, "downgrade_plan": -5.885143280029297, "bug_report": -3.466033458709717, "feature_request": -5.976245880126953, "speak_to_human": -3.6071243286132812, "complaint": -4.924311637878418, "invoice_copy": -1.040328025817871, "tax_document": 1.1274348497390747, "appointment": -6.984220027923584, "outage": -4.913554668426514, "data_export": -6.447947025299072, "close_account": -7.374061584472656, "promo_code": -7.700891017913818, "partner_inquiry": -5.487749099731445, "other": -4.549826622009277 }, "probabilities": { "order_status": 0.0012134413148740897, "refund_request": 0.0006055931188704244, "cancel_subscription": 0.0003364348430293166, "update_payment": 0.019258318964526504, "login_problem": 0.0006260466042419256, "password_reset": 0.0001846061850550285, "shipping_delay": 0.0002446668047704523, "missing_item": 0.0002204700352506386, "damaged_item": 0.00016893935273737203, "change_address": 5.93715361544087e-05, "change_order": 0.0002114693402428749, "product_question": 0.005640940024030442, "pricing_question": 0.007012812311090389, "upgrade_plan": 0.06165050729077763, "downgrade_plan": 0.0007093773565701576, "bug_report": 0.00797045966098197, "feature_request": 0.0006476076422433923, "speak_to_human": 0.006921630028449075, "complaint": 0.0018542197467408268, "invoice_copy": 0.09014753427629273, "tax_document": 0.7877733908901141, "appointment": 0.0002363493187381653, "outage": 0.0018742731957136327, "data_export": 0.0004040682815557133, "close_account": 0.00016004728747033749, "promo_code": 0.00011542718344888035, "partner_inquiry": 0.0010555125982180944, "other": 0.0026964848078113697 } }, { "domain": "support_intent", "row": 80, "task": "intent", "multi_label": false, "gold": [ "update_payment" ], "selected": [ "update_payment" ], "correct": true, "logits": { "order_status": -5.121742248535156, "refund_request": -5.365330696105957, "cancel_subscription": -6.9118757247924805, "update_payment": 4.264257431030273, "login_problem": -5.125242233276367, "password_reset": -6.329968452453613, "shipping_delay": -6.044774055480957, "missing_item": -8.372689247131348, "damaged_item": -8.858010292053223, "change_address": -7.1621294021606445, "change_order": -6.743169784545898, "product_question": -5.380218505859375, "pricing_question": -6.25780725479126, "upgrade_plan": -6.691341876983643, "downgrade_plan": -7.536308288574219, "bug_report": -5.691773414611816, "feature_request": -6.275050163269043, "speak_to_human": -2.954301118850708, "complaint": -4.61686897277832, "invoice_copy": -6.428763389587402, "tax_document": -9.231139183044434, "appointment": -6.702818393707275, "outage": -6.73769474029541, "data_export": -7.867280960083008, "close_account": -7.184324264526367, "promo_code": -7.403904438018799, "partner_inquiry": -6.302883625030518, "other": -4.886442184448242 }, "probabilities": { "order_status": 8.375508250178412e-05, "refund_request": 6.564808351623443e-05, "cancel_subscription": 1.398189640921622e-05, "update_payment": 0.998387278887987, "login_problem": 8.346245338846763e-05, "password_reset": 2.5019878565601203e-05, "shipping_delay": 3.3276952774895185e-05, "missing_item": 3.2444630186762668e-06, "damaged_item": 1.9969655670254383e-06, "change_address": 1.0886349901419007e-05, "change_order": 1.6551377308893108e-05, "product_question": 6.467796671391132e-05, "pricing_question": 2.6892080799539334e-05, "upgrade_plan": 1.743181921996675e-05, "downgrade_plan": 7.488217726034954e-06, "bug_report": 4.7364153597292216e-05, "feature_request": 2.643235798103986e-05, "speak_to_human": 0.0007316761405642626, "complaint": 0.00013876337317276615, "invoice_copy": 2.266622008995225e-05, "tax_document": 1.3750635124754563e-06, "appointment": 1.7232906252402868e-05, "outage": 1.6642245358372648e-05, "data_export": 5.378223654469066e-06, "close_account": 1.0647390508549596e-05, "promo_code": 8.548319090073918e-06, "partner_inquiry": 2.5706798216313282e-05, "other": 0.000105974332603397 } }, { "domain": "support_intent", "row": 90, "task": "intent", "multi_label": false, "gold": [ "upgrade_plan" ], "selected": [ "upgrade_plan" ], "correct": true, "logits": { "order_status": -5.221330642700195, "refund_request": -3.0017921924591064, "cancel_subscription": -5.783247947692871, "update_payment": -3.3080594539642334, "login_problem": -7.023136615753174, "password_reset": -7.852404594421387, "shipping_delay": -7.1607255935668945, "missing_item": -7.326998710632324, "damaged_item": -7.770396709442139, "change_address": -8.235823631286621, "change_order": -5.46442985534668, "product_question": -4.769911766052246, "pricing_question": -3.3717823028564453, "upgrade_plan": 2.8535830974578857, "downgrade_plan": -4.366948127746582, "bug_report": -6.526366710662842, "feature_request": -7.337463855743408, "speak_to_human": -3.105828046798706, "complaint": -5.463658332824707, "invoice_copy": -9.307291984558105, "tax_document": -8.25460147857666, "appointment": -6.998409748077393, "outage": -5.813423156738281, "data_export": -8.194966316223145, "close_account": -6.899844169616699, "promo_code": -8.008478164672852, "partner_inquiry": -6.085308074951172, "other": -4.447988033294678 }, "probabilities": { "order_status": 0.00030719745169459334, "refund_request": 0.0028271634011427294, "cancel_subscription": 0.00017513785400300009, "update_payment": 0.002081328966165397, "login_problem": 5.068777376623336e-05, "password_reset": 2.2118552903822004e-05, "shipping_delay": 4.417220544102782e-05, "missing_item": 3.740568274913351e-05, "damaged_item": 2.4008901063416143e-05, "change_address": 1.5074396631799746e-05, "change_order": 0.0002409023087260325, "product_question": 0.0004824655805669832, "pricing_question": 0.0019528381321884415, "upgrade_plan": 0.9869793992712553, "downgrade_plan": 0.0007218903235603213, "bug_report": 8.330050720180552e-05, "feature_request": 3.701626804590128e-05, "speak_to_human": 0.0025478198250182826, "complaint": 0.00024108824199944084, "invoice_copy": 5.163059675427984e-06, "tax_document": 1.4793973029556219e-05, "appointment": 5.195674786274189e-05, "outage": 0.0001699319719454369, "data_export": 1.5703051121958946e-05, "close_account": 5.733877973764541e-05, "promo_code": 1.8922339042120255e-05, "partner_inquiry": 0.00012947829665844058, "other": 0.0006656961368029791 } }, { "domain": "support_topic", "row": 0, "task": "topic", "multi_label": false, "gold": [ "technical_outage" ], "selected": [ "account_access" ], "correct": false, "logits": { "billing": -0.5503880977630615, "account_access": 0.7084431648254395, "shipping": -3.840087890625, "returns": -3.3235087394714355, "technical_outage": 0.37296736240386963, "onboarding": -3.302107334136963, "integrations": -3.5104820728302, "security": -3.1000077724456787, "compliance": -4.5588884353637695, "pricing": -0.6100298166275024, "data_privacy": -5.064023017883301, "other": -2.690596103668213 }, "probabilities": { "billing": 0.11874573955369752, "account_access": 0.41813980043797044, "shipping": 0.004425064134232182, "returns": 0.00741766207368115, "technical_outage": 0.29896904063277036, "onboarding": 0.007578121369145975, "integrations": 0.006152697415225395, "security": 0.009275392170617472, "compliance": 0.002156495018754113, "pricing": 0.11187059997220015, "data_privacy": 0.0013012816256537166, "other": 0.013968105596051432 } }, { "domain": "support_topic", "row": 10, "task": "topic", "multi_label": false, "gold": [ "onboarding" ], "selected": [ "technical_outage" ], "correct": false, "logits": { "billing": -0.48283565044403076, "account_access": -2.203760862350464, "shipping": -4.7139811515808105, "returns": -5.125668525695801, "technical_outage": 0.3660912811756134, "onboarding": -4.24928617477417, "integrations": -3.886432647705078, "security": -3.7131521701812744, "compliance": -3.5192508697509766, "pricing": -1.1996161937713623, "data_privacy": -4.661879539489746, "other": -0.9602875709533691 }, "probabilities": { "billing": 0.20798061139073312, "account_access": 0.037207845849834636, "shipping": 0.003023150077400697, "returns": 0.002002931756856423, "technical_outage": 0.48607930641351954, "onboarding": 0.004811429996766375, "integrations": 0.006916071206166797, "security": 0.008224589126277515, "compliance": 0.009984457576755328, "pricing": 0.10156147879801147, "data_privacy": 0.003184836558972562, "other": 0.12902329124870562 } }, { "domain": "support_topic", "row": 20, "task": "topic", "multi_label": false, "gold": [ "technical_outage" ], "selected": [ "billing" ], "correct": false, "logits": { "billing": 0.7157478332519531, "account_access": -1.698106050491333, "shipping": -2.8195323944091797, "returns": -1.6956802606582642, "technical_outage": -1.5013623237609863, "onboarding": -2.2862303256988525, "integrations": -3.2178401947021484, "security": -2.9028544425964355, "compliance": -2.878180980682373, "pricing": -0.7644081115722656, "data_privacy": -3.211500644683838, "other": -2.121594190597534 }, "probabilities": { "billing": 0.5725132842214343, "account_access": 0.051222662204340914, "shipping": 0.016689098325486172, "returns": 0.05134706844823447, "technical_outage": 0.06236010837160376, "onboarding": 0.02844742879143339, "integrations": 0.011205983885136207, "security": 0.015354884980705402, "compliance": 0.015738455700266274, "pricing": 0.13030527857091495, "data_privacy": 0.011277250440744579, "other": 0.03353849605969955 } }, { "domain": "support_topic", "row": 30, "task": "topic", "multi_label": false, "gold": [ "billing" ], "selected": [ "billing" ], "correct": true, "logits": { "billing": 0.5692142844200134, "account_access": -1.7661385536193848, "shipping": -3.087441921234131, "returns": -3.607203960418701, "technical_outage": -2.5897841453552246, "onboarding": -2.7886664867401123, "integrations": -3.1051132678985596, "security": -2.473745107650757, "compliance": -1.7835564613342285, "pricing": -2.7127578258514404, "data_privacy": -2.8520185947418213, "other": -1.9501054286956787 }, "probabilities": { "billing": 0.6518345931078902, "account_access": 0.06308215977347964, "shipping": 0.01682952244428868, "returns": 0.01000787810516249, "technical_outage": 0.027682277539446975, "onboarding": 0.02268967708360254, "integrations": 0.01653473444072105, "security": 0.031088297000108685, "compliance": 0.06199291426310262, "pricing": 0.024479076393721856, "data_privacy": 0.021296824115234956, "other": 0.05248204573324026 } }, { "domain": "support_topic", "row": 40, "task": "topic", "multi_label": false, "gold": [ "pricing" ], "selected": [ "pricing" ], "correct": true, "logits": { "billing": -0.6344589591026306, "account_access": -5.3570709228515625, "shipping": -7.353207588195801, "returns": -7.962557792663574, "technical_outage": -6.058128356933594, "onboarding": -8.076258659362793, "integrations": 1.1450117826461792, "security": -7.978071212768555, "compliance": -5.920740127563477, "pricing": 1.732386589050293, "data_privacy": -3.3387842178344727, "other": -5.887244701385498 }, "probabilities": { "billing": 0.05654819805768602, "account_access": 0.0005028222121763723, "shipping": 6.831299331510891e-05, "returns": 3.714205118613879e-05, "technical_outage": 0.0002494302254637083, "onboarding": 3.315020551601711e-05, "integrations": 0.33514526986493653, "security": 3.6570297338120987e-05, "compliance": 0.0002861646796240181, "pricing": 0.6030130785452794, "data_privacy": 0.003783948642187884, "other": 0.0002959122252906102 } }, { "domain": "support_topic", "row": 50, "task": "topic", "multi_label": false, "gold": [ "account_access" ], "selected": [ "billing" ], "correct": false, "logits": { "billing": 0.5381897687911987, "account_access": -2.145878791809082, "shipping": 0.24366557598114014, "returns": -0.667994499206543, "technical_outage": -3.7045159339904785, "onboarding": -4.225235462188721, "integrations": -3.6512722969055176, "security": -4.187540531158447, "compliance": -4.7311296463012695, "pricing": -3.702528715133667, "data_privacy": -4.526429176330566, "other": -3.3859739303588867 }, "probabilities": { "billing": 0.4535042562796457, "account_access": 0.03096743270355509, "shipping": 0.3378089375391009, "returns": 0.13575073686671152, "technical_outage": 0.00651624931599538, "onboarding": 0.003871257632670863, "integrations": 0.006872600670208757, "security": 0.004019969658490017, "compliance": 0.0023342373937731875, "pricing": 0.006529211404499512, "data_privacy": 0.0028644767831054107, "other": 0.008960633752243777 } }, { "domain": "support_topic", "row": 60, "task": "topic", "multi_label": false, "gold": [ "billing" ], "selected": [ "billing" ], "correct": true, "logits": { "billing": 1.198785662651062, "account_access": -2.5442895889282227, "shipping": -4.306269645690918, "returns": -2.852661371231079, "technical_outage": -2.952920913696289, "onboarding": -3.4594063758850098, "integrations": -1.0067044496536255, "security": -3.245941162109375, "compliance": -3.642920970916748, "pricing": -2.086603879928589, "data_privacy": -3.8605480194091797, "other": -2.343938112258911 }, "probabilities": { "billing": 0.7856038232382477, "account_access": 0.01860401415271454, "shipping": 0.0031943937344046454, "returns": 0.013667292775027819, "technical_outage": 0.012363468644515336, "onboarding": 0.00745034682604791, "integrations": 0.08657079438238818, "security": 0.009223234586869635, "compliance": 0.006201219647900791, "pricing": 0.02940201122728676, "data_privacy": 0.004988418579585713, "other": 0.02273098220501086 } }, { "domain": "support_topic", "row": 70, "task": "topic", "multi_label": false, "gold": [ "technical_outage" ], "selected": [ "shipping" ], "correct": false, "logits": { "billing": -0.8739895820617676, "account_access": -3.3213744163513184, "shipping": 0.053467556834220886, "returns": -4.020265102386475, "technical_outage": -1.587437629699707, "onboarding": -3.919719934463501, "integrations": -4.252699375152588, "security": -3.8356428146362305, "compliance": -4.455219745635986, "pricing": -1.9570229053497314, "data_privacy": -4.085301876068115, "other": -1.6669950485229492 }, "probabilities": { "billing": 0.19454733754934594, "account_access": 0.016832148851383386, "shipping": 0.4918297662481721, "returns": 0.008367875204891236, "technical_outage": 0.09531883846559733, "onboarding": 0.009252975373117976, "integrations": 0.006632393310742148, "security": 0.010064579518209998, "compliance": 0.005416475626806601, "pricing": 0.06586730519631373, "data_privacy": 0.007840975245276926, "other": 0.08802932941014263 } }, { "domain": "support_topic", "row": 80, "task": "topic", "multi_label": false, "gold": [ "pricing" ], "selected": [ "billing" ], "correct": false, "logits": { "billing": 0.38617992401123047, "account_access": -2.5084147453308105, "shipping": -6.59140157699585, "returns": -0.24877044558525085, "technical_outage": -6.141608238220215, "onboarding": -5.590522766113281, "integrations": -7.131365776062012, "security": 0.3807837963104248, "compliance": -5.170551300048828, "pricing": -3.0588371753692627, "data_privacy": -5.745449066162109, "other": -3.9489564895629883 }, "probabilities": { "billing": 0.3793028407173665, "account_access": 0.020983580872464498, "shipping": 0.00035372108815141253, "returns": 0.2010160123740414, "technical_outage": 0.0005546304603580609, "onboarding": 0.0009623589662870549, "integrations": 0.00020613772573903978, "security": 0.3772615865388197, "compliance": 0.00146463155709509, "pricing": 0.012101359947645421, "data_privacy": 0.0008242395690932221, "other": 0.004968900182938651 } }, { "domain": "support_topic", "row": 90, "task": "topic", "multi_label": false, "gold": [ "returns" ], "selected": [ "returns" ], "correct": true, "logits": { "billing": 1.2673429250717163, "account_access": -3.091336250305176, "shipping": -2.4067485332489014, "returns": 1.8193740844726562, "technical_outage": -3.3185205459594727, "onboarding": -6.0709943771362305, "integrations": -1.9612377882003784, "security": -5.1895952224731445, "compliance": -4.67839241027832, "pricing": -4.486160755157471, "data_privacy": -6.409502983093262, "other": -2.9347758293151855 }, "probabilities": { "billing": 0.3511002856246131, "account_access": 0.004492425342616267, "shipping": 0.008908273381556411, "returns": 0.6097829389703535, "technical_outage": 0.003579447381029197, "onboarding": 0.000228261037064313, "integrations": 0.013908375093663917, "security": 0.0005510849812062516, "compliance": 0.0009188214721518869, "pricing": 0.0011135668217404214, "data_privacy": 0.0001627119187451599, "other": 0.005253807975259526 } }, { "domain": "ticket_route", "row": 0, "task": "queue", "multi_label": false, "gold": [ "compliance" ], "selected": [ "compliance" ], "correct": true, "logits": { "billing_disputes": -4.760284423828125, "refunds": -6.218425750732422, "shipping": -7.028085708618164, "returns": -6.806574821472168, "identity": -5.858701229095459, "payments": -3.821186065673828, "outages": -5.609563827514648, "mobile_app": -6.296731948852539, "api": -6.515371799468994, "account_closure": -5.801508903503418, "compliance": 3.1084518432617188, "fraud_review": -5.452180862426758, "onboarding": -5.609125137329102, "partnerships": -6.850734233856201, "product_feedback": -5.760807037353516, "general": -5.327487945556641 }, "probabilities": { "billing_disputes": 0.0003814214183684081, "refunds": 8.874468346288195e-05, "shipping": 3.949221500619535e-05, "returns": 4.928473761033718e-05, "identity": 0.00012716532907088254, "payments": 0.0009755517486361864, "outages": 0.00016314272727706997, "mobile_app": 8.206054461625549e-05, "api": 6.594476338337881e-05, "account_closure": 0.00013465020869271325, "compliance": 0.9971346708614365, "fraud_review": 0.00019094938830253178, "onboarding": 0.0001632143120909994, "partnerships": 4.715570689637623e-05, "product_feedback": 0.00014024378578551382, "general": 0.00021630756936370188 } }, { "domain": "ticket_route", "row": 0, "task": "urgency", "multi_label": false, "gold": [ "low" ], "selected": [ "normal" ], "correct": false, "logits": { "low": -1.0382864475250244, "normal": -0.8173197507858276, "high": -0.8651484251022339, "critical": -1.5184040069580078 }, "probabilities": { "low": 0.24660774097001123, "normal": 0.3075893695707969, "high": 0.2932240531919728, "critical": 0.15257883626721902 } }, { "domain": "ticket_route", "row": 0, "task": "contains_pii", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.5433745980262756, "no": 0.6605632305145264 }, "probabilities": { "yes": 0.23077543954694998, "no": 0.76922456045305 } }, { "domain": "ticket_route", "row": 10, "task": "queue", "multi_label": false, "gold": [ "payments" ], "selected": [ "payments" ], "correct": true, "logits": { "billing_disputes": -4.449762344360352, "refunds": -4.3577423095703125, "shipping": -5.387696266174316, "returns": -4.718380928039551, "identity": -4.857824325561523, "payments": 2.0137522220611572, "outages": -4.018718719482422, "mobile_app": -5.156729221343994, "api": -4.877002239227295, "account_closure": -4.286023139953613, "compliance": -5.152366638183594, "fraud_review": -5.161977767944336, "onboarding": -4.163640022277832, "partnerships": -5.776210784912109, "product_feedback": -4.682781219482422, "general": -4.049808502197266 }, "probabilities": { "billing_disputes": 0.0015291528604288057, "refunds": 0.0016765429926087743, "shipping": 0.0005985650773891811, "returns": 0.001168937612018262, "identity": 0.001016791330774865, "payments": 0.9806625969061309, "outages": 0.0023531562707475697, "mobile_app": 0.0007540828918212652, "api": 0.000997477188852859, "account_closure": 0.0018011999838447917, "compliance": 0.0007573798274957234, "fraud_review": 0.0007501354209273147, "onboarding": 0.002035692830401527, "partnerships": 0.0004058650578389324, "product_feedback": 0.0012113010391828498, "general": 0.002281122709536423 } }, { "domain": "ticket_route", "row": 10, "task": "urgency", "multi_label": false, "gold": [ "critical" ], "selected": [ "critical" ], "correct": true, "logits": { "low": -2.208606719970703, "normal": -2.2321090698242188, "high": -0.4837099015712738, "critical": -0.01836702600121498 }, "probabilities": { "low": 0.060510458782082, "normal": 0.059104902445583495, "high": 0.33958118209319615, "critical": 0.5408034566791383 } }, { "domain": "ticket_route", "row": 10, "task": "contains_pii", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.19242911040782928, "no": 0.32624101638793945 }, "probabilities": { "yes": 0.3731632550651498, "no": 0.6268367449348502 } }, { "domain": "ticket_route", "row": 20, "task": "queue", "multi_label": false, "gold": [ "account_closure" ], "selected": [ "account_closure" ], "correct": true, "logits": { "billing_disputes": -6.412771701812744, "refunds": -7.270195960998535, "shipping": -9.084826469421387, "returns": -7.951539993286133, "identity": -7.191817283630371, "payments": -5.175693511962891, "outages": -6.887063980102539, "mobile_app": -7.785207748413086, "api": -7.281895637512207, "account_closure": 4.888452529907227, "compliance": -6.786934852600098, "fraud_review": -7.130214691162109, "onboarding": -6.1977033615112305, "partnerships": -8.24056339263916, "product_feedback": -7.533954620361328, "general": -6.441407203674316 }, "probabilities": { "billing_disputes": 1.2356139887812974e-05, "refunds": 5.242134890731848e-06, "shipping": 8.539337441478959e-07, "returns": 2.6521875896044766e-06, "identity": 5.669537329205014e-06, "payments": 4.257346987098986e-05, "outages": 7.689538880153679e-06, "mobile_app": 3.132141794209933e-06, "api": 5.181160991238885e-06, "account_closure": 0.9998667790707612, "compliance": 8.499352173020627e-06, "fraud_review": 6.029777475108027e-06, "onboarding": 1.5320954043323962e-05, "partnerships": 1.986474392871809e-06, "product_feedback": 4.026792598823515e-06, "general": 1.2007333577466919e-05 } }, { "domain": "ticket_route", "row": 20, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "normal" ], "correct": true, "logits": { "low": -0.8974136114120483, "normal": -0.7906292080879211, "high": -1.1500240564346313, "critical": -1.597617268562317 }, "probabilities": { "low": 0.295338181255047, "normal": 0.3286211213149683, "high": 0.22940996240893283, "critical": 0.1466307350210519 } }, { "domain": "ticket_route", "row": 20, "task": "contains_pii", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.5853595733642578, "no": 0.7041714191436768 }, "probabilities": { "yes": 0.21593220578573025, "no": 0.7840677942142698 } }, { "domain": "ticket_route", "row": 30, "task": "queue", "multi_label": false, "gold": [ "returns" ], "selected": [ "refunds" ], "correct": false, "logits": { "billing_disputes": -3.7923977375030518, "refunds": 1.1411290168762207, "shipping": -4.951615333557129, "returns": -2.168872117996216, "identity": -4.7326436042785645, "payments": -2.5436110496520996, "outages": -4.478428363800049, "mobile_app": -4.8148274421691895, "api": -4.741133213043213, "account_closure": -4.6649088859558105, "compliance": -4.59922456741333, "fraud_review": -4.333104133605957, "onboarding": -4.620458126068115, "partnerships": -5.28761100769043, "product_feedback": -4.336694240570068, "general": -3.9887521266937256 }, "probabilities": { "billing_disputes": 0.006498155894861673, "refunds": 0.9023885412183226, "shipping": 0.0020386765189816273, "returns": 0.03295173936847064, "identity": 0.0025377365492195996, "payments": 0.022653290470576844, "outages": 0.0032722826578044113, "mobile_app": 0.0023375157578864264, "api": 0.0025162833522438645, "account_closure": 0.0027155846739240668, "compliance": 0.002899944498194399, "fraud_review": 0.0037841149629711593, "onboarding": 0.0028390174942842165, "partnerships": 0.0014568916480543522, "product_feedback": 0.0037705539428115695, "general": 0.005339670991392583 } }, { "domain": "ticket_route", "row": 30, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "high" ], "correct": false, "logits": { "low": -2.058431386947632, "normal": -1.7871023416519165, "high": -0.1743873655796051, "critical": -0.6354442834854126 }, "probabilities": { "low": 0.07667968046070602, "normal": 0.10058124373385881, "high": 0.5045569776958044, "critical": 0.3181820981096307 } }, { "domain": "ticket_route", "row": 30, "task": "contains_pii", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.14620256423950195, "no": 0.22855088114738464 }, "probabilities": { "yes": 0.4073929228700382, "no": 0.5926070771299619 } }, { "domain": "ticket_route", "row": 40, "task": "queue", "multi_label": false, "gold": [ "identity" ], "selected": [ "identity" ], "correct": true, "logits": { "billing_disputes": -4.954477310180664, "refunds": -8.575634956359863, "shipping": -13.023073196411133, "returns": -9.89857006072998, "identity": 4.533246994018555, "payments": -5.8636274337768555, "outages": -7.876767635345459, "mobile_app": -10.207670211791992, "api": -10.54984188079834, "account_closure": -7.653388023376465, "compliance": -7.19204568862915, "fraud_review": -6.321946144104004, "onboarding": -7.956093788146973, "partnerships": -7.916787147521973, "product_feedback": -7.120641231536865, "general": -6.727438926696777 }, "probabilities": { "billing_disputes": 7.576306399065659e-05, "refunds": 2.026789968669421e-06, "shipping": 2.3730716983035434e-08, "returns": 5.39840334980383e-07, "identity": 0.999824653052508, "payments": 3.0522397789030526e-05, "outages": 4.076833427133143e-06, "mobile_app": 3.9630070090350116e-07, "api": 2.8146316876165386e-07, "account_closure": 5.097244900875451e-06, "compliance": 8.085258885561219e-06, "fraud_review": 1.9300713372224002e-05, "onboarding": 3.7659283835673956e-06, "partnerships": 3.916902073942407e-06, "product_feedback": 8.683693617126915e-06, "general": 1.2866786161546505e-05 } }, { "domain": "ticket_route", "row": 40, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "high" ], "correct": false, "logits": { "low": -1.7087904214859009, "normal": -0.9198161363601685, "high": -0.5482565760612488, "critical": -1.2121522426605225 }, "probabilities": { "low": 0.12444060740334369, "normal": 0.2739108910196332, "high": 0.3971692048341071, "critical": 0.20447929674291604 } }, { "domain": "ticket_route", "row": 40, "task": "contains_pii", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 2.2876713275909424, "no": -2.216097593307495 }, "probabilities": { "yes": 0.9890539359399928, "no": 0.010946064060007183 } }, { "domain": "ticket_route", "row": 50, "task": "queue", "multi_label": false, "gold": [ "shipping" ], "selected": [ "shipping" ], "correct": true, "logits": { "billing_disputes": -4.526716232299805, "refunds": -5.226365566253662, "shipping": 0.7304133176803589, "returns": -4.330713748931885, "identity": -5.187150955200195, "payments": -4.339653968811035, "outages": -2.324406147003174, "mobile_app": -4.539621353149414, "api": -3.337012767791748, "account_closure": -5.133054733276367, "compliance": -4.876622200012207, "fraud_review": -4.24254035949707, "onboarding": -4.445404052734375, "partnerships": -5.275176048278809, "product_feedback": -3.2505738735198975, "general": -2.7063283920288086 }, "probabilities": { "billing_disputes": 0.0044727084315914585, "refunds": 0.0022218602696366585, "shipping": 0.8584459229721505, "returns": 0.005441183659190828, "identity": 0.002310720580131481, "payments": 0.005392755084492214, "outages": 0.04045961125405459, "mobile_app": 0.004415358438979745, "api": 0.014697796766916795, "account_closure": 0.002439164682038537, "compliance": 0.003152160751477927, "fraud_review": 0.005942738244309969, "onboarding": 0.004851589182665391, "partnerships": 0.0021160144063417986, "product_feedback": 0.016024783571258965, "general": 0.02761563170476304 } }, { "domain": "ticket_route", "row": 50, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "normal" ], "correct": true, "logits": { "low": -0.9921056032180786, "normal": -0.6878994107246399, "high": -0.8266997337341309, "critical": -1.6151533126831055 }, "probabilities": { "low": 0.24559598134715266, "normal": 0.3329172717855906, "high": 0.2897717960494972, "critical": 0.13171495081775947 } }, { "domain": "ticket_route", "row": 50, "task": "contains_pii", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.288636565208435, "no": 1.520585298538208 }, "probabilities": { "yes": 0.056827873338722815, "no": 0.9431721266612771 } }, { "domain": "ticket_route", "row": 60, "task": "queue", "multi_label": false, "gold": [ "identity" ], "selected": [ "identity" ], "correct": true, "logits": { "billing_disputes": -5.014692306518555, "refunds": -5.4179368019104, "shipping": -5.243519306182861, "returns": -5.386427879333496, "identity": 1.4553415775299072, "payments": -2.3949193954467773, "outages": -4.775130271911621, "mobile_app": -4.738880634307861, "api": -3.376063108444214, "account_closure": -5.5371270179748535, "compliance": -4.856471538543701, "fraud_review": -4.484904766082764, "onboarding": -4.568835735321045, "partnerships": -5.335317134857178, "product_feedback": -4.887321472167969, "general": -4.52079963684082 }, "probabilities": { "billing_disputes": 0.0014735073625635837, "refunds": 0.000984522058308388, "shipping": 0.001172125210292185, "returns": 0.0010160371841079815, "identity": 0.9511572526104237, "payments": 0.020235094145113548, "outages": 0.0018723747677004156, "mobile_app": 0.0019414928588928836, "api": 0.007585774506775294, "account_closure": 0.0008739001177364817, "compliance": 0.0017261030368984315, "fraud_review": 0.0025028574527505678, "onboarding": 0.002301364231052203, "partnerships": 0.001069317603724878, "product_feedback": 0.00167366587301266, "general": 0.00241461098064685 } }, { "domain": "ticket_route", "row": 60, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "normal" ], "correct": true, "logits": { "low": -0.919865071773529, "normal": -0.5900625586509705, "high": -0.979164183139801, "critical": -1.687505841255188 }, "probabilities": { "low": 0.26335020920069757, "normal": 0.36623941300203555, "high": 0.24818777798358063, "critical": 0.12222259981368616 } }, { "domain": "ticket_route", "row": 60, "task": "contains_pii", "multi_label": false, "gold": [ "yes" ], "selected": [ "yes" ], "correct": true, "logits": { "yes": 2.080767869949341, "no": -1.8551030158996582 }, "probabilities": { "yes": 0.9808453798718396, "no": 0.019154620128160292 } }, { "domain": "ticket_route", "row": 70, "task": "queue", "multi_label": false, "gold": [ "account_closure" ], "selected": [ "account_closure" ], "correct": true, "logits": { "billing_disputes": -7.5202317237854, "refunds": -8.022978782653809, "shipping": -9.598139762878418, "returns": -8.689126014709473, "identity": -7.375067710876465, "payments": -5.389012813568115, "outages": -7.297669887542725, "mobile_app": -8.872563362121582, "api": -8.175393104553223, "account_closure": 4.780829906463623, "compliance": -7.7211012840271, "fraud_review": -8.126166343688965, "onboarding": -6.879460334777832, "partnerships": -9.025206565856934, "product_feedback": -7.983757495880127, "general": -6.696450710296631 }, "probabilities": { "billing_disputes": 4.5465004567751575e-06, "refunds": 2.7500270494200323e-06, "shipping": 5.691847344455144e-07, "returns": 1.4126445509062618e-06, "identity": 5.256796546066256e-06, "payments": 3.830485754012997e-05, "outages": 5.67982057254779e-06, "mobile_app": 1.1758910120048807e-06, "api": 2.361262481718193e-06, "account_closure": 0.9999088830874, "compliance": 3.7191243337026784e-06, "fraud_review": 2.4804083075037874e-06, "onboarding": 8.629004936934054e-06, "partnerships": 1.0094271225948437e-06, "product_feedback": 2.8600297719677862e-06, "general": 1.0361933183422249e-05 } }, { "domain": "ticket_route", "row": 70, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "normal" ], "correct": true, "logits": { "low": -0.46223416924476624, "normal": -0.4017505645751953, "high": -1.656516432762146, "critical": -2.212411880493164 }, "probabilities": { "low": 0.393853593556527, "normal": 0.4184104359474984, "high": 0.11930664014385925, "critical": 0.06842933035211542 } }, { "domain": "ticket_route", "row": 70, "task": "contains_pii", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -1.1524704694747925, "no": 1.283806324005127 }, "probabilities": { "yes": 0.08044791037589194, "no": 0.9195520896241082 } }, { "domain": "ticket_route", "row": 80, "task": "queue", "multi_label": false, "gold": [ "payments" ], "selected": [ "payments" ], "correct": true, "logits": { "billing_disputes": -4.487617015838623, "refunds": -4.082760810852051, "shipping": -5.7031450271606445, "returns": -3.416539430618286, "identity": -5.784582138061523, "payments": 2.0142085552215576, "outages": -5.1146626472473145, "mobile_app": -6.252780914306641, "api": -5.748059272766113, "account_closure": -5.774703502655029, "compliance": -5.458523750305176, "fraud_review": -5.379446983337402, "onboarding": -5.120261192321777, "partnerships": -6.343985557556152, "product_feedback": -4.973154067993164, "general": -4.929891586303711 }, "probabilities": { "billing_disputes": 0.0014785387160563747, "refunds": 0.0022164580548823245, "shipping": 0.0004384656676065858, "returns": 0.004315148898936105, "identity": 0.00040417356529307767, "payments": 0.9852346311953334, "outages": 0.0007897874776037863, "mobile_app": 0.0002530648111612141, "api": 0.00041920802149088537, "account_closure": 0.00040818603481680466, "compliance": 0.000559980964126725, "fraud_review": 0.0006060603432841235, "onboarding": 0.0007853781711785542, "partnerships": 0.00023100537606730273, "product_feedback": 0.0009098433782025287, "general": 0.0009500693239603408 } }, { "domain": "ticket_route", "row": 80, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "high" ], "correct": false, "logits": { "low": -2.1052134037017822, "normal": -1.8843379020690918, "high": 0.18119299411773682, "critical": -0.9675232768058777 }, "probabilities": { "low": 0.06576248864313153, "normal": 0.08201688133331256, "high": 0.647070978292175, "critical": 0.20514965173138094 } }, { "domain": "ticket_route", "row": 80, "task": "contains_pii", "multi_label": false, "gold": [ "no" ], "selected": [ "no" ], "correct": true, "logits": { "yes": -0.44927388429641724, "no": 0.5323253870010376 }, "probabilities": { "yes": 0.27257456855903334, "no": 0.7274254314409666 } }, { "domain": "ticket_route", "row": 90, "task": "queue", "multi_label": false, "gold": [ "shipping" ], "selected": [ "shipping" ], "correct": true, "logits": { "billing_disputes": -4.558888912200928, "refunds": -4.755166530609131, "shipping": 1.3981778621673584, "returns": -4.385894775390625, "identity": -4.455512046813965, "payments": -3.9933855533599854, "outages": -3.67502760887146, "mobile_app": -4.297508239746094, "api": -4.317506313323975, "account_closure": -4.725268840789795, "compliance": -4.494952201843262, "fraud_review": -4.690605163574219, "onboarding": -4.160389423370361, "partnerships": -4.942963600158691, "product_feedback": -3.7586915493011475, "general": -3.3346779346466064 }, "probabilities": { "billing_disputes": 0.0024513760130955003, "refunds": 0.0020145017341583727, "shipping": 0.9473951576367132, "returns": 0.0029143407654024273, "identity": 0.002718353511263024, "payments": 0.004315239658306427, "outages": 0.005932886211776666, "mobile_app": 0.0031836558580270054, "api": 0.0031206212599313807, "account_closure": 0.002075640075645964, "compliance": 0.002613227938571972, "fraud_review": 0.0021488509416361682, "onboarding": 0.0036515399901607248, "partnerships": 0.0016695845398370219, "product_feedback": 0.005456714494996437, "general": 0.00833830937047757 } }, { "domain": "ticket_route", "row": 90, "task": "urgency", "multi_label": false, "gold": [ "normal" ], "selected": [ "normal" ], "correct": true, "logits": { "low": -0.4575599431991577, "normal": -0.39656123518943787, "high": -1.7999773025512695, "critical": -2.0628294944763184 }, "probabilities": { "low": 0.396048009423418, "normal": 0.42095845530756715, "high": 0.10345307005672581, "critical": 0.07954046521228904 } }, { "domain": "ticket_route", "row": 90, "task": "contains_pii", "multi_label": false, "gold": [ "yes" ], "selected": [ "no" ], "correct": false, "logits": { "yes": -0.778555154800415, "no": 0.9079996943473816 }, "probabilities": { "yes": 0.15622944846175937, "no": 0.8437705515382405 } }, { "domain": "travel_request", "row": 0, "task": "intent", "multi_label": false, "gold": [ "hotel" ], "selected": [ "hotel" ], "correct": true, "logits": { "new_booking": -3.6751222610473633, "change_flight": -5.968661785125732, "cancel_trip": -6.70306396484375, "seat": -2.805928945541382, "baggage": -5.555136203765869, "refund": -6.03434944152832, "hotel": 0.5718575716018677, "car": -5.229447841644287, "visa_docs": -0.8174301385879517, "loyalty": -4.9076642990112305, "disruption": -5.3460612297058105, "other": -3.8126978874206543 }, "probabilities": { "new_booking": 0.010792249494513184, "change_flight": 0.001089031438879353, "cancel_trip": 0.0005225087929410001, "seat": 0.025739365528269754, "baggage": 0.0016467675311462458, "refund": 0.0010197944186365719, "hotel": 0.7543135042756957, "car": 0.0022807461689509497, "visa_docs": 0.188014739164167, "loyalty": 0.0031464857626172156, "disruption": 0.002029702576841656, "other": 0.009405104847341465 } }, { "domain": "travel_request", "row": 10, "task": "intent", "multi_label": false, "gold": [ "cancel_trip" ], "selected": [ "cancel_trip" ], "correct": true, "logits": { "new_booking": -5.360355377197266, "change_flight": -7.601502895355225, "cancel_trip": 0.5519402027130127, "seat": -9.456254959106445, "baggage": -8.722764015197754, "refund": -0.32812973856925964, "hotel": -3.7120449542999268, "car": -9.077423095703125, "visa_docs": -11.47689437866211, "loyalty": -9.931441307067871, "disruption": -6.60952091217041, "other": -5.0550642013549805 }, "probabilities": { "new_booking": 0.0018837225964296587, "change_flight": 0.00020030830093555315, "cancel_trip": 0.6961363201310379, "seat": 3.134659411714978e-05, "baggage": 6.52741807973192e-05, "refund": 0.28872525527046816, "hotel": 0.009791947325352993, "car": 4.578412814375093e-05, "visa_docs": 4.155638974748673e-06, "loyalty": 1.9490346127833716e-05, "disruption": 0.0005401461048567468, "other": 0.00255624938275819 } }, { "domain": "travel_request", "row": 20, "task": "intent", "multi_label": false, "gold": [ "new_booking" ], "selected": [ "visa_docs" ], "correct": false, "logits": { "new_booking": -2.7952322959899902, "change_flight": -4.440430641174316, "cancel_trip": -5.4160943031311035, "seat": -6.694589614868164, "baggage": -1.1326167583465576, "refund": -6.916153907775879, "hotel": -6.176924228668213, "car": -5.544305801391602, "visa_docs": -0.14313048124313354, "loyalty": -7.031331539154053, "disruption": -5.34149169921875, "other": -2.3123254776000977 }, "probabilities": { "new_booking": 0.0443052337804025, "change_flight": 0.008549770743089664, "cancel_trip": 0.0032227686102025655, "seat": 0.0008973991749863312, "baggage": 0.23362525370929127, "refund": 0.0007190540158793183, "hotel": 0.001505930352188591, "car": 0.0028349641984808708, "visa_docs": 0.6284174210385882, "loyalty": 0.0006408265643786995, "disruption": 0.0034723910180778166, "other": 0.07180898679443422 } }, { "domain": "travel_request", "row": 30, "task": "intent", "multi_label": false, "gold": [ "car" ], "selected": [ "car" ], "correct": true, "logits": { "new_booking": -2.9466376304626465, "change_flight": -3.722877264022827, "cancel_trip": -6.536761283874512, "seat": -6.381529808044434, "baggage": -6.026378154754639, "refund": -4.036191463470459, "hotel": -5.135722637176514, "car": 2.934067964553833, "visa_docs": -6.998324394226074, "loyalty": -6.748903274536133, "disruption": -3.5805277824401855, "other": -4.178962707519531 }, "probabilities": { "new_booking": 0.002770559210017012, "change_flight": 0.0012748258067711152, "cancel_trip": 7.645335423261027e-05, "seat": 8.929203427865527e-05, "baggage": 0.00012736588586070318, "refund": 0.0009319234039180265, "hotel": 0.00031035582225326846, "car": 0.9920314155782329, "visa_docs": 4.81883695316113e-05, "loyalty": 6.183928336055835e-05, "disruption": 0.001469848050908573, "other": 0.0008079332006348095 } }, { "domain": "travel_request", "row": 40, "task": "intent", "multi_label": false, "gold": [ "visa_docs" ], "selected": [ "visa_docs" ], "correct": true, "logits": { "new_booking": -4.486355304718018, "change_flight": -5.619746685028076, "cancel_trip": -7.353476524353027, "seat": -6.750648498535156, "baggage": -6.342803955078125, "refund": -7.5083489418029785, "hotel": -5.05666446685791, "car": -5.228070259094238, "visa_docs": 3.806062698364258, "loyalty": -5.524506568908691, "disruption": -5.751307964324951, "other": -4.385922431945801 }, "probabilities": { "new_booking": 0.00025012818649155333, "change_flight": 8.052616414963066e-05, "cancel_trip": 1.4222885365870582e-05, "seat": 2.59891810666672e-05, "baggage": 3.907664140921459e-05, "refund": 1.2182249038195879e-05, "hotel": 0.0001414101269805222, "car": 0.00011913515179671129, "visa_docs": 0.9988816034093253, "loyalty": 8.857257462622787e-05, "disruption": 7.059934542193477e-05, "other": 0.00027655408432816466 } }, { "domain": "travel_request", "row": 50, "task": "intent", "multi_label": false, "gold": [ "baggage" ], "selected": [ "baggage" ], "correct": true, "logits": { "new_booking": -3.6527674198150635, "change_flight": -5.860159873962402, "cancel_trip": -6.1384100914001465, "seat": -5.87619161605835, "baggage": 2.2144172191619873, "refund": -5.8755574226379395, "hotel": -4.909104347229004, "car": -4.43424654006958, "visa_docs": -5.962222099304199, "loyalty": -5.532224178314209, "disruption": -5.4171295166015625, "other": -2.938622236251831 }, "probabilities": { "new_booking": 0.0027943030103950886, "change_flight": 0.00030733720431100375, "cancel_trip": 0.00023268725826352803, "seat": 0.00030244933870357375, "baggage": 0.9870960445991229, "refund": 0.0003026412109198046, "hotel": 0.0007955240317312713, "car": 0.0012790320553330169, "visa_docs": 0.00027751730867346127, "loyalty": 0.00042661468373602924, "disruption": 0.0004786529897288768, "other": 0.00570719630908154 } }, { "domain": "travel_request", "row": 60, "task": "intent", "multi_label": false, "gold": [ "disruption" ], "selected": [ "hotel" ], "correct": false, "logits": { "new_booking": -3.8579564094543457, "change_flight": -4.972869396209717, "cancel_trip": -5.470054626464844, "seat": -5.978825569152832, "baggage": -5.533156394958496, "refund": -5.977100372314453, "hotel": 0.5627816915512085, "car": -4.953600883483887, "visa_docs": -5.4791693687438965, "loyalty": -0.7503677606582642, "disruption": -2.756760597229004, "other": -3.16448974609375 }, "probabilities": { "new_booking": 0.00884800610950456, "change_flight": 0.0029016488620013927, "cancel_trip": 0.0017648998000516704, "seat": 0.001061117900567788, "baggage": 0.0016569725127963421, "refund": 0.001062950117828271, "hotel": 0.7357793196089851, "car": 0.0029581014521389425, "visa_docs": 0.0017488862836282308, "loyalty": 0.1979037465719609, "disruption": 0.02661268408677802, "other": 0.017701666693758812 } }, { "domain": "travel_request", "row": 70, "task": "intent", "multi_label": false, "gold": [ "seat" ], "selected": [ "seat" ], "correct": true, "logits": { "new_booking": -4.386347770690918, "change_flight": -5.605575084686279, "cancel_trip": -8.506549835205078, "seat": 2.667126178741455, "baggage": -7.324128150939941, "refund": -8.083160400390625, "hotel": -6.269394874572754, "car": -3.2121098041534424, "visa_docs": -9.959331512451172, "loyalty": -8.922587394714355, "disruption": -6.94024658203125, "other": -3.3312220573425293 }, "probabilities": { "new_booking": 0.0008586546330052856, "change_flight": 0.00025369670280941064, "cancel_trip": 1.3945609388996407e-05, "seat": 0.993352369477233, "baggage": 4.549427090493109e-05, "refund": 2.1296743088931314e-05, "hotel": 0.0001306235706437347, "car": 0.0027783284549216416, "visa_docs": 3.2621387623694142e-06, "loyalty": 9.199297846201104e-06, "disruption": 6.678429664621184e-05, "other": 0.0024663448047490947 } }, { "domain": "travel_request", "row": 80, "task": "intent", "multi_label": false, "gold": [ "loyalty" ], "selected": [ "loyalty" ], "correct": true, "logits": { "new_booking": -3.214038610458374, "change_flight": -4.212662696838379, "cancel_trip": -4.366559028625488, "seat": -6.975316524505615, "baggage": -5.9364423751831055, "refund": -5.617195129394531, "hotel": -5.635258197784424, "car": -6.154680252075195, "visa_docs": -7.801497459411621, "loyalty": 1.669306993484497, "disruption": -4.758963584899902, "other": -2.090637445449829 }, "probabilities": { "new_booking": 0.0072792992131041675, "change_flight": 0.0026815916279324895, "cancel_trip": 0.0022990918047899075, "seat": 0.00016927285854972476, "baggage": 0.00047837077412587044, "refund": 0.0006582819636243064, "hotel": 0.0006464981180949317, "car": 0.00038457861677658337, "visa_docs": 7.409373832298058e-05, "loyalty": 0.9613900439103343, "disruption": 0.0015528774374653923, "other": 0.022385999936879213 } }, { "domain": "travel_request", "row": 90, "task": "intent", "multi_label": false, "gold": [ "loyalty" ], "selected": [ "loyalty" ], "correct": true, "logits": { "new_booking": -4.253607273101807, "change_flight": -4.469066619873047, "cancel_trip": -5.196622371673584, "seat": -6.056311130523682, "baggage": -0.6719348430633545, "refund": -6.139847755432129, "hotel": -5.879680156707764, "car": -5.279245376586914, "visa_docs": -8.703460693359375, "loyalty": 1.4890140295028687, "disruption": -3.6982522010803223, "other": -3.210610866546631 }, "probabilities": { "new_booking": 0.002813146902848401, "change_flight": 0.0022678775729159427, "cancel_trip": 0.0010955852029322485, "seat": 0.0004637544328324108, "baggage": 0.10108645906698567, "refund": 0.0004265879451362614, "hotel": 0.0005533474732156403, "car": 0.0010087033029130736, "visa_docs": 3.285834051659156e-05, "loyalty": 0.8773667163364355, "disruption": 0.004902076167375254, "other": 0.007982887255893143 } } ] } }