{ "short_cn": { "question": { "type": "choice", "instructions": "根据对话状态判断如何处理用户刚说的话。注意:对问题的简短回答也是有效输入;附和不需要回复。", "criteria": { "interrupt": "助手还在说话,用户要求停下、暂停、取消,纠正助手,或必须立即改变当前话题。", "discard": "噪声、助手回声、对别人说话、单纯附和或感谢。助手正说话时要求继续,或空闲无任务时要求停止,也忽略。", "enqueue": "其他有意义的问题、请求、补充或回答;保留并稍后处理。助手空闲时也接纳。" } }, "summary": { "cases": 48, "correct": 34, "confusion": { "interrupt": { "interrupt": 11, "discard": 0, "enqueue": 0 }, "discard": { "interrupt": 4, "discard": 10, "enqueue": 0 }, "enqueue": { "interrupt": 7, "discard": 3, "enqueue": 13 } }, "false_discard": 3, "false_interrupt": 11, "missed_interrupt": 0, "median_seconds": 0.0789364164811559 }, "records": [ { "id": "duplex_stop_01", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.9308630228042603, 0.03596342355012894, 0.03317355364561081 ], "seconds": 0.2473604999249801, "tokens": 218 }, { "id": "duplex_stop_02", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.6084311604499817, 0.1538606435060501, 0.2377082109451294 ], "seconds": 0.07908791594672948, "tokens": 218 }, { "id": "duplex_stop_03", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.9178251624107361, 0.02585407719016075, 0.05632082000374794 ], "seconds": 0.07947220804635435, "tokens": 219 }, { "id": "duplex_stop_04", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.9476383924484253, 0.029724864289164543, 0.022636760026216507 ], "seconds": 0.07833625003695488, "tokens": 220 }, { "id": "duplex_correct_01", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.7275057435035706, 0.07282276451587677, 0.19967152178287506 ], "seconds": 0.0755601660348475, "tokens": 222 }, { "id": "duplex_correct_02", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.8345935344696045, 0.057409752160310745, 0.10799670219421387 ], "seconds": 0.0829312918940559, "tokens": 222 }, { "id": "duplex_correct_03", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.6642118692398071, 0.06811445951461792, 0.26767367124557495 ], "seconds": 0.07992016698699445, "tokens": 221 }, { "id": "duplex_switch_01", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.7909741997718811, 0.09334924072027206, 0.11567656695842743 ], "seconds": 0.08040650002658367, "tokens": 223 }, { "id": "duplex_urgent_01", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.8331965804100037, 0.11239644140005112, 0.05440700426697731 ], "seconds": 0.0784068750217557, "tokens": 223 }, { "id": "duplex_ack_01", "expected": "discard", "choice": "discard", "probabilities": [ 0.05335284024477005, 0.7110395431518555, 0.2356075942516327 ], "seconds": 0.07608508295379579, "tokens": 216 }, { "id": "duplex_ack_02", "expected": "discard", "choice": "discard", "probabilities": [ 0.0421481616795063, 0.6526758670806885, 0.3051760196685791 ], "seconds": 0.07729579193983227, "tokens": 218 }, { "id": "duplex_ack_03", "expected": "discard", "choice": "discard", "probabilities": [ 0.040575310587882996, 0.6564949750900269, 0.30292972922325134 ], "seconds": 0.07996416697278619, "tokens": 217 }, { "id": "duplex_ack_04", "expected": "discard", "choice": "discard", "probabilities": [ 0.011969413608312607, 0.9218174815177917, 0.06621308624744415 ], "seconds": 0.07615158404223621, "tokens": 216 }, { "id": "duplex_continue_01", "expected": "discard", "choice": "discard", "probabilities": [ 0.04026160016655922, 0.8019729852676392, 0.15776535868644714 ], "seconds": 0.07585625001229346, "tokens": 218 }, { "id": "duplex_negation_01", "expected": "discard", "choice": "interrupt", "probabilities": [ 0.6325174570083618, 0.2781596779823303, 0.08932279795408249 ], "seconds": 0.07802729099057615, "tokens": 220 }, { "id": "duplex_negation_02", "expected": "discard", "choice": "interrupt", "probabilities": [ 0.8180356621742249, 0.07483810931444168, 0.10712619870901108 ], "seconds": 0.08067808300256729, "tokens": 218 }, { "id": "duplex_noise_01", "expected": "discard", "choice": "discard", "probabilities": [ 0.16634003818035126, 0.7751500010490417, 0.05850996449589729 ], "seconds": 0.07809220894705504, "tokens": 218 }, { "id": "duplex_noise_02", "expected": "discard", "choice": "discard", "probabilities": [ 0.140708327293396, 0.5931727290153503, 0.2661188840866089 ], "seconds": 0.07563637499697506, "tokens": 214 }, { "id": "duplex_echo_01", "expected": "discard", "choice": "discard", "probabilities": [ 0.06657762825489044, 0.8454098701477051, 0.08801253885030746 ], "seconds": 0.07592029101215303, "tokens": 220 }, { "id": "duplex_other_01", "expected": "discard", "choice": "interrupt", "probabilities": [ 0.6339613795280457, 0.18493230640888214, 0.1811063289642334 ], "seconds": 0.07726554095279425, "tokens": 221 }, { "id": "duplex_queue_01", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.4120011627674103, 0.08334042876958847, 0.5046584010124207 ], "seconds": 0.07846620806958526, "tokens": 221 }, { "id": "duplex_queue_02", "expected": "enqueue", "choice": "interrupt", "probabilities": [ 0.4853450059890747, 0.04944157600402832, 0.46521347761154175 ], "seconds": 0.07490887504536659, "tokens": 222 }, { "id": "duplex_queue_03", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.20899155735969543, 0.124051034450531, 0.6669573783874512 ], "seconds": 0.07843541703186929, "tokens": 221 }, { "id": "duplex_queue_04", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.09898572415113449, 0.14187583327293396, 0.7591385245323181 ], "seconds": 0.0787995409918949, "tokens": 220 }, { "id": "duplex_queue_05", "expected": "enqueue", "choice": "interrupt", "probabilities": [ 0.5718603730201721, 0.149464413523674, 0.2786753177642822 ], "seconds": 0.07880450005177408, "tokens": 222 }, { "id": "duplex_queue_06", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.08183945715427399, 0.1113441213965416, 0.8068164587020874 ], "seconds": 0.07932108291424811, "tokens": 222 }, { "id": "duplex_queue_07", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.20892539620399475, 0.1285468339920044, 0.6625277400016785 ], "seconds": 0.0786395410541445, "tokens": 218 }, { "id": "duplex_idle_01", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.3379806578159332, 0.19012808799743652, 0.4718911945819855 ], "seconds": 0.08041737508028746, "tokens": 220 }, { "id": "duplex_idle_02", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.22359061241149902, 0.26534196734428406, 0.5110673904418945 ], "seconds": 0.08483033289667219, "tokens": 214 }, { "id": "duplex_idle_03", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.2232709527015686, 0.14533931016921997, 0.6313897371292114 ], "seconds": 0.08540104201529175, "tokens": 216 }, { "id": "duplex_idle_04", "expected": "enqueue", "choice": "discard", "probabilities": [ 0.025776008144021034, 0.7816474437713623, 0.19257661700248718 ], "seconds": 0.08535249996930361, "tokens": 217 }, { "id": "duplex_idle_05", "expected": "enqueue", "choice": "interrupt", "probabilities": [ 0.4170467257499695, 0.3020874261856079, 0.2808658480644226 ], "seconds": 0.0853517500218004, "tokens": 217 }, { "id": "duplex_idle_06", "expected": "enqueue", "choice": "discard", "probabilities": [ 0.32025524973869324, 0.40157341957092285, 0.2781713306903839 ], "seconds": 0.08389929099939764, "tokens": 218 }, { "id": "duplex_idle_07", "expected": "discard", "choice": "interrupt", "probabilities": [ 0.595440149307251, 0.18141186237335205, 0.22314797341823578 ], "seconds": 0.07983308401890099, "tokens": 221 }, { "id": "duplex_idle_08", "expected": "discard", "choice": "discard", "probabilities": [ 0.023530151695013046, 0.7448784708976746, 0.2315913885831833 ], "seconds": 0.07574620901141316, "tokens": 217 }, { "id": "duplex_answer_01", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.36920368671417236, 0.21816276013851166, 0.4126335680484772 ], "seconds": 0.07867716706823558, "tokens": 218 }, { "id": "duplex_answer_02", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.3135741651058197, 0.19335414469242096, 0.49307164549827576 ], "seconds": 0.07869200001005083, "tokens": 222 }, { "id": "duplex_answer_03", "expected": "enqueue", "choice": "discard", "probabilities": [ 0.35142919421195984, 0.3655969202518463, 0.28297391533851624 ], "seconds": 0.0753501660656184, "tokens": 219 }, { "id": "duplex_question_stop", "expected": "enqueue", "choice": "interrupt", "probabilities": [ 0.8747069239616394, 0.0360749326646328, 0.0892181396484375 ], "seconds": 0.07888579193968326, "tokens": 222 }, { "id": "duplex_quoted_stop", "expected": "enqueue", "choice": "interrupt", "probabilities": [ 0.5751999020576477, 0.07048548012971878, 0.3543146252632141 ], "seconds": 0.07968554203398526, "tokens": 224 }, { "id": "duplex_mid_ack_request", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.28059422969818115, 0.11433308571577072, 0.6050726771354675 ], "seconds": 0.08052141696680337, "tokens": 222 }, { "id": "duplex_immediate_01", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.6138381958007812, 0.24859386682510376, 0.1375679224729538 ], "seconds": 0.07979370898101479, "tokens": 221 }, { "id": "duplex_wait_01", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.6828376650810242, 0.1435249298810959, 0.17363739013671875 ], "seconds": 0.07526308309752494, "tokens": 221 }, { "id": "duplex_wait_02", "expected": "enqueue", "choice": "interrupt", "probabilities": [ 0.8187282681465149, 0.07863655686378479, 0.10263523459434509 ], "seconds": 0.08320104097947478, "tokens": 222 }, { "id": "duplex_blank_01", "expected": "discard", "choice": "discard", "probabilities": [ 0.23855049908161163, 0.5664229989051819, 0.19502650201320648 ], "seconds": 0.08054837491363287, "tokens": 213 }, { "id": "duplex_idle_correction", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.3666907846927643, 0.09975643455982208, 0.5335528254508972 ], "seconds": 0.07898704102262855, "tokens": 222 }, { "id": "duplex_queue_constraint", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.23049047589302063, 0.0817626342177391, 0.6877469420433044 ], "seconds": 0.08358375006355345, "tokens": 222 }, { "id": "duplex_late_request", "expected": "enqueue", "choice": "interrupt", "probabilities": [ 0.6512891054153442, 0.1271398365497589, 0.22157108783721924 ], "seconds": 0.08454970794264227, "tokens": 224 } ] }, "short_en": { "question": { "type": "choice", "instructions": "How should a voice assistant handle the latest user utterance? Use the playback state and preceding question. Short answers to a question are meaningful.", "criteria": { "interrupt": "While the assistant is speaking: stop/pause/cancel it, correct what it is saying, urgently switch tasks, or warn of danger.", "discard": "Noise, confirmed echo, speech to another person, or acknowledgment needing no reply. Also continue when already speaking, or stop when idle with no task.", "enqueue": "A meaningful question, request, information, or answer that can be processed next. When idle, accept meaningful input this way." } }, "summary": { "cases": 48, "correct": 35, "confusion": { "interrupt": { "interrupt": 5, "discard": 2, "enqueue": 4 }, "discard": { "interrupt": 1, "discard": 9, "enqueue": 4 }, "enqueue": { "interrupt": 0, "discard": 2, "enqueue": 21 } }, "false_discard": 4, "false_interrupt": 1, "missed_interrupt": 6, "median_seconds": 0.08071058348286897 }, "records": [ { "id": "duplex_stop_01", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.6268212795257568, 0.19353297352790833, 0.17964576184749603 ], "seconds": 0.08400687505491078, "tokens": 228 }, { "id": "duplex_stop_02", "expected": "interrupt", "choice": "enqueue", "probabilities": [ 0.10337332636117935, 0.363031804561615, 0.5335948467254639 ], "seconds": 0.08210300002247095, "tokens": 228 }, { "id": "duplex_stop_03", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.5057383179664612, 0.3377652168273926, 0.15649652481079102 ], "seconds": 0.08048545802012086, "tokens": 229 }, { "id": "duplex_stop_04", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.6581788659095764, 0.23939505219459534, 0.10242605209350586 ], "seconds": 0.08602541696745902, "tokens": 230 }, { "id": "duplex_correct_01", "expected": "interrupt", "choice": "enqueue", "probabilities": [ 0.18660441040992737, 0.3201097846031189, 0.4932858347892761 ], "seconds": 0.08271174994297326, "tokens": 232 }, { "id": "duplex_correct_02", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.35816383361816406, 0.3007599413394928, 0.3410762548446655 ], "seconds": 0.08182208298239857, "tokens": 232 }, { "id": "duplex_correct_03", "expected": "interrupt", "choice": "enqueue", "probabilities": [ 0.16310663521289825, 0.22092485427856445, 0.6159685254096985 ], "seconds": 0.08308241597842425, "tokens": 231 }, { "id": "duplex_switch_01", "expected": "interrupt", "choice": "enqueue", "probabilities": [ 0.12074483931064606, 0.3092057704925537, 0.5700494050979614 ], "seconds": 0.08102087501902133, "tokens": 233 }, { "id": "duplex_urgent_01", "expected": "interrupt", "choice": "interrupt", "probabilities": [ 0.7929849624633789, 0.12692339718341827, 0.08009161800146103 ], "seconds": 0.08281858405098319, "tokens": 233 }, { "id": "duplex_ack_01", "expected": "discard", "choice": "enqueue", "probabilities": [ 0.04723662510514259, 0.4310700595378876, 0.5216932892799377 ], "seconds": 0.08203183300793171, "tokens": 226 }, { "id": "duplex_ack_02", "expected": "discard", "choice": "discard", "probabilities": [ 0.020713020116090775, 0.894315779209137, 0.08497121185064316 ], "seconds": 0.09132337488699704, "tokens": 228 }, { "id": "duplex_ack_03", "expected": "discard", "choice": "discard", "probabilities": [ 0.05350596830248833, 0.5620473027229309, 0.3844466507434845 ], "seconds": 0.08070604200474918, "tokens": 227 }, { "id": "duplex_ack_04", "expected": "discard", "choice": "discard", "probabilities": [ 0.06928054243326187, 0.686016321182251, 0.24470311403274536 ], "seconds": 0.07941279199440032, "tokens": 226 }, { "id": "duplex_continue_01", "expected": "discard", "choice": "enqueue", "probabilities": [ 0.03238857537508011, 0.14694948494434357, 0.8206619620323181 ], "seconds": 0.08031204203143716, "tokens": 228 }, { "id": "duplex_negation_01", "expected": "discard", "choice": "enqueue", "probabilities": [ 0.23833583295345306, 0.3484111726284027, 0.41325291991233826 ], "seconds": 0.0811872499762103, "tokens": 230 }, { "id": "duplex_negation_02", "expected": "discard", "choice": "interrupt", "probabilities": [ 0.6158183217048645, 0.25162509083747864, 0.13255663216114044 ], "seconds": 0.07968854194041342, "tokens": 228 }, { "id": "duplex_noise_01", "expected": "discard", "choice": "discard", "probabilities": [ 0.14316071569919586, 0.7374182343482971, 0.11942102760076523 ], "seconds": 0.08046829199884087, "tokens": 228 }, { "id": "duplex_noise_02", "expected": "discard", "choice": "discard", "probabilities": [ 0.06295707821846008, 0.7446295619010925, 0.1924133449792862 ], "seconds": 0.07624604203738272, "tokens": 224 }, { "id": "duplex_echo_01", "expected": "discard", "choice": "discard", "probabilities": [ 0.03901330381631851, 0.7296719551086426, 0.2313147783279419 ], "seconds": 0.08117908309213817, "tokens": 230 }, { "id": "duplex_other_01", "expected": "discard", "choice": "discard", "probabilities": [ 0.15541087090969086, 0.546040415763855, 0.29854869842529297 ], "seconds": 0.07968662504572421, "tokens": 231 }, { "id": "duplex_queue_01", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.06113162636756897, 0.19676589965820312, 0.7421025037765503 ], "seconds": 0.07658904197160155, "tokens": 231 }, { "id": "duplex_queue_02", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.05522550642490387, 0.13346858322620392, 0.8113059401512146 ], "seconds": 0.08071512496098876, "tokens": 232 }, { "id": "duplex_queue_03", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.07100681215524673, 0.20129087567329407, 0.7277023196220398 ], "seconds": 0.08109366591088474, "tokens": 231 }, { "id": "duplex_queue_04", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.09818831831216812, 0.28104570508003235, 0.6207659840583801 ], "seconds": 0.08061987499240786, "tokens": 230 }, { "id": "duplex_queue_05", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.1275750994682312, 0.37047719955444336, 0.5019477009773254 ], "seconds": 0.08042445802129805, "tokens": 232 }, { "id": "duplex_queue_06", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.042777255177497864, 0.38734444975852966, 0.5698782801628113 ], "seconds": 0.07603337499313056, "tokens": 232 }, { "id": "duplex_queue_07", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.036512259393930435, 0.0908428356051445, 0.8726449012756348 ], "seconds": 0.08036591706331819, "tokens": 228 }, { "id": "duplex_idle_01", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.0399501696228981, 0.30694684386253357, 0.6531029939651489 ], "seconds": 0.07956312492024153, "tokens": 230 }, { "id": "duplex_idle_02", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.04318610951304436, 0.158489391207695, 0.7983245253562927 ], "seconds": 0.07597420900128782, "tokens": 224 }, { "id": "duplex_idle_03", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.02974857948720455, 0.11033587902784348, 0.8599154949188232 ], "seconds": 0.07996929192449898, "tokens": 226 }, { "id": "duplex_idle_04", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.0368136391043663, 0.3497421443462372, 0.6134442090988159 ], "seconds": 0.07963024999480695, "tokens": 227 }, { "id": "duplex_idle_05", "expected": "enqueue", "choice": "discard", "probabilities": [ 0.07173047214746475, 0.4679584801197052, 0.46031105518341064 ], "seconds": 0.0805491249775514, "tokens": 227 }, { "id": "duplex_idle_06", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.08198647201061249, 0.1435299515724182, 0.7744835615158081 ], "seconds": 0.08135799993760884, "tokens": 228 }, { "id": "duplex_idle_07", "expected": "discard", "choice": "enqueue", "probabilities": [ 0.3057652711868286, 0.33856794238090515, 0.35566675662994385 ], "seconds": 0.07646591705270112, "tokens": 231 }, { "id": "duplex_idle_08", "expected": "discard", "choice": "discard", "probabilities": [ 0.08129880577325821, 0.628534734249115, 0.290166437625885 ], "seconds": 0.08163958298973739, "tokens": 227 }, { "id": "duplex_answer_01", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.03204804286360741, 0.13417251408100128, 0.8337793946266174 ], "seconds": 0.08044004091061652, "tokens": 228 }, { "id": "duplex_answer_02", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.03557829186320305, 0.16976748406887054, 0.7946541905403137 ], "seconds": 0.0781737919896841, "tokens": 232 }, { "id": "duplex_answer_03", "expected": "enqueue", "choice": "discard", "probabilities": [ 0.07960195094347, 0.5172246694564819, 0.40317344665527344 ], "seconds": 0.08178474998567253, "tokens": 229 }, { "id": "duplex_question_stop", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.3803572356700897, 0.1342829465866089, 0.4853598475456238 ], "seconds": 0.08033037499990314, "tokens": 232 }, { "id": "duplex_quoted_stop", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.043692104518413544, 0.11582167446613312, 0.8404862284660339 ], "seconds": 0.08421558304689825, "tokens": 234 }, { "id": "duplex_mid_ack_request", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.03667035326361656, 0.10981252789497375, 0.8535171151161194 ], "seconds": 0.08180612500291318, "tokens": 232 }, { "id": "duplex_immediate_01", "expected": "interrupt", "choice": "discard", "probabilities": [ 0.18807342648506165, 0.5113693475723267, 0.3005572557449341 ], "seconds": 0.08018233301118016, "tokens": 231 }, { "id": "duplex_wait_01", "expected": "interrupt", "choice": "discard", "probabilities": [ 0.17067477107048035, 0.495370090007782, 0.33395519852638245 ], "seconds": 0.08099654200486839, "tokens": 231 }, { "id": "duplex_wait_02", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.16579405963420868, 0.40518027544021606, 0.42902567982673645 ], "seconds": 0.08069149998482317, "tokens": 232 }, { "id": "duplex_blank_01", "expected": "discard", "choice": "discard", "probabilities": [ 0.17456237971782684, 0.5632202625274658, 0.26221734285354614 ], "seconds": 0.08505883300676942, "tokens": 223 }, { "id": "duplex_idle_correction", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.07158209383487701, 0.14543120563030243, 0.7829867005348206 ], "seconds": 0.0851495829410851, "tokens": 232 }, { "id": "duplex_queue_constraint", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.06015212833881378, 0.14546778798103333, 0.7943800687789917 ], "seconds": 0.08386312506627291, "tokens": 232 }, { "id": "duplex_late_request", "expected": "enqueue", "choice": "enqueue", "probabilities": [ 0.1653900444507599, 0.28522974252700806, 0.5493802428245544 ], "seconds": 0.0879432080546394, "tokens": 234 } ] } }