{ "architecture": "integrated_memory_v22", "model_id": "convaiinnovations/laya-typed-decisions", "mode": "extended", "candidate_layers": [ 18, 21, 6, 12, 15, 9, 3, 24, 0, 27 ], "target_policy": "all_full_attention", "accepted_layers": [ 3, 6, 9, 18, 21, 24, 27 ], "replacement_trainable_parameters": 1871408, "training_steps_per_candidate": 2400, "history": [ { "layer": 18, "attention_type": "full_attention", "local": { "nmse": 0.11362843960523605, "cosine": 0.9304072856903076, "core_nmse": 0.09447988867759705, "core_cosine": 0.9509731531143188, "step": 2400, "selection": "fixed_local_probe", "stopped_early": false, "max_steps": 2400 }, "gate": { "n": 1000, "accuracy": 0.748, "brier": 0.43005077296976196, "nll": 0.7588641452374054, "ece": 0.22215451396119248, "soft_accuracy": 0.44061819893755433, "brier_vs_soft": 0.06826150707343509, "score_mae": 0.21814103696153125, "label": "student", "seconds": 29.796996033000028, "ms_per_case": 148.98498016500014, "by_type": { "choice": { "n": 400, "accuracy": 0.7225, "brier": 0.4633154731224705, "nll": 0.8631592748584117, "ece": 0.23906230695972133, "soft_accuracy": 0.4074813313924448, "brier_vs_soft": 0.08710198037979514, "score_mae": null }, "noul": { "n": 200, "accuracy": 0.805, "brier": 0.3668836548, "nll": 0.5536335617227892, "ece": 0.20105199999999995, "soft_accuracy": 0.555346783943, "brier_vs_soft": 0.0686715534697, "score_mae": null }, "score": { "n": 400, "accuracy": 0.745, "brier": 0.4283696319019343, "nll": 0.7571843073737071, "ece": 0.21579797794325972, "soft_accuracy": 0.4163907739799411, "brier_vs_soft": 0.04921601056894259, "score_mae": 0.21814103696153125 } }, "by_workflow": { "agent_trace_observability": { "n": 500, "accuracy": 0.734, "brier": 0.463968746093529, "nll": 0.8048626838894065, "ece": 0.23622216207166163, "soft_accuracy": 0.4129923353760827, "brier_vs_soft": 0.05741277133495497, "score_mae": 0.21219134789438676 }, "customer_service": { "n": 500, "accuracy": 0.762, "brier": 0.3961327998459949, "nll": 0.7128656065854045, "ece": 0.20808686585072322, "soft_accuracy": 0.468244062499026, "brier_vs_soft": 0.07911024281191521, "score_mae": 0.22409072602867572 } }, "teacher_agreement": 0.982, "mean_teacher_kl": 0.00043920122280005636, "mean_probability_l1": 0.005125921018297884 }, "checks": { "local_nmse": true, "local_cosine": true, "teacher_agreement": true, "teacher_kl": true, "accuracy_drop": true }, "accuracy_drop": 0.0, "accepted": true }, { "layer": 21, "attention_type": "full_attention", "local": { "nmse": 0.1493961215019226, "cosine": 0.9177722334861755, "core_nmse": 0.1261713206768036, "core_cosine": 0.9333838820457458, "step": 2400, "selection": "fixed_local_probe", "stopped_early": false, "max_steps": 2400 }, "gate": { "n": 1000, "accuracy": 0.745, "brier": 0.430396009642453, "nll": 0.7594591707309896, "ece": 0.2200981587171782, "soft_accuracy": 0.44031246996928314, "brier_vs_soft": 0.06823893554210717, "score_mae": 0.21864299795897335, "label": "student", "seconds": 29.971145822000153, "ms_per_case": 149.85572911000077, "by_type": { "choice": { "n": 400, "accuracy": 0.72, "brier": 0.4636103755738864, "nll": 0.8638339820718056, "ece": 0.23745904625977457, "soft_accuracy": 0.4071218741164909, "brier_vs_soft": 0.08715588528575308, "score_mae": null }, "noul": { "n": 200, "accuracy": 0.8, "brier": 0.36631926570000006, "nll": 0.5530994787718863, "ece": 0.19571449999999999, "soft_accuracy": 0.555447987248, "brier_vs_soft": 0.0683717577597, "score_mae": null }, "score": { "n": 400, "accuracy": 0.7425, "brier": 0.4292200156822459, "nll": 0.758264205369725, "ece": 0.21492910053317102, "soft_accuracy": 0.4159353071827171, "brier_vs_soft": 0.04925557468966485, "score_mae": 0.21864299795897335 } }, "by_workflow": { "agent_trace_observability": { "n": 500, "accuracy": 0.73, "brier": 0.4641768395079895, "nll": 0.8052829411532971, "ece": 0.2330059310973693, "soft_accuracy": 0.4128266115908907, "brier_vs_soft": 0.057323004663230426, "score_mae": 0.213290374771492 }, "customer_service": { "n": 500, "accuracy": 0.76, "brier": 0.39661517977691646, "nll": 0.7136354003086819, "ece": 0.20719038633698714, "soft_accuracy": 0.4677983283476757, "brier_vs_soft": 0.07915486642098392, "score_mae": 0.2239956211464547 } }, "teacher_agreement": 0.989, "mean_teacher_kl": 0.0004926837096799726, "mean_probability_l1": 0.005603137292484325 }, "checks": { "local_nmse": true, "local_cosine": true, "teacher_agreement": true, "teacher_kl": true, "accuracy_drop": true }, "accuracy_drop": 0.0030000000000000027, "accepted": true }, { "layer": 6, "attention_type": "full_attention", "local": { "nmse": 0.13072997331619263, "cosine": 0.9300991296768188, "core_nmse": 0.14032618701457977, "core_cosine": 0.924555778503418, "step": 2400, "selection": "fixed_local_probe", "stopped_early": false, "max_steps": 2400 }, "gate": { "n": 1000, "accuracy": 0.752, "brier": 0.43184477212854155, "nll": 0.7614958322014271, "ece": 0.2287535478132063, "soft_accuracy": 0.43975343306166553, "brier_vs_soft": 0.06854340648688903, "score_mae": 0.2193585090743151, "label": "student", "seconds": 30.871834197001363, "ms_per_case": 154.3591709850068, "by_type": { "choice": { "n": 400, "accuracy": 0.725, "brier": 0.46423416737090173, "nll": 0.8641887590826992, "ece": 0.24488789129460095, "soft_accuracy": 0.40668406216151026, "brier_vs_soft": 0.08750800599237085, "score_mae": null }, "noul": { "n": 200, "accuracy": 0.805, "brier": 0.36921538589999997, "nll": 0.556216914613668, "ece": 0.2016294999999999, "soft_accuracy": 0.5545180106319999, "brier_vs_soft": 0.0695238311917, "score_mae": null }, "score": { "n": 400, "accuracy": 0.7525, "brier": 0.43077007000045214, "nll": 0.7614423641140347, "ece": 0.22618122823841474, "soft_accuracy": 0.41544051517665376, "brier_vs_soft": 0.049088594629001704, "score_mae": 0.2193585090743151 } }, "by_workflow": { "agent_trace_observability": { "n": 500, "accuracy": 0.736, "brier": 0.4661739240962809, "nll": 0.8080530995796666, "ece": 0.24098506181170062, "soft_accuracy": 0.41200836490914017, "brier_vs_soft": 0.05782665066994771, "score_mae": 0.21337884047415664 }, "customer_service": { "n": 500, "accuracy": 0.768, "brier": 0.39751562016080216, "nll": 0.7149385648231876, "ece": 0.21652203381471197, "soft_accuracy": 0.46749850121419106, "brier_vs_soft": 0.07926016230383034, "score_mae": 0.22533817767447353 } }, "teacher_agreement": 0.98, "mean_teacher_kl": 0.0010271029373721174, "mean_probability_l1": 0.00812321257742765 }, "checks": { "local_nmse": true, "local_cosine": true, "teacher_agreement": true, "teacher_kl": true, "accuracy_drop": true }, "accuracy_drop": -0.0040000000000000036, "accepted": true }, { "layer": 12, "attention_type": "full_attention", "local": { "nmse": 0.11989885568618774, "cosine": 0.9406139850616455, "core_nmse": 0.10705665498971939, "core_cosine": 0.9476187229156494, "step": 2400, "selection": "fixed_local_probe", "stopped_early": false, "max_steps": 2400 }, "gate": { "n": 1000, "accuracy": 0.737, "brier": 0.4451121526506912, "nll": 0.7814492962575, "ece": 0.22224317739536295, "soft_accuracy": 0.4311548665614169, "brier_vs_soft": 0.07736479544545603, "score_mae": 0.22905537353219457, "label": "student", "seconds": 31.777739685998313, "ms_per_case": 158.88869842999156, "by_type": { "choice": { "n": 400, "accuracy": 0.695, "brier": 0.48762291455921897, "nll": 0.9019045301351207, "ece": 0.2325170915848214, "soft_accuracy": 0.38991291995291616, "brier_vs_soft": 0.10415383692291727, "score_mae": null }, "noul": { "n": 200, "accuracy": 0.81, "brier": 0.36981991960000005, "nll": 0.5570194723908173, "ece": 0.20439000000000002, "soft_accuracy": 0.5549397299490001, "brier_vs_soft": 0.06940792625770001, "score_mae": null }, "score": { "n": 400, "accuracy": 0.7425, "brier": 0.440247507267509, "nll": 0.7732089743132206, "ece": 0.22089585190358604, "soft_accuracy": 0.410504381476126, "brier_vs_soft": 0.054554188561872836, "score_mae": 0.22905537353219457 } }, "by_workflow": { "agent_trace_observability": { "n": 500, "accuracy": 0.722, "brier": 0.47220048988571256, "nll": 0.815223399204577, "ece": 0.22962526407533265, "soft_accuracy": 0.4098919200249277, "brier_vs_soft": 0.0603082157959773, "score_mae": 0.21736215985438315 }, "customer_service": { "n": 500, "accuracy": 0.752, "brier": 0.4180238154156699, "nll": 0.7476751933104229, "ece": 0.21486109071539328, "soft_accuracy": 0.4524178130979061, "brier_vs_soft": 0.09442137509493477, "score_mae": 0.24074858721000605 } }, "teacher_agreement": 0.945, "mean_teacher_kl": 0.0068911724306467805, "mean_probability_l1": 0.0186815110149613 }, "checks": { "local_nmse": true, "local_cosine": true, "teacher_agreement": true, "teacher_kl": false, "accuracy_drop": false }, "accuracy_drop": 0.01100000000000001, "accepted": false }, { "layer": 15, "attention_type": "full_attention", "local": { "nmse": 0.14503473043441772, "cosine": 0.9216394424438477, "core_nmse": 0.10070706903934479, "core_cosine": 0.9476621150970459, "step": 2400, "selection": "fixed_local_probe", "stopped_early": false, "max_steps": 2400 }, "gate": { "n": 1000, "accuracy": 0.734, "brier": 0.4400537367778078, "nll": 0.7728026000429391, "ece": 0.21887670220421243, "soft_accuracy": 0.43518839510643464, "brier_vs_soft": 0.07176150191934047, "score_mae": 0.232792562179009, "label": "student", "seconds": 31.843662048999704, "ms_per_case": 159.21831024499852, "by_type": { "choice": { "n": 400, "accuracy": 0.7225, "brier": 0.4654415268266622, "nll": 0.8652935729713741, "ece": 0.24448123000659328, "soft_accuracy": 0.4056319169477455, "brier_vs_soft": 0.08822336533879506, "score_mae": null }, "noul": { "n": 200, "accuracy": 0.785, "brier": 0.36850824470000004, "nll": 0.5555531212091062, "ece": 0.21048849999999997, "soft_accuracy": 0.554711740936, "brier_vs_soft": 0.06918922938370001, "score_mae": null }, "score": { "n": 400, "accuracy": 0.72, "brier": 0.45043869276785725, "nll": 0.7889363665314206, "ece": 0.21185527550393773, "soft_accuracy": 0.4049832003503412, "brier_vs_soft": 0.056585774767706126, "score_mae": 0.232792562179009 } }, "by_workflow": { "agent_trace_observability": { "n": 500, "accuracy": 0.716, "brier": 0.4737819488996494, "nll": 0.817835790730735, "ece": 0.22955851485822512, "soft_accuracy": 0.4088191008209613, "brier_vs_soft": 0.05985231643415291, "score_mae": 0.22148235973829358 }, "customer_service": { "n": 500, "accuracy": 0.752, "brier": 0.40632552465596616, "nll": 0.7277694093551432, "ece": 0.20819488955019963, "soft_accuracy": 0.46155768939190794, "brier_vs_soft": 0.08367068740452804, "score_mae": 0.2441027646197244 } }, "teacher_agreement": 0.945, "mean_teacher_kl": 0.004948329810964568, "mean_probability_l1": 0.01701411528191785 }, "checks": { "local_nmse": true, "local_cosine": true, "teacher_agreement": true, "teacher_kl": true, "accuracy_drop": false }, "accuracy_drop": 0.014000000000000012, "accepted": false }, { "layer": 9, "attention_type": "full_attention", "local": { "nmse": 0.16373053193092346, "cosine": 0.917041003704071, "core_nmse": 0.1001967117190361, "core_cosine": 0.9492738842964172, "step": 2400, "selection": "fixed_local_probe", "stopped_early": false, "max_steps": 2400 }, "gate": { "n": 1000, "accuracy": 0.746, "brier": 0.4326894920393437, "nll": 0.7625643616838623, "ece": 0.22263564072602246, "soft_accuracy": 0.4394347678305118, "brier_vs_soft": 0.06912313849866122, "score_mae": 0.22078179979357135, "label": "student", "seconds": 31.906277011002658, "ms_per_case": 159.5313850550133, "by_type": { "choice": { "n": 400, "accuracy": 0.725, "brier": 0.4645873524934436, "nll": 0.8642223217305464, "ece": 0.2445124128747341, "soft_accuracy": 0.4065933804090691, "brier_vs_soft": 0.08789013758568491, "score_mae": null }, "noul": { "n": 200, "accuracy": 0.785, "brier": 0.36889253190000004, "nll": 0.5558684054922831, "ece": 0.20576750000000008, "soft_accuracy": 0.554740147296, "brier_vs_soft": 0.0693927038637, "score_mae": null }, "score": { "n": 400, "accuracy": 0.7475, "brier": 0.4326901116549156, "nll": 0.764254379732968, "ece": 0.22180893894032183, "soft_accuracy": 0.4146234655192103, "brier_vs_soft": 0.050221356729118136, "score_mae": 0.22078179979357135 } }, "by_workflow": { "agent_trace_observability": { "n": 500, "accuracy": 0.73, "brier": 0.4651749865710483, "nll": 0.8059116003203998, "ece": 0.23456739533337795, "soft_accuracy": 0.41251951180578633, "brier_vs_soft": 0.05714289214798141, "score_mae": 0.2112158322606016 }, "customer_service": { "n": 500, "accuracy": 0.762, "brier": 0.4002039975076392, "nll": 0.719217123047325, "ece": 0.21070388611866686, "soft_accuracy": 0.46635002385523716, "brier_vs_soft": 0.08110338484934101, "score_mae": 0.23034776732654108 } }, "teacher_agreement": 0.97, "mean_teacher_kl": 0.001959835004120846, "mean_probability_l1": 0.010466561451642415 }, "checks": { "local_nmse": true, "local_cosine": true, "teacher_agreement": true, "teacher_kl": true, "accuracy_drop": true }, "accuracy_drop": 0.0020000000000000018, "accepted": true }, { "layer": 3, "attention_type": "full_attention", "local": { "nmse": 0.08624166995286942, "cosine": 0.9507652521133423, "core_nmse": 0.08466289937496185, "core_cosine": 0.9518517851829529, "step": 2400, "selection": "fixed_local_probe", "stopped_early": false, "max_steps": 2400 }, "gate": { "n": 1000, "accuracy": 0.751, "brier": 0.43303843016039956, "nll": 0.7634377173871496, "ece": 0.22739254162679365, "soft_accuracy": 0.43889166842389105, "brier_vs_soft": 0.06977970007029172, "score_mae": 0.22193333491449338, "label": "student", "seconds": 32.575867269999435, "ms_per_case": 162.87933634999717, "by_type": { "choice": { "n": 400, "accuracy": 0.725, "brier": 0.4673841244008108, "nll": 0.8684986012572022, "ece": 0.24415339580571704, "soft_accuracy": 0.4053181651195057, "brier_vs_soft": 0.08921730749392555, "score_mae": null }, "noul": { "n": 200, "accuracy": 0.81, "brier": 0.3665819496, "nll": 0.5535460082950603, "ece": 0.20607499999999995, "soft_accuracy": 0.555090147532, "brier_vs_soft": 0.0685651210917, "score_mae": null }, "score": { "n": 400, "accuracy": 0.7475, "brier": 0.43192097620018827, "nll": 0.7633226880631417, "ece": 0.2212904582612671, "soft_accuracy": 0.41436593217422185, "brier_vs_soft": 0.050949382135953715, "score_mae": 0.22193333491449338 } }, "by_workflow": { "agent_trace_observability": { "n": 500, "accuracy": 0.736, "brier": 0.4653529733049378, "nll": 0.8063698589707317, "ece": 0.2411249899314159, "soft_accuracy": 0.41216580514081547, "brier_vs_soft": 0.05756798112570154, "score_mae": 0.2106706858008335 }, "customer_service": { "n": 500, "accuracy": 0.766, "brier": 0.40072388701586137, "nll": 0.7205055758035676, "ece": 0.2136600933221714, "soft_accuracy": 0.46561753170696646, "brier_vs_soft": 0.0819914190148819, "score_mae": 0.2331959840281532 } }, "teacher_agreement": 0.963, "mean_teacher_kl": 0.002522368066867648, "mean_probability_l1": 0.011552580771417378 }, "checks": { "local_nmse": true, "local_cosine": true, "teacher_agreement": true, "teacher_kl": true, "accuracy_drop": true }, "accuracy_drop": -0.0030000000000000027, "accepted": true }, { "layer": 24, "attention_type": "full_attention", "local": { "nmse": 0.09044437110424042, "cosine": 0.9499064683914185, "core_nmse": 0.10123464465141296, "core_cosine": 0.94173663854599, "step": 2400, "selection": "fixed_local_probe", "stopped_early": false, "max_steps": 2400 }, "gate": { "n": 1000, "accuracy": 0.749, "brier": 0.43318270866692654, "nll": 0.7635784062304858, "ece": 0.22556581206177212, "soft_accuracy": 0.4388625438397355, "brier_vs_soft": 0.06976353556847982, "score_mae": 0.22186239742387145, "label": "student", "seconds": 33.56214573299894, "ms_per_case": 167.8107286649947, "by_type": { "choice": { "n": 400, "accuracy": 0.72, "brier": 0.46739712928921817, "nll": 0.8685046476391023, "ece": 0.23909144950995453, "soft_accuracy": 0.405395389850902, "brier_vs_soft": 0.08919915490222602, "score_mae": null }, "noul": { "n": 200, "accuracy": 0.81, "brier": 0.3667069796, "nll": 0.5537291724018233, "ece": 0.20610299999999998, "soft_accuracy": 0.554977924251, "brier_vs_soft": 0.0686345976537, "score_mae": null }, "score": { "n": 400, "accuracy": 0.7475, "brier": 0.4322061525780982, "nll": 0.7635767817362002, "ece": 0.2217715806444758, "soft_accuracy": 0.4142720076229368, "brier_vs_soft": 0.05089238519212353, "score_mae": 0.22186239742387145 } }, "by_workflow": { "agent_trace_observability": { "n": 500, "accuracy": 0.734, "brier": 0.46557783976917466, "nll": 0.8066150806956318, "ece": 0.23939162214437618, "soft_accuracy": 0.4120801081863282, "brier_vs_soft": 0.05757080955315063, "score_mae": 0.21037775187010915 }, "customer_service": { "n": 500, "accuracy": 0.764, "brier": 0.4007875775646784, "nll": 0.7205417317653398, "ece": 0.21174000197916812, "soft_accuracy": 0.4656449794931428, "brier_vs_soft": 0.081956261583809, "score_mae": 0.23334704297763376 } }, "teacher_agreement": 0.966, "mean_teacher_kl": 0.002522971636062206, "mean_probability_l1": 0.011632607242906505 }, "checks": { "local_nmse": true, "local_cosine": true, "teacher_agreement": true, "teacher_kl": true, "accuracy_drop": true }, "accuracy_drop": -0.0010000000000000009, "accepted": true }, { "layer": 0, "attention_type": "full_attention", "local": { "nmse": 0.06483590602874756, "cosine": 0.9628668427467346, "core_nmse": 0.07933718711137772, "core_cosine": 0.9528903961181641, "step": 2400, "selection": "fixed_local_probe", "stopped_early": false, "max_steps": 2400 }, "gate": { "n": 1000, "accuracy": 0.739, "brier": 0.4346743207237487, "nll": 0.7655480970372092, "ece": 0.2159526114042908, "soft_accuracy": 0.4384471725912035, "brier_vs_soft": 0.07051399034492577, "score_mae": 0.22281430516891476, "label": "student", "seconds": 34.48142596600155, "ms_per_case": 172.40712983000776, "by_type": { "choice": { "n": 400, "accuracy": 0.72, "brier": 0.46749879578435005, "nll": 0.8688783597970884, "ece": 0.23952216160213824, "soft_accuracy": 0.40516191952424613, "brier_vs_soft": 0.0894530732436099, "score_mae": null }, "noul": { "n": 200, "accuracy": 0.785, "brier": 0.37000114849999993, "nll": 0.557076220128829, "ece": 0.18139750000000002, "soft_accuracy": 0.5540828738670001, "brier_vs_soft": 0.0703898673217, "score_mae": null }, "score": { "n": 400, "accuracy": 0.735, "brier": 0.4341864317750217, "nll": 0.76645377273152, "ece": 0.2096606169085887, "soft_accuracy": 0.4139145750202627, "brier_vs_soft": 0.05163696895785451, "score_mae": 0.22281430516891476 } }, "by_workflow": { "agent_trace_observability": { "n": 500, "accuracy": 0.736, "brier": 0.4662215099089698, "nll": 0.8074091011173287, "ece": 0.24143334167850336, "soft_accuracy": 0.4119885667478453, "brier_vs_soft": 0.057816476001840686, "score_mae": 0.21138923880984406 }, "customer_service": { "n": 500, "accuracy": 0.742, "brier": 0.40312713153852753, "nll": 0.7236870929570896, "ece": 0.19047188113007807, "soft_accuracy": 0.4649057784345617, "brier_vs_soft": 0.08321150468801085, "score_mae": 0.2342393715279854 } }, "teacher_agreement": 0.957, "mean_teacher_kl": 0.0028734395869273914, "mean_probability_l1": 0.012472829674752776 }, "checks": { "local_nmse": true, "local_cosine": true, "teacher_agreement": true, "teacher_kl": true, "accuracy_drop": false }, "accuracy_drop": 0.009000000000000008, "accepted": false }, { "layer": 27, "attention_type": "full_attention", "local": { "nmse": 0.07668855786323547, "cosine": 0.9628040194511414, "core_nmse": 0.057369399815797806, "core_cosine": 0.9719479084014893, "step": 2400, "selection": "fixed_local_probe", "stopped_early": false, "max_steps": 2400 }, "gate": { "n": 1000, "accuracy": 0.749, "brier": 0.4330321863186232, "nll": 0.7633199546701852, "ece": 0.2253725827434718, "soft_accuracy": 0.43896820859508484, "brier_vs_soft": 0.06971366934257177, "score_mae": 0.22179166365122913, "label": "student", "seconds": 34.60054616900379, "ms_per_case": 173.00273084501896, "by_type": { "choice": { "n": 400, "accuracy": 0.72, "brier": 0.4672340437610136, "nll": 0.8682603920607386, "ece": 0.23892174802640742, "soft_accuracy": 0.40550195800542055, "brier_vs_soft": 0.08911393645740949, "score_mae": null }, "noul": { "n": 200, "accuracy": 0.81, "brier": 0.3665461023, "nll": 0.5535361754039433, "ece": 0.20588249999999997, "soft_accuracy": 0.555074440901, "brier_vs_soft": 0.0685756870537, "score_mae": null }, "score": { "n": 400, "accuracy": 0.7475, "brier": 0.4320733708855446, "nll": 0.7632714069127531, "ece": 0.2215684588322721, "soft_accuracy": 0.4143813430317915, "brier_vs_soft": 0.05088239337216994, "score_mae": 0.22179166365122913 } }, "by_workflow": { "agent_trace_observability": { "n": 500, "accuracy": 0.734, "brier": 0.46541567472509543, "nll": 0.8063145387748955, "ece": 0.2392596678763307, "soft_accuracy": 0.41216757063707904, "brier_vs_soft": 0.05751229555397543, "score_mae": 0.2102616243003979 }, "customer_service": { "n": 500, "accuracy": 0.764, "brier": 0.40064869791215113, "nll": 0.7203253705654751, "ece": 0.21148549761061305, "soft_accuracy": 0.46576884655309064, "brier_vs_soft": 0.08191504313116811, "score_mae": 0.23332170300206037 } }, "teacher_agreement": 0.966, "mean_teacher_kl": 0.002521283806788623, "mean_probability_l1": 0.01163878468964715 }, "checks": { "local_nmse": true, "local_cosine": true, "teacher_agreement": true, "teacher_kl": true, "accuracy_drop": true }, "accuracy_drop": -0.0010000000000000009, "accepted": true } ], "teacher_final": { "n": 2000, "accuracy": 0.7665, "brier": 0.399591796314754, "nll": 0.7068979180717072, "ece": 0.2137387190950032, "soft_accuracy": 0.47065861261740327, "brier_vs_soft": 0.061465647879173455, "score_mae": 0.24240958086979567, "label": "teacher", "seconds": 62.99023255499651, "ms_per_case": 157.47558138749127, "by_type": { "choice": { "n": 600, "accuracy": 0.7333333333333333, "brier": 0.46509578972312426, "nll": 0.8650092521058057, "ece": 0.2548481686896733, "soft_accuracy": 0.398595875704782, "brier_vs_soft": 0.082128886416041, "score_mae": null }, "noul": { "n": 600, "accuracy": 0.8566666666666667, "brier": 0.2872277414, "nll": 0.4560772115719036, "ece": 0.1920716666666667, "soft_accuracy": 0.618722929173, "brier_vs_soft": 0.049437733831919996, "score_mae": null }, "score": { "n": 800, "accuracy": 0.72375, "brier": 0.43473684244454175, "nll": 0.7764299474209859, "ece": 0.199926421220253, "soft_accuracy": 0.4136574278851717, "brier_vs_soft": 0.05498915451196287, "score_mae": 0.24240958086979567 } }, "by_workflow": { "agent_trace_observability": { "n": 500, "accuracy": 0.73, "brier": 0.46119840934084355, "nll": 0.8008350310185499, "ece": 0.22930310799388903, "soft_accuracy": 0.4141887346759963, "brier_vs_soft": 0.0569316048974981, "score_mae": 0.21008644820503272 }, "customer_service": { "n": 500, "accuracy": 0.766, "brier": 0.39565424279208455, "nll": 0.7121864685594826, "ece": 0.21476239227480592, "soft_accuracy": 0.4685693174955372, "brier_vs_soft": 0.07900562228400827, "score_mae": 0.21979993301548778 }, "invoice_processing": { "n": 500, "accuracy": 0.804, "brier": 0.3024242175344631, "nll": 0.5564036962833907, "ece": 0.16495172278038128, "soft_accuracy": 0.5735877318126, "brier_vs_soft": 0.06397238746428649, "score_mae": 0.29189428531304207 }, "security_incidents": { "n": 500, "accuracy": 0.766, "brier": 0.43909031559162476, "nll": 0.7581664764254056, "ece": 0.24889776943293365, "soft_accuracy": 0.4262886664854796, "brier_vs_soft": 0.045952976870900976, "score_mae": 0.24785765694562018 } } }, "student_final": { "n": 2000, "accuracy": 0.7575, "brier": 0.40687765482004096, "nll": 0.7185002945841144, "ece": 0.21033691615516764, "soft_accuracy": 0.4667292438683568, "brier_vs_soft": 0.06450369549417787, "score_mae": 0.25252500871759453, "label": "integrated_memory_v22", "seconds": 76.24023421799393, "ms_per_case": 190.60058554498482, "by_type": { "choice": { "n": 600, "accuracy": 0.7233333333333334, "brier": 0.47158730978370766, "nll": 0.8749608360461051, "ece": 0.24948156287451229, "soft_accuracy": 0.3950037715292289, "brier_vs_soft": 0.0853422022728963, "score_mae": null }, "noul": { "n": 600, "accuracy": 0.8433333333333334, "brier": 0.29105601643333334, "nll": 0.46114644940359806, "ece": 0.18118383333333332, "soft_accuracy": 0.6162034307686667, "brier_vs_soft": 0.051389339007253336, "score_mae": null }, "score": { "n": 800, "accuracy": 0.71875, "brier": 0.4452116423873217, "nll": 0.7941702723730087, "ece": 0.20284324323203495, "soft_accuracy": 0.40841770794747034, "brier_vs_soft": 0.05871058277533245, "score_mae": 0.25252500871759453 } }, "by_workflow": { "agent_trace_observability": { "n": 500, "accuracy": 0.734, "brier": 0.46541567472509543, "nll": 0.8063145387748955, "ece": 0.2392596678763307, "soft_accuracy": 0.41216757063707904, "brier_vs_soft": 0.05751229555397543, "score_mae": 0.2102616243003979 }, "customer_service": { "n": 500, "accuracy": 0.764, "brier": 0.40064869791215113, "nll": 0.7203253705654751, "ece": 0.21148549761061305, "soft_accuracy": 0.46576884655309064, "brier_vs_soft": 0.08191504313116811, "score_mae": 0.23332170300206037 }, "invoice_processing": { "n": 500, "accuracy": 0.786, "brier": 0.3159522117429286, "nll": 0.5780789912403593, "ece": 0.1566913383592634, "soft_accuracy": 0.5651686292940649, "brier_vs_soft": 0.07064340927194246, "score_mae": 0.311955511641426 }, "security_incidents": { "n": 500, "accuracy": 0.746, "brier": 0.4454940348999888, "nll": 0.769282277755728, "ece": 0.23391116077446364, "soft_accuracy": 0.42381192898919273, "brier_vs_soft": 0.04794403401962545, "score_mae": 0.25456119592649395 } }, "teacher_agreement": 0.951, "mean_teacher_kl": 0.002814041964657275, "mean_probability_l1": 0.01388176486059745 }, "latency": { "teacher": { "median_ms": 49.591165999572695, "p95_ms": 59.87610415008932, "mean_ms": 52.572452700087524, "repeats": 20 }, "student": { "median_ms": 65.16387300052884, "p95_ms": 84.38417299985304, "mean_ms": 66.04240354995454, "repeats": 20 } }, "demo": { "teacher": { "model": "laya-rl-agent", "answers": { "department": { "type": "choice", "choice": "billing", "probabilities": { "billing": 0.8037, "technical": 0.0677, "sales": 0.0608, "other": 0.0679 }, "confidence": 0.4873, "action": { "act_probability": 1.0 } }, "urgency": { "type": "score", "score": 1.6379, "legend": { "0": "not urgent", "1": "soon", "2": "critical deadline or blocking issue" }, "probabilities": { "0": 0.044, "1": 0.2742, "2": 0.6819 }, "confidence": 0.3144, "action": { "act_probability": 1.0 } }, "churn_risk": { "type": "noul", "noul": 0.7062, "confidence": 0.7062, "action": { "act_probability": 1.0 } }, "refund_requested": { "type": "noul", "noul": 0.6678, "confidence": 0.6678, "action": { "act_probability": 1.0 } } }, "usage": { "input_tokens": 348, "output_tokens": 0 } }, "student": { "model": "laya-rl-agent", "answers": { "department": { "type": "choice", "choice": "billing", "probabilities": { "billing": 0.7816, "technical": 0.0752, "sales": 0.0752, "other": 0.068 }, "confidence": 0.4485, "action": { "act_probability": 1.0 } }, "urgency": { "type": "score", "score": 1.6618, "legend": { "0": "not urgent", "1": "soon", "2": "critical deadline or blocking issue" }, "probabilities": { "0": 0.0544, "1": 0.2293, "2": 0.7162 }, "confidence": 0.3308, "action": { "act_probability": 1.0 } }, "churn_risk": { "type": "noul", "noul": 0.7175, "confidence": 0.7175, "action": { "act_probability": 1.0 } }, "refund_requested": { "type": "noul", "noul": 0.693, "confidence": 0.693, "action": { "act_probability": 1.0 } } }, "usage": { "input_tokens": 348, "output_tokens": 0 } } }, "note": "Laya uses a bidirectional ModernBERT encoder. Integrated Memory V2.2 uses learned attention transfer and tries full-attention layers first; these results are not interchangeable with causal-LM conversions.", "conversion_succeeded": true, "student_is_unmodified_teacher": false }