| { |
| "architecture": "integrated_memory_v22", |
| "model_id": "convaiinnovations/laya-typed-decisions", |
| "mode": "extended", |
| "candidate_layers": [ |
| 18, |
| 21, |
| 6, |
| 12, |
| 15, |
| 9, |
| 3, |
| 24, |
| 0, |
| 27 |
| ], |
| "target_policy": "all_full_attention", |
| "accepted_layers": [ |
| 3, |
| 6, |
| 9, |
| 18, |
| 21, |
| 24, |
| 27 |
| ], |
| "replacement_trainable_parameters": 1871408, |
| "training_steps_per_candidate": 2400, |
| "history": [ |
| { |
| "layer": 18, |
| "attention_type": "full_attention", |
| "local": { |
| "nmse": 0.11362843960523605, |
| "cosine": 0.9304072856903076, |
| "core_nmse": 0.09447988867759705, |
| "core_cosine": 0.9509731531143188, |
| "step": 2400, |
| "selection": "fixed_local_probe", |
| "stopped_early": false, |
| "max_steps": 2400 |
| }, |
| "gate": { |
| "n": 1000, |
| "accuracy": 0.748, |
| "brier": 0.43005077296976196, |
| "nll": 0.7588641452374054, |
| "ece": 0.22215451396119248, |
| "soft_accuracy": 0.44061819893755433, |
| "brier_vs_soft": 0.06826150707343509, |
| "score_mae": 0.21814103696153125, |
| "label": "student", |
| "seconds": 29.796996033000028, |
| "ms_per_case": 148.98498016500014, |
| "by_type": { |
| "choice": { |
| "n": 400, |
| "accuracy": 0.7225, |
| "brier": 0.4633154731224705, |
| "nll": 0.8631592748584117, |
| "ece": 0.23906230695972133, |
| "soft_accuracy": 0.4074813313924448, |
| "brier_vs_soft": 0.08710198037979514, |
| "score_mae": null |
| }, |
| "noul": { |
| "n": 200, |
| "accuracy": 0.805, |
| "brier": 0.3668836548, |
| "nll": 0.5536335617227892, |
| "ece": 0.20105199999999995, |
| "soft_accuracy": 0.555346783943, |
| "brier_vs_soft": 0.0686715534697, |
| "score_mae": null |
| }, |
| "score": { |
| "n": 400, |
| "accuracy": 0.745, |
| "brier": 0.4283696319019343, |
| "nll": 0.7571843073737071, |
| "ece": 0.21579797794325972, |
| "soft_accuracy": 0.4163907739799411, |
| "brier_vs_soft": 0.04921601056894259, |
| "score_mae": 0.21814103696153125 |
| } |
| }, |
| "by_workflow": { |
| "agent_trace_observability": { |
| "n": 500, |
| "accuracy": 0.734, |
| "brier": 0.463968746093529, |
| "nll": 0.8048626838894065, |
| "ece": 0.23622216207166163, |
| "soft_accuracy": 0.4129923353760827, |
| "brier_vs_soft": 0.05741277133495497, |
| "score_mae": 0.21219134789438676 |
| }, |
| "customer_service": { |
| "n": 500, |
| "accuracy": 0.762, |
| "brier": 0.3961327998459949, |
| "nll": 0.7128656065854045, |
| "ece": 0.20808686585072322, |
| "soft_accuracy": 0.468244062499026, |
| "brier_vs_soft": 0.07911024281191521, |
| "score_mae": 0.22409072602867572 |
| } |
| }, |
| "teacher_agreement": 0.982, |
| "mean_teacher_kl": 0.00043920122280005636, |
| "mean_probability_l1": 0.005125921018297884 |
| }, |
| "checks": { |
| "local_nmse": true, |
| "local_cosine": true, |
| "teacher_agreement": true, |
| "teacher_kl": true, |
| "accuracy_drop": true |
| }, |
| "accuracy_drop": 0.0, |
| "accepted": true |
| }, |
| { |
| "layer": 21, |
| "attention_type": "full_attention", |
| "local": { |
| "nmse": 0.1493961215019226, |
| "cosine": 0.9177722334861755, |
| "core_nmse": 0.1261713206768036, |
| "core_cosine": 0.9333838820457458, |
| "step": 2400, |
| "selection": "fixed_local_probe", |
| "stopped_early": false, |
| "max_steps": 2400 |
| }, |
| "gate": { |
| "n": 1000, |
| "accuracy": 0.745, |
| "brier": 0.430396009642453, |
| "nll": 0.7594591707309896, |
| "ece": 0.2200981587171782, |
| "soft_accuracy": 0.44031246996928314, |
| "brier_vs_soft": 0.06823893554210717, |
| "score_mae": 0.21864299795897335, |
| "label": "student", |
| "seconds": 29.971145822000153, |
| "ms_per_case": 149.85572911000077, |
| "by_type": { |
| "choice": { |
| "n": 400, |
| "accuracy": 0.72, |
| "brier": 0.4636103755738864, |
| "nll": 0.8638339820718056, |
| "ece": 0.23745904625977457, |
| "soft_accuracy": 0.4071218741164909, |
| "brier_vs_soft": 0.08715588528575308, |
| "score_mae": null |
| }, |
| "noul": { |
| "n": 200, |
| "accuracy": 0.8, |
| "brier": 0.36631926570000006, |
| "nll": 0.5530994787718863, |
| "ece": 0.19571449999999999, |
| "soft_accuracy": 0.555447987248, |
| "brier_vs_soft": 0.0683717577597, |
| "score_mae": null |
| }, |
| "score": { |
| "n": 400, |
| "accuracy": 0.7425, |
| "brier": 0.4292200156822459, |
| "nll": 0.758264205369725, |
| "ece": 0.21492910053317102, |
| "soft_accuracy": 0.4159353071827171, |
| "brier_vs_soft": 0.04925557468966485, |
| "score_mae": 0.21864299795897335 |
| } |
| }, |
| "by_workflow": { |
| "agent_trace_observability": { |
| "n": 500, |
| "accuracy": 0.73, |
| "brier": 0.4641768395079895, |
| "nll": 0.8052829411532971, |
| "ece": 0.2330059310973693, |
| "soft_accuracy": 0.4128266115908907, |
| "brier_vs_soft": 0.057323004663230426, |
| "score_mae": 0.213290374771492 |
| }, |
| "customer_service": { |
| "n": 500, |
| "accuracy": 0.76, |
| "brier": 0.39661517977691646, |
| "nll": 0.7136354003086819, |
| "ece": 0.20719038633698714, |
| "soft_accuracy": 0.4677983283476757, |
| "brier_vs_soft": 0.07915486642098392, |
| "score_mae": 0.2239956211464547 |
| } |
| }, |
| "teacher_agreement": 0.989, |
| "mean_teacher_kl": 0.0004926837096799726, |
| "mean_probability_l1": 0.005603137292484325 |
| }, |
| "checks": { |
| "local_nmse": true, |
| "local_cosine": true, |
| "teacher_agreement": true, |
| "teacher_kl": true, |
| "accuracy_drop": true |
| }, |
| "accuracy_drop": 0.0030000000000000027, |
| "accepted": true |
| }, |
| { |
| "layer": 6, |
| "attention_type": "full_attention", |
| "local": { |
| "nmse": 0.13072997331619263, |
| "cosine": 0.9300991296768188, |
| "core_nmse": 0.14032618701457977, |
| "core_cosine": 0.924555778503418, |
| "step": 2400, |
| "selection": "fixed_local_probe", |
| "stopped_early": false, |
| "max_steps": 2400 |
| }, |
| "gate": { |
| "n": 1000, |
| "accuracy": 0.752, |
| "brier": 0.43184477212854155, |
| "nll": 0.7614958322014271, |
| "ece": 0.2287535478132063, |
| "soft_accuracy": 0.43975343306166553, |
| "brier_vs_soft": 0.06854340648688903, |
| "score_mae": 0.2193585090743151, |
| "label": "student", |
| "seconds": 30.871834197001363, |
| "ms_per_case": 154.3591709850068, |
| "by_type": { |
| "choice": { |
| "n": 400, |
| "accuracy": 0.725, |
| "brier": 0.46423416737090173, |
| "nll": 0.8641887590826992, |
| "ece": 0.24488789129460095, |
| "soft_accuracy": 0.40668406216151026, |
| "brier_vs_soft": 0.08750800599237085, |
| "score_mae": null |
| }, |
| "noul": { |
| "n": 200, |
| "accuracy": 0.805, |
| "brier": 0.36921538589999997, |
| "nll": 0.556216914613668, |
| "ece": 0.2016294999999999, |
| "soft_accuracy": 0.5545180106319999, |
| "brier_vs_soft": 0.0695238311917, |
| "score_mae": null |
| }, |
| "score": { |
| "n": 400, |
| "accuracy": 0.7525, |
| "brier": 0.43077007000045214, |
| "nll": 0.7614423641140347, |
| "ece": 0.22618122823841474, |
| "soft_accuracy": 0.41544051517665376, |
| "brier_vs_soft": 0.049088594629001704, |
| "score_mae": 0.2193585090743151 |
| } |
| }, |
| "by_workflow": { |
| "agent_trace_observability": { |
| "n": 500, |
| "accuracy": 0.736, |
| "brier": 0.4661739240962809, |
| "nll": 0.8080530995796666, |
| "ece": 0.24098506181170062, |
| "soft_accuracy": 0.41200836490914017, |
| "brier_vs_soft": 0.05782665066994771, |
| "score_mae": 0.21337884047415664 |
| }, |
| "customer_service": { |
| "n": 500, |
| "accuracy": 0.768, |
| "brier": 0.39751562016080216, |
| "nll": 0.7149385648231876, |
| "ece": 0.21652203381471197, |
| "soft_accuracy": 0.46749850121419106, |
| "brier_vs_soft": 0.07926016230383034, |
| "score_mae": 0.22533817767447353 |
| } |
| }, |
| "teacher_agreement": 0.98, |
| "mean_teacher_kl": 0.0010271029373721174, |
| "mean_probability_l1": 0.00812321257742765 |
| }, |
| "checks": { |
| "local_nmse": true, |
| "local_cosine": true, |
| "teacher_agreement": true, |
| "teacher_kl": true, |
| "accuracy_drop": true |
| }, |
| "accuracy_drop": -0.0040000000000000036, |
| "accepted": true |
| }, |
| { |
| "layer": 12, |
| "attention_type": "full_attention", |
| "local": { |
| "nmse": 0.11989885568618774, |
| "cosine": 0.9406139850616455, |
| "core_nmse": 0.10705665498971939, |
| "core_cosine": 0.9476187229156494, |
| "step": 2400, |
| "selection": "fixed_local_probe", |
| "stopped_early": false, |
| "max_steps": 2400 |
| }, |
| "gate": { |
| "n": 1000, |
| "accuracy": 0.737, |
| "brier": 0.4451121526506912, |
| "nll": 0.7814492962575, |
| "ece": 0.22224317739536295, |
| "soft_accuracy": 0.4311548665614169, |
| "brier_vs_soft": 0.07736479544545603, |
| "score_mae": 0.22905537353219457, |
| "label": "student", |
| "seconds": 31.777739685998313, |
| "ms_per_case": 158.88869842999156, |
| "by_type": { |
| "choice": { |
| "n": 400, |
| "accuracy": 0.695, |
| "brier": 0.48762291455921897, |
| "nll": 0.9019045301351207, |
| "ece": 0.2325170915848214, |
| "soft_accuracy": 0.38991291995291616, |
| "brier_vs_soft": 0.10415383692291727, |
| "score_mae": null |
| }, |
| "noul": { |
| "n": 200, |
| "accuracy": 0.81, |
| "brier": 0.36981991960000005, |
| "nll": 0.5570194723908173, |
| "ece": 0.20439000000000002, |
| "soft_accuracy": 0.5549397299490001, |
| "brier_vs_soft": 0.06940792625770001, |
| "score_mae": null |
| }, |
| "score": { |
| "n": 400, |
| "accuracy": 0.7425, |
| "brier": 0.440247507267509, |
| "nll": 0.7732089743132206, |
| "ece": 0.22089585190358604, |
| "soft_accuracy": 0.410504381476126, |
| "brier_vs_soft": 0.054554188561872836, |
| "score_mae": 0.22905537353219457 |
| } |
| }, |
| "by_workflow": { |
| "agent_trace_observability": { |
| "n": 500, |
| "accuracy": 0.722, |
| "brier": 0.47220048988571256, |
| "nll": 0.815223399204577, |
| "ece": 0.22962526407533265, |
| "soft_accuracy": 0.4098919200249277, |
| "brier_vs_soft": 0.0603082157959773, |
| "score_mae": 0.21736215985438315 |
| }, |
| "customer_service": { |
| "n": 500, |
| "accuracy": 0.752, |
| "brier": 0.4180238154156699, |
| "nll": 0.7476751933104229, |
| "ece": 0.21486109071539328, |
| "soft_accuracy": 0.4524178130979061, |
| "brier_vs_soft": 0.09442137509493477, |
| "score_mae": 0.24074858721000605 |
| } |
| }, |
| "teacher_agreement": 0.945, |
| "mean_teacher_kl": 0.0068911724306467805, |
| "mean_probability_l1": 0.0186815110149613 |
| }, |
| "checks": { |
| "local_nmse": true, |
| "local_cosine": true, |
| "teacher_agreement": true, |
| "teacher_kl": false, |
| "accuracy_drop": false |
| }, |
| "accuracy_drop": 0.01100000000000001, |
| "accepted": false |
| }, |
| { |
| "layer": 15, |
| "attention_type": "full_attention", |
| "local": { |
| "nmse": 0.14503473043441772, |
| "cosine": 0.9216394424438477, |
| "core_nmse": 0.10070706903934479, |
| "core_cosine": 0.9476621150970459, |
| "step": 2400, |
| "selection": "fixed_local_probe", |
| "stopped_early": false, |
| "max_steps": 2400 |
| }, |
| "gate": { |
| "n": 1000, |
| "accuracy": 0.734, |
| "brier": 0.4400537367778078, |
| "nll": 0.7728026000429391, |
| "ece": 0.21887670220421243, |
| "soft_accuracy": 0.43518839510643464, |
| "brier_vs_soft": 0.07176150191934047, |
| "score_mae": 0.232792562179009, |
| "label": "student", |
| "seconds": 31.843662048999704, |
| "ms_per_case": 159.21831024499852, |
| "by_type": { |
| "choice": { |
| "n": 400, |
| "accuracy": 0.7225, |
| "brier": 0.4654415268266622, |
| "nll": 0.8652935729713741, |
| "ece": 0.24448123000659328, |
| "soft_accuracy": 0.4056319169477455, |
| "brier_vs_soft": 0.08822336533879506, |
| "score_mae": null |
| }, |
| "noul": { |
| "n": 200, |
| "accuracy": 0.785, |
| "brier": 0.36850824470000004, |
| "nll": 0.5555531212091062, |
| "ece": 0.21048849999999997, |
| "soft_accuracy": 0.554711740936, |
| "brier_vs_soft": 0.06918922938370001, |
| "score_mae": null |
| }, |
| "score": { |
| "n": 400, |
| "accuracy": 0.72, |
| "brier": 0.45043869276785725, |
| "nll": 0.7889363665314206, |
| "ece": 0.21185527550393773, |
| "soft_accuracy": 0.4049832003503412, |
| "brier_vs_soft": 0.056585774767706126, |
| "score_mae": 0.232792562179009 |
| } |
| }, |
| "by_workflow": { |
| "agent_trace_observability": { |
| "n": 500, |
| "accuracy": 0.716, |
| "brier": 0.4737819488996494, |
| "nll": 0.817835790730735, |
| "ece": 0.22955851485822512, |
| "soft_accuracy": 0.4088191008209613, |
| "brier_vs_soft": 0.05985231643415291, |
| "score_mae": 0.22148235973829358 |
| }, |
| "customer_service": { |
| "n": 500, |
| "accuracy": 0.752, |
| "brier": 0.40632552465596616, |
| "nll": 0.7277694093551432, |
| "ece": 0.20819488955019963, |
| "soft_accuracy": 0.46155768939190794, |
| "brier_vs_soft": 0.08367068740452804, |
| "score_mae": 0.2441027646197244 |
| } |
| }, |
| "teacher_agreement": 0.945, |
| "mean_teacher_kl": 0.004948329810964568, |
| "mean_probability_l1": 0.01701411528191785 |
| }, |
| "checks": { |
| "local_nmse": true, |
| "local_cosine": true, |
| "teacher_agreement": true, |
| "teacher_kl": true, |
| "accuracy_drop": false |
| }, |
| "accuracy_drop": 0.014000000000000012, |
| "accepted": false |
| }, |
| { |
| "layer": 9, |
| "attention_type": "full_attention", |
| "local": { |
| "nmse": 0.16373053193092346, |
| "cosine": 0.917041003704071, |
| "core_nmse": 0.1001967117190361, |
| "core_cosine": 0.9492738842964172, |
| "step": 2400, |
| "selection": "fixed_local_probe", |
| "stopped_early": false, |
| "max_steps": 2400 |
| }, |
| "gate": { |
| "n": 1000, |
| "accuracy": 0.746, |
| "brier": 0.4326894920393437, |
| "nll": 0.7625643616838623, |
| "ece": 0.22263564072602246, |
| "soft_accuracy": 0.4394347678305118, |
| "brier_vs_soft": 0.06912313849866122, |
| "score_mae": 0.22078179979357135, |
| "label": "student", |
| "seconds": 31.906277011002658, |
| "ms_per_case": 159.5313850550133, |
| "by_type": { |
| "choice": { |
| "n": 400, |
| "accuracy": 0.725, |
| "brier": 0.4645873524934436, |
| "nll": 0.8642223217305464, |
| "ece": 0.2445124128747341, |
| "soft_accuracy": 0.4065933804090691, |
| "brier_vs_soft": 0.08789013758568491, |
| "score_mae": null |
| }, |
| "noul": { |
| "n": 200, |
| "accuracy": 0.785, |
| "brier": 0.36889253190000004, |
| "nll": 0.5558684054922831, |
| "ece": 0.20576750000000008, |
| "soft_accuracy": 0.554740147296, |
| "brier_vs_soft": 0.0693927038637, |
| "score_mae": null |
| }, |
| "score": { |
| "n": 400, |
| "accuracy": 0.7475, |
| "brier": 0.4326901116549156, |
| "nll": 0.764254379732968, |
| "ece": 0.22180893894032183, |
| "soft_accuracy": 0.4146234655192103, |
| "brier_vs_soft": 0.050221356729118136, |
| "score_mae": 0.22078179979357135 |
| } |
| }, |
| "by_workflow": { |
| "agent_trace_observability": { |
| "n": 500, |
| "accuracy": 0.73, |
| "brier": 0.4651749865710483, |
| "nll": 0.8059116003203998, |
| "ece": 0.23456739533337795, |
| "soft_accuracy": 0.41251951180578633, |
| "brier_vs_soft": 0.05714289214798141, |
| "score_mae": 0.2112158322606016 |
| }, |
| "customer_service": { |
| "n": 500, |
| "accuracy": 0.762, |
| "brier": 0.4002039975076392, |
| "nll": 0.719217123047325, |
| "ece": 0.21070388611866686, |
| "soft_accuracy": 0.46635002385523716, |
| "brier_vs_soft": 0.08110338484934101, |
| "score_mae": 0.23034776732654108 |
| } |
| }, |
| "teacher_agreement": 0.97, |
| "mean_teacher_kl": 0.001959835004120846, |
| "mean_probability_l1": 0.010466561451642415 |
| }, |
| "checks": { |
| "local_nmse": true, |
| "local_cosine": true, |
| "teacher_agreement": true, |
| "teacher_kl": true, |
| "accuracy_drop": true |
| }, |
| "accuracy_drop": 0.0020000000000000018, |
| "accepted": true |
| }, |
| { |
| "layer": 3, |
| "attention_type": "full_attention", |
| "local": { |
| "nmse": 0.08624166995286942, |
| "cosine": 0.9507652521133423, |
| "core_nmse": 0.08466289937496185, |
| "core_cosine": 0.9518517851829529, |
| "step": 2400, |
| "selection": "fixed_local_probe", |
| "stopped_early": false, |
| "max_steps": 2400 |
| }, |
| "gate": { |
| "n": 1000, |
| "accuracy": 0.751, |
| "brier": 0.43303843016039956, |
| "nll": 0.7634377173871496, |
| "ece": 0.22739254162679365, |
| "soft_accuracy": 0.43889166842389105, |
| "brier_vs_soft": 0.06977970007029172, |
| "score_mae": 0.22193333491449338, |
| "label": "student", |
| "seconds": 32.575867269999435, |
| "ms_per_case": 162.87933634999717, |
| "by_type": { |
| "choice": { |
| "n": 400, |
| "accuracy": 0.725, |
| "brier": 0.4673841244008108, |
| "nll": 0.8684986012572022, |
| "ece": 0.24415339580571704, |
| "soft_accuracy": 0.4053181651195057, |
| "brier_vs_soft": 0.08921730749392555, |
| "score_mae": null |
| }, |
| "noul": { |
| "n": 200, |
| "accuracy": 0.81, |
| "brier": 0.3665819496, |
| "nll": 0.5535460082950603, |
| "ece": 0.20607499999999995, |
| "soft_accuracy": 0.555090147532, |
| "brier_vs_soft": 0.0685651210917, |
| "score_mae": null |
| }, |
| "score": { |
| "n": 400, |
| "accuracy": 0.7475, |
| "brier": 0.43192097620018827, |
| "nll": 0.7633226880631417, |
| "ece": 0.2212904582612671, |
| "soft_accuracy": 0.41436593217422185, |
| "brier_vs_soft": 0.050949382135953715, |
| "score_mae": 0.22193333491449338 |
| } |
| }, |
| "by_workflow": { |
| "agent_trace_observability": { |
| "n": 500, |
| "accuracy": 0.736, |
| "brier": 0.4653529733049378, |
| "nll": 0.8063698589707317, |
| "ece": 0.2411249899314159, |
| "soft_accuracy": 0.41216580514081547, |
| "brier_vs_soft": 0.05756798112570154, |
| "score_mae": 0.2106706858008335 |
| }, |
| "customer_service": { |
| "n": 500, |
| "accuracy": 0.766, |
| "brier": 0.40072388701586137, |
| "nll": 0.7205055758035676, |
| "ece": 0.2136600933221714, |
| "soft_accuracy": 0.46561753170696646, |
| "brier_vs_soft": 0.0819914190148819, |
| "score_mae": 0.2331959840281532 |
| } |
| }, |
| "teacher_agreement": 0.963, |
| "mean_teacher_kl": 0.002522368066867648, |
| "mean_probability_l1": 0.011552580771417378 |
| }, |
| "checks": { |
| "local_nmse": true, |
| "local_cosine": true, |
| "teacher_agreement": true, |
| "teacher_kl": true, |
| "accuracy_drop": true |
| }, |
| "accuracy_drop": -0.0030000000000000027, |
| "accepted": true |
| }, |
| { |
| "layer": 24, |
| "attention_type": "full_attention", |
| "local": { |
| "nmse": 0.09044437110424042, |
| "cosine": 0.9499064683914185, |
| "core_nmse": 0.10123464465141296, |
| "core_cosine": 0.94173663854599, |
| "step": 2400, |
| "selection": "fixed_local_probe", |
| "stopped_early": false, |
| "max_steps": 2400 |
| }, |
| "gate": { |
| "n": 1000, |
| "accuracy": 0.749, |
| "brier": 0.43318270866692654, |
| "nll": 0.7635784062304858, |
| "ece": 0.22556581206177212, |
| "soft_accuracy": 0.4388625438397355, |
| "brier_vs_soft": 0.06976353556847982, |
| "score_mae": 0.22186239742387145, |
| "label": "student", |
| "seconds": 33.56214573299894, |
| "ms_per_case": 167.8107286649947, |
| "by_type": { |
| "choice": { |
| "n": 400, |
| "accuracy": 0.72, |
| "brier": 0.46739712928921817, |
| "nll": 0.8685046476391023, |
| "ece": 0.23909144950995453, |
| "soft_accuracy": 0.405395389850902, |
| "brier_vs_soft": 0.08919915490222602, |
| "score_mae": null |
| }, |
| "noul": { |
| "n": 200, |
| "accuracy": 0.81, |
| "brier": 0.3667069796, |
| "nll": 0.5537291724018233, |
| "ece": 0.20610299999999998, |
| "soft_accuracy": 0.554977924251, |
| "brier_vs_soft": 0.0686345976537, |
| "score_mae": null |
| }, |
| "score": { |
| "n": 400, |
| "accuracy": 0.7475, |
| "brier": 0.4322061525780982, |
| "nll": 0.7635767817362002, |
| "ece": 0.2217715806444758, |
| "soft_accuracy": 0.4142720076229368, |
| "brier_vs_soft": 0.05089238519212353, |
| "score_mae": 0.22186239742387145 |
| } |
| }, |
| "by_workflow": { |
| "agent_trace_observability": { |
| "n": 500, |
| "accuracy": 0.734, |
| "brier": 0.46557783976917466, |
| "nll": 0.8066150806956318, |
| "ece": 0.23939162214437618, |
| "soft_accuracy": 0.4120801081863282, |
| "brier_vs_soft": 0.05757080955315063, |
| "score_mae": 0.21037775187010915 |
| }, |
| "customer_service": { |
| "n": 500, |
| "accuracy": 0.764, |
| "brier": 0.4007875775646784, |
| "nll": 0.7205417317653398, |
| "ece": 0.21174000197916812, |
| "soft_accuracy": 0.4656449794931428, |
| "brier_vs_soft": 0.081956261583809, |
| "score_mae": 0.23334704297763376 |
| } |
| }, |
| "teacher_agreement": 0.966, |
| "mean_teacher_kl": 0.002522971636062206, |
| "mean_probability_l1": 0.011632607242906505 |
| }, |
| "checks": { |
| "local_nmse": true, |
| "local_cosine": true, |
| "teacher_agreement": true, |
| "teacher_kl": true, |
| "accuracy_drop": true |
| }, |
| "accuracy_drop": -0.0010000000000000009, |
| "accepted": true |
| }, |
| { |
| "layer": 0, |
| "attention_type": "full_attention", |
| "local": { |
| "nmse": 0.06483590602874756, |
| "cosine": 0.9628668427467346, |
| "core_nmse": 0.07933718711137772, |
| "core_cosine": 0.9528903961181641, |
| "step": 2400, |
| "selection": "fixed_local_probe", |
| "stopped_early": false, |
| "max_steps": 2400 |
| }, |
| "gate": { |
| "n": 1000, |
| "accuracy": 0.739, |
| "brier": 0.4346743207237487, |
| "nll": 0.7655480970372092, |
| "ece": 0.2159526114042908, |
| "soft_accuracy": 0.4384471725912035, |
| "brier_vs_soft": 0.07051399034492577, |
| "score_mae": 0.22281430516891476, |
| "label": "student", |
| "seconds": 34.48142596600155, |
| "ms_per_case": 172.40712983000776, |
| "by_type": { |
| "choice": { |
| "n": 400, |
| "accuracy": 0.72, |
| "brier": 0.46749879578435005, |
| "nll": 0.8688783597970884, |
| "ece": 0.23952216160213824, |
| "soft_accuracy": 0.40516191952424613, |
| "brier_vs_soft": 0.0894530732436099, |
| "score_mae": null |
| }, |
| "noul": { |
| "n": 200, |
| "accuracy": 0.785, |
| "brier": 0.37000114849999993, |
| "nll": 0.557076220128829, |
| "ece": 0.18139750000000002, |
| "soft_accuracy": 0.5540828738670001, |
| "brier_vs_soft": 0.0703898673217, |
| "score_mae": null |
| }, |
| "score": { |
| "n": 400, |
| "accuracy": 0.735, |
| "brier": 0.4341864317750217, |
| "nll": 0.76645377273152, |
| "ece": 0.2096606169085887, |
| "soft_accuracy": 0.4139145750202627, |
| "brier_vs_soft": 0.05163696895785451, |
| "score_mae": 0.22281430516891476 |
| } |
| }, |
| "by_workflow": { |
| "agent_trace_observability": { |
| "n": 500, |
| "accuracy": 0.736, |
| "brier": 0.4662215099089698, |
| "nll": 0.8074091011173287, |
| "ece": 0.24143334167850336, |
| "soft_accuracy": 0.4119885667478453, |
| "brier_vs_soft": 0.057816476001840686, |
| "score_mae": 0.21138923880984406 |
| }, |
| "customer_service": { |
| "n": 500, |
| "accuracy": 0.742, |
| "brier": 0.40312713153852753, |
| "nll": 0.7236870929570896, |
| "ece": 0.19047188113007807, |
| "soft_accuracy": 0.4649057784345617, |
| "brier_vs_soft": 0.08321150468801085, |
| "score_mae": 0.2342393715279854 |
| } |
| }, |
| "teacher_agreement": 0.957, |
| "mean_teacher_kl": 0.0028734395869273914, |
| "mean_probability_l1": 0.012472829674752776 |
| }, |
| "checks": { |
| "local_nmse": true, |
| "local_cosine": true, |
| "teacher_agreement": true, |
| "teacher_kl": true, |
| "accuracy_drop": false |
| }, |
| "accuracy_drop": 0.009000000000000008, |
| "accepted": false |
| }, |
| { |
| "layer": 27, |
| "attention_type": "full_attention", |
| "local": { |
| "nmse": 0.07668855786323547, |
| "cosine": 0.9628040194511414, |
| "core_nmse": 0.057369399815797806, |
| "core_cosine": 0.9719479084014893, |
| "step": 2400, |
| "selection": "fixed_local_probe", |
| "stopped_early": false, |
| "max_steps": 2400 |
| }, |
| "gate": { |
| "n": 1000, |
| "accuracy": 0.749, |
| "brier": 0.4330321863186232, |
| "nll": 0.7633199546701852, |
| "ece": 0.2253725827434718, |
| "soft_accuracy": 0.43896820859508484, |
| "brier_vs_soft": 0.06971366934257177, |
| "score_mae": 0.22179166365122913, |
| "label": "student", |
| "seconds": 34.60054616900379, |
| "ms_per_case": 173.00273084501896, |
| "by_type": { |
| "choice": { |
| "n": 400, |
| "accuracy": 0.72, |
| "brier": 0.4672340437610136, |
| "nll": 0.8682603920607386, |
| "ece": 0.23892174802640742, |
| "soft_accuracy": 0.40550195800542055, |
| "brier_vs_soft": 0.08911393645740949, |
| "score_mae": null |
| }, |
| "noul": { |
| "n": 200, |
| "accuracy": 0.81, |
| "brier": 0.3665461023, |
| "nll": 0.5535361754039433, |
| "ece": 0.20588249999999997, |
| "soft_accuracy": 0.555074440901, |
| "brier_vs_soft": 0.0685756870537, |
| "score_mae": null |
| }, |
| "score": { |
| "n": 400, |
| "accuracy": 0.7475, |
| "brier": 0.4320733708855446, |
| "nll": 0.7632714069127531, |
| "ece": 0.2215684588322721, |
| "soft_accuracy": 0.4143813430317915, |
| "brier_vs_soft": 0.05088239337216994, |
| "score_mae": 0.22179166365122913 |
| } |
| }, |
| "by_workflow": { |
| "agent_trace_observability": { |
| "n": 500, |
| "accuracy": 0.734, |
| "brier": 0.46541567472509543, |
| "nll": 0.8063145387748955, |
| "ece": 0.2392596678763307, |
| "soft_accuracy": 0.41216757063707904, |
| "brier_vs_soft": 0.05751229555397543, |
| "score_mae": 0.2102616243003979 |
| }, |
| "customer_service": { |
| "n": 500, |
| "accuracy": 0.764, |
| "brier": 0.40064869791215113, |
| "nll": 0.7203253705654751, |
| "ece": 0.21148549761061305, |
| "soft_accuracy": 0.46576884655309064, |
| "brier_vs_soft": 0.08191504313116811, |
| "score_mae": 0.23332170300206037 |
| } |
| }, |
| "teacher_agreement": 0.966, |
| "mean_teacher_kl": 0.002521283806788623, |
| "mean_probability_l1": 0.01163878468964715 |
| }, |
| "checks": { |
| "local_nmse": true, |
| "local_cosine": true, |
| "teacher_agreement": true, |
| "teacher_kl": true, |
| "accuracy_drop": true |
| }, |
| "accuracy_drop": -0.0010000000000000009, |
| "accepted": true |
| } |
| ], |
| "teacher_final": { |
| "n": 2000, |
| "accuracy": 0.7665, |
| "brier": 0.399591796314754, |
| "nll": 0.7068979180717072, |
| "ece": 0.2137387190950032, |
| "soft_accuracy": 0.47065861261740327, |
| "brier_vs_soft": 0.061465647879173455, |
| "score_mae": 0.24240958086979567, |
| "label": "teacher", |
| "seconds": 62.99023255499651, |
| "ms_per_case": 157.47558138749127, |
| "by_type": { |
| "choice": { |
| "n": 600, |
| "accuracy": 0.7333333333333333, |
| "brier": 0.46509578972312426, |
| "nll": 0.8650092521058057, |
| "ece": 0.2548481686896733, |
| "soft_accuracy": 0.398595875704782, |
| "brier_vs_soft": 0.082128886416041, |
| "score_mae": null |
| }, |
| "noul": { |
| "n": 600, |
| "accuracy": 0.8566666666666667, |
| "brier": 0.2872277414, |
| "nll": 0.4560772115719036, |
| "ece": 0.1920716666666667, |
| "soft_accuracy": 0.618722929173, |
| "brier_vs_soft": 0.049437733831919996, |
| "score_mae": null |
| }, |
| "score": { |
| "n": 800, |
| "accuracy": 0.72375, |
| "brier": 0.43473684244454175, |
| "nll": 0.7764299474209859, |
| "ece": 0.199926421220253, |
| "soft_accuracy": 0.4136574278851717, |
| "brier_vs_soft": 0.05498915451196287, |
| "score_mae": 0.24240958086979567 |
| } |
| }, |
| "by_workflow": { |
| "agent_trace_observability": { |
| "n": 500, |
| "accuracy": 0.73, |
| "brier": 0.46119840934084355, |
| "nll": 0.8008350310185499, |
| "ece": 0.22930310799388903, |
| "soft_accuracy": 0.4141887346759963, |
| "brier_vs_soft": 0.0569316048974981, |
| "score_mae": 0.21008644820503272 |
| }, |
| "customer_service": { |
| "n": 500, |
| "accuracy": 0.766, |
| "brier": 0.39565424279208455, |
| "nll": 0.7121864685594826, |
| "ece": 0.21476239227480592, |
| "soft_accuracy": 0.4685693174955372, |
| "brier_vs_soft": 0.07900562228400827, |
| "score_mae": 0.21979993301548778 |
| }, |
| "invoice_processing": { |
| "n": 500, |
| "accuracy": 0.804, |
| "brier": 0.3024242175344631, |
| "nll": 0.5564036962833907, |
| "ece": 0.16495172278038128, |
| "soft_accuracy": 0.5735877318126, |
| "brier_vs_soft": 0.06397238746428649, |
| "score_mae": 0.29189428531304207 |
| }, |
| "security_incidents": { |
| "n": 500, |
| "accuracy": 0.766, |
| "brier": 0.43909031559162476, |
| "nll": 0.7581664764254056, |
| "ece": 0.24889776943293365, |
| "soft_accuracy": 0.4262886664854796, |
| "brier_vs_soft": 0.045952976870900976, |
| "score_mae": 0.24785765694562018 |
| } |
| } |
| }, |
| "student_final": { |
| "n": 2000, |
| "accuracy": 0.7575, |
| "brier": 0.40687765482004096, |
| "nll": 0.7185002945841144, |
| "ece": 0.21033691615516764, |
| "soft_accuracy": 0.4667292438683568, |
| "brier_vs_soft": 0.06450369549417787, |
| "score_mae": 0.25252500871759453, |
| "label": "integrated_memory_v22", |
| "seconds": 76.24023421799393, |
| "ms_per_case": 190.60058554498482, |
| "by_type": { |
| "choice": { |
| "n": 600, |
| "accuracy": 0.7233333333333334, |
| "brier": 0.47158730978370766, |
| "nll": 0.8749608360461051, |
| "ece": 0.24948156287451229, |
| "soft_accuracy": 0.3950037715292289, |
| "brier_vs_soft": 0.0853422022728963, |
| "score_mae": null |
| }, |
| "noul": { |
| "n": 600, |
| "accuracy": 0.8433333333333334, |
| "brier": 0.29105601643333334, |
| "nll": 0.46114644940359806, |
| "ece": 0.18118383333333332, |
| "soft_accuracy": 0.6162034307686667, |
| "brier_vs_soft": 0.051389339007253336, |
| "score_mae": null |
| }, |
| "score": { |
| "n": 800, |
| "accuracy": 0.71875, |
| "brier": 0.4452116423873217, |
| "nll": 0.7941702723730087, |
| "ece": 0.20284324323203495, |
| "soft_accuracy": 0.40841770794747034, |
| "brier_vs_soft": 0.05871058277533245, |
| "score_mae": 0.25252500871759453 |
| } |
| }, |
| "by_workflow": { |
| "agent_trace_observability": { |
| "n": 500, |
| "accuracy": 0.734, |
| "brier": 0.46541567472509543, |
| "nll": 0.8063145387748955, |
| "ece": 0.2392596678763307, |
| "soft_accuracy": 0.41216757063707904, |
| "brier_vs_soft": 0.05751229555397543, |
| "score_mae": 0.2102616243003979 |
| }, |
| "customer_service": { |
| "n": 500, |
| "accuracy": 0.764, |
| "brier": 0.40064869791215113, |
| "nll": 0.7203253705654751, |
| "ece": 0.21148549761061305, |
| "soft_accuracy": 0.46576884655309064, |
| "brier_vs_soft": 0.08191504313116811, |
| "score_mae": 0.23332170300206037 |
| }, |
| "invoice_processing": { |
| "n": 500, |
| "accuracy": 0.786, |
| "brier": 0.3159522117429286, |
| "nll": 0.5780789912403593, |
| "ece": 0.1566913383592634, |
| "soft_accuracy": 0.5651686292940649, |
| "brier_vs_soft": 0.07064340927194246, |
| "score_mae": 0.311955511641426 |
| }, |
| "security_incidents": { |
| "n": 500, |
| "accuracy": 0.746, |
| "brier": 0.4454940348999888, |
| "nll": 0.769282277755728, |
| "ece": 0.23391116077446364, |
| "soft_accuracy": 0.42381192898919273, |
| "brier_vs_soft": 0.04794403401962545, |
| "score_mae": 0.25456119592649395 |
| } |
| }, |
| "teacher_agreement": 0.951, |
| "mean_teacher_kl": 0.002814041964657275, |
| "mean_probability_l1": 0.01388176486059745 |
| }, |
| "latency": { |
| "teacher": { |
| "median_ms": 49.591165999572695, |
| "p95_ms": 59.87610415008932, |
| "mean_ms": 52.572452700087524, |
| "repeats": 20 |
| }, |
| "student": { |
| "median_ms": 65.16387300052884, |
| "p95_ms": 84.38417299985304, |
| "mean_ms": 66.04240354995454, |
| "repeats": 20 |
| } |
| }, |
| "demo": { |
| "teacher": { |
| "model": "laya-rl-agent", |
| "answers": { |
| "department": { |
| "type": "choice", |
| "choice": "billing", |
| "probabilities": { |
| "billing": 0.8037, |
| "technical": 0.0677, |
| "sales": 0.0608, |
| "other": 0.0679 |
| }, |
| "confidence": 0.4873, |
| "action": { |
| "act_probability": 1.0 |
| } |
| }, |
| "urgency": { |
| "type": "score", |
| "score": 1.6379, |
| "legend": { |
| "0": "not urgent", |
| "1": "soon", |
| "2": "critical deadline or blocking issue" |
| }, |
| "probabilities": { |
| "0": 0.044, |
| "1": 0.2742, |
| "2": 0.6819 |
| }, |
| "confidence": 0.3144, |
| "action": { |
| "act_probability": 1.0 |
| } |
| }, |
| "churn_risk": { |
| "type": "noul", |
| "noul": 0.7062, |
| "confidence": 0.7062, |
| "action": { |
| "act_probability": 1.0 |
| } |
| }, |
| "refund_requested": { |
| "type": "noul", |
| "noul": 0.6678, |
| "confidence": 0.6678, |
| "action": { |
| "act_probability": 1.0 |
| } |
| } |
| }, |
| "usage": { |
| "input_tokens": 348, |
| "output_tokens": 0 |
| } |
| }, |
| "student": { |
| "model": "laya-rl-agent", |
| "answers": { |
| "department": { |
| "type": "choice", |
| "choice": "billing", |
| "probabilities": { |
| "billing": 0.7816, |
| "technical": 0.0752, |
| "sales": 0.0752, |
| "other": 0.068 |
| }, |
| "confidence": 0.4485, |
| "action": { |
| "act_probability": 1.0 |
| } |
| }, |
| "urgency": { |
| "type": "score", |
| "score": 1.6618, |
| "legend": { |
| "0": "not urgent", |
| "1": "soon", |
| "2": "critical deadline or blocking issue" |
| }, |
| "probabilities": { |
| "0": 0.0544, |
| "1": 0.2293, |
| "2": 0.7162 |
| }, |
| "confidence": 0.3308, |
| "action": { |
| "act_probability": 1.0 |
| } |
| }, |
| "churn_risk": { |
| "type": "noul", |
| "noul": 0.7175, |
| "confidence": 0.7175, |
| "action": { |
| "act_probability": 1.0 |
| } |
| }, |
| "refund_requested": { |
| "type": "noul", |
| "noul": 0.693, |
| "confidence": 0.693, |
| "action": { |
| "act_probability": 1.0 |
| } |
| } |
| }, |
| "usage": { |
| "input_tokens": 348, |
| "output_tokens": 0 |
| } |
| } |
| }, |
| "note": "Laya uses a bidirectional ModernBERT encoder. Integrated Memory V2.2 uses learned attention transfer and tries full-attention layers first; these results are not interchangeable with causal-LM conversions.", |
| "conversion_succeeded": true, |
| "student_is_unmodified_teacher": false |
| } |
