| { |
| "protocol_sha256": "23ca33b71c7723cb6c67586e7805b74a0988ca5f8b7139b637420bbb5ec89d61", |
| "comparison_sha256": "5b6144be9d2325b224acf29be30940241f3c75c2cf35b1d4d910458ce2aa580b", |
| "dns_comparison_sha256": "3978d60071fd70c519a6bc8240eaa5b043dd136a486850789cc14a3c16b75462", |
| "checks": { |
| "policy_false_alert_rate": false, |
| "policy_useful_change": true, |
| "dns_no_regression": false, |
| "development/excytin_file": true, |
| "development/excytin_parent_join": true, |
| "development/excytin_process": true, |
| "development/excytin_signin": true, |
| "development/sev_guide_detector": true, |
| "development/sev_guide_triage": true, |
| "development/sev_swarmtraces_static": true, |
| "development/sev_sysmon_native_file": true, |
| "development/sev_sysmon_native_join": true, |
| "development/sev_sysmon_native_other_effect": true, |
| "development/sev_sysmon_native_process": true, |
| "development/sev_sysmon_native_registry": true, |
| "transfer/composition_holdout": true, |
| "transfer/emotion": true, |
| "transfer/legacy_holdout": true, |
| "transfer/mmlu": true, |
| "transfer/otrf_sysmon_file": true, |
| "transfer/otrf_sysmon_join": true, |
| "transfer/otrf_sysmon_process": true, |
| "transfer/otrf_sysmon_registry": true, |
| "transfer/paws": true, |
| "transfer/qnli": true, |
| "transfer/sciq": true, |
| "transfer/sev_swarmtraces_manual_v1": true, |
| "transfer/tweet_offensive": true |
| }, |
| "passed": false, |
| "failed": [ |
| "policy_false_alert_rate", |
| "dns_no_regression" |
| ], |
| "retained_development": [ |
| { |
| "source": "excytin_file", |
| "questions": 80, |
| "parent_correct": 80, |
| "candidate_correct": 80, |
| "parent_calibrated_nll": 0.0030639977354718724, |
| "candidate_calibrated_nll": 0.0025800477441721577, |
| "parent_calibrated_brier": 5.723205099455454e-05, |
| "candidate_calibrated_brier": 4.4624375778219304e-05 |
| }, |
| { |
| "source": "excytin_parent_join", |
| "questions": 78, |
| "parent_correct": 78, |
| "candidate_correct": 78, |
| "parent_calibrated_nll": 0.01903091706838521, |
| "candidate_calibrated_nll": 0.014932533018017379, |
| "parent_calibrated_brier": 0.0013900388516520708, |
| "candidate_calibrated_brier": 0.0008621856358382366 |
| }, |
| { |
| "source": "excytin_process", |
| "questions": 120, |
| "parent_correct": 120, |
| "candidate_correct": 120, |
| "parent_calibrated_nll": 0.007587462315793864, |
| "candidate_calibrated_nll": 0.007136704365819422, |
| "parent_calibrated_brier": 0.00023474191911792246, |
| "candidate_calibrated_brier": 0.00021856930435733632 |
| }, |
| { |
| "source": "excytin_signin", |
| "questions": 120, |
| "parent_correct": 120, |
| "candidate_correct": 120, |
| "parent_calibrated_nll": 0.009411479962491658, |
| "candidate_calibrated_nll": 0.00847100086283192, |
| "parent_calibrated_brier": 0.0010010974637190823, |
| "candidate_calibrated_brier": 0.0008193865392318428 |
| }, |
| { |
| "source": "sev_guide_detector", |
| "questions": 1064, |
| "parent_correct": 638, |
| "candidate_correct": 639, |
| "parent_calibrated_nll": 0.785071927880167, |
| "candidate_calibrated_nll": 0.7966730081647482, |
| "parent_calibrated_brier": 0.46075730837789786, |
| "candidate_calibrated_brier": 0.46843654552918773 |
| }, |
| { |
| "source": "sev_guide_triage", |
| "questions": 231, |
| "parent_correct": 130, |
| "candidate_correct": 131, |
| "parent_calibrated_nll": 0.7793526965750157, |
| "candidate_calibrated_nll": 0.7761249864817235, |
| "parent_calibrated_brier": 0.4718856823661165, |
| "candidate_calibrated_brier": 0.46407372894941223 |
| }, |
| { |
| "source": "sev_swarmtraces_static", |
| "questions": 354, |
| "parent_correct": 345, |
| "candidate_correct": 347, |
| "parent_calibrated_nll": 0.08603305952328294, |
| "candidate_calibrated_nll": 0.06588860080731399, |
| "parent_calibrated_brier": 0.03970483443698841, |
| "candidate_calibrated_brier": 0.028907916842541088 |
| }, |
| { |
| "source": "sev_sysmon_native_file", |
| "questions": 62, |
| "parent_correct": 62, |
| "candidate_correct": 62, |
| "parent_calibrated_nll": 0.04080521144437412, |
| "candidate_calibrated_nll": 0.03509983435601263, |
| "parent_calibrated_brier": 0.008675036111100168, |
| "candidate_calibrated_brier": 0.007419485561600889 |
| }, |
| { |
| "source": "sev_sysmon_native_join", |
| "questions": 120, |
| "parent_correct": 120, |
| "candidate_correct": 120, |
| "parent_calibrated_nll": 0.013824573239669817, |
| "candidate_calibrated_nll": 0.009879536093519296, |
| "parent_calibrated_brier": 0.00044603592410257366, |
| "candidate_calibrated_brier": 0.00022722202758135087 |
| }, |
| { |
| "source": "sev_sysmon_native_other_effect", |
| "questions": 88, |
| "parent_correct": 88, |
| "candidate_correct": 88, |
| "parent_calibrated_nll": 0.03669008514897713, |
| "candidate_calibrated_nll": 0.02789592785139411, |
| "parent_calibrated_brier": 0.006053607090618847, |
| "candidate_calibrated_brier": 0.004783993285232044 |
| }, |
| { |
| "source": "sev_sysmon_native_process", |
| "questions": 74, |
| "parent_correct": 74, |
| "candidate_correct": 74, |
| "parent_calibrated_nll": 0.02430026482825407, |
| "candidate_calibrated_nll": 0.023528453108139967, |
| "parent_calibrated_brier": 0.004038757822471681, |
| "candidate_calibrated_brier": 0.004027925361032504 |
| }, |
| { |
| "source": "sev_sysmon_native_registry", |
| "questions": 66, |
| "parent_correct": 65, |
| "candidate_correct": 65, |
| "parent_calibrated_nll": 0.03908316195438624, |
| "candidate_calibrated_nll": 0.03582391955947498, |
| "parent_calibrated_brier": 0.01748193378444616, |
| "candidate_calibrated_brier": 0.016990289728223678 |
| } |
| ], |
| "training_metrics": { |
| "wall_seconds": 887.5156674385071, |
| "records_seen": 6577, |
| "requested_records": 6577, |
| "truncated_records": 0, |
| "rejected_records": 0, |
| "optimizer_steps": 823, |
| "forward_tokens": 1505379, |
| "peak_device_bytes": 22225622528, |
| "device": "cuda", |
| "dtype": "bf16", |
| "batch": 4, |
| "peak_rss_bytes": 31556661248 |
| }, |
| "candidate_checkpoint": { |
| "requested": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint", |
| "resolved": "/runs/sev-r2-response-policy-4b-v2/00-trial-0/checkpoint", |
| "head_sha256": "506dcd3587847917a288d95757c497c15a2aef8e2d4bc6523b04ad991c961a2b", |
| "adapter_sha256": "fa1916b34fe1eeb10d15c36927c225df43115407d7c97b4f879bba3102c3ed21", |
| "inference_temperature": 1.0 |
| }, |
| "candidate_temperature_fit": { |
| "temperature": 1.6817928305074288, |
| "aggregation": "micro", |
| "objective": "raw-logit NLL", |
| "split": "calibration", |
| "rows_sha256": "026b68705b7ac76bc4361bc2fda1cbfe74265b436a807ea68bca77f87d3cf9da", |
| "suite_sha256": "3b9a80dfd212b6932b66ea520c35ab1a3b19e132a81af1b6b7b306d776a70a02", |
| "n": 2674 |
| }, |
| "fit_recomputed": true, |
| "run_seconds": 1285.6242614819998, |
| "decision": "Keep checkpoint as a conditional-policy research result; do not replace published model. Fixed false-alert gate and DNS regression gate fail. No threshold retuning on development." |
| } |
|
|
