| { |
| "split": "validation", |
| "fix_windows": 3060, |
| "background_windows": 10000, |
| "graphs": { |
| "model.onnx": { |
| "timing": { |
| "provider": "CUDAExecutionProvider", |
| "seconds": 589.2025489807129, |
| "windows": 13060, |
| "tokens": 23609014 |
| }, |
| "metrics": { |
| "auroc": 0.5831678804605528, |
| "pre_above_post": 0.668, |
| "auroc_vs_background": 0.9367932826047658, |
| "fixed_vs_background": 0.899418856843914, |
| "ordinary_flagged": 0.007495139794538148, |
| "per_label": { |
| "cwe_79": { |
| "auroc": 0.5943831094923743, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9208015706806284 |
| }, |
| "cwe_787": { |
| "auroc": 0.514030612244898, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9300489699425869 |
| }, |
| "cwe_89": { |
| "auroc": 0.6052631578947368, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9858538485427489 |
| }, |
| "cwe_352": { |
| "auroc": 0.686804451510334, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9919243243243243 |
| }, |
| "cwe_22": { |
| "auroc": 0.6477427671864542, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9307166301838322 |
| }, |
| "cwe_125": { |
| "auroc": 0.5975274725274725, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.8971631205673758 |
| }, |
| "cwe_78": { |
| "auroc": 0.5625, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.8711819428181397 |
| }, |
| "cwe_416": { |
| "auroc": 0.4666666666666666, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9810874704491725 |
| }, |
| "cwe_862": { |
| "auroc": 0.6651340996168582, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9461515117028944 |
| }, |
| "cwe_434": { |
| "auroc": 0.5784615384615385, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9945794579457945 |
| }, |
| "cwe_94": { |
| "auroc": 0.5349768875192604, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.895850110965493 |
| }, |
| "cwe_20": { |
| "auroc": 0.5329481940726128, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9524488872728332 |
| }, |
| "cwe_77": { |
| "auroc": 0.6023809523809524, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.962347619047619 |
| }, |
| "cwe_287": { |
| "auroc": 0.5519266461674839, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9576837264878321 |
| }, |
| "cwe_269": { |
| "auroc": 0.5869140625, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.93596875 |
| }, |
| "cwe_502": { |
| "auroc": 0.626219512195122, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9243799979508097 |
| }, |
| "cwe_200": { |
| "auroc": 0.584201982228298, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9678985022592503 |
| }, |
| "cwe_863": { |
| "auroc": 0.5786790780141844, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9332364583333334 |
| }, |
| "cwe_918": { |
| "auroc": 0.5463917525773195, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9270726622037488 |
| }, |
| "cwe_119": { |
| "auroc": 0.5555555555555556, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9072760703966378 |
| }, |
| "cwe_476": { |
| "auroc": 0.5833333333333333, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9756195462478184 |
| }, |
| "cwe_798": { |
| "auroc": 0.615079365079365, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9451111777844451 |
| }, |
| "cwe_190": { |
| "auroc": 0.588235294117647, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9148750821827746 |
| }, |
| "cwe_400": { |
| "auroc": 0.6270021191661328, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9346607964778867 |
| }, |
| "cwe_306": { |
| "auroc": 0.593692022263451, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.8543763467255817 |
| } |
| } |
| }, |
| "drift_vs_pytorch_bf16": { |
| "cells": 209896, |
| "mean_abs_dp": 0.00034063943894580007, |
| "p99_abs_dp": 0.005703164730221033, |
| "max_abs_dp": 0.060594260692596436, |
| "spearman": 0.9999457143488498, |
| "top500_overlap": 0.998, |
| "flag_agreement": 0.9997903723748904 |
| } |
| }, |
| "model_q8.onnx": { |
| "timing": { |
| "provider": "CUDAExecutionProvider", |
| "seconds": 606.5846574306488, |
| "windows": 13060, |
| "tokens": 23609014 |
| }, |
| "metrics": { |
| "auroc": 0.5830152950187844, |
| "pre_above_post": 0.6691428571428572, |
| "auroc_vs_background": 0.9370235201270853, |
| "fixed_vs_background": 0.8998818862870871, |
| "ordinary_flagged": 0.007611493840507692, |
| "per_label": { |
| "cwe_79": { |
| "auroc": 0.5948952879581152, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9212685863874346 |
| }, |
| "cwe_787": { |
| "auroc": 0.5137117346938775, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9297323539344816 |
| }, |
| "cwe_89": { |
| "auroc": 0.6097744360902255, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9856511966986172 |
| }, |
| "cwe_352": { |
| "auroc": 0.6883942766295709, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9917486486486486 |
| }, |
| "cwe_22": { |
| "auroc": 0.6476931849170737, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9309643206024291 |
| }, |
| "cwe_125": { |
| "auroc": 0.5975274725274725, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.89661431948666 |
| }, |
| "cwe_78": { |
| "auroc": 0.5625, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.8711407539909033 |
| }, |
| "cwe_416": { |
| "auroc": 0.4666666666666666, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9812844759653271 |
| }, |
| "cwe_862": { |
| "auroc": 0.6655172413793103, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9464498173955327 |
| }, |
| "cwe_434": { |
| "auroc": 0.5784615384615385, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9946034603460346 |
| }, |
| "cwe_94": { |
| "auroc": 0.5328197226502311, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.8970097809003719 |
| }, |
| "cwe_20": { |
| "auroc": 0.5325254802498708, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9530237791991119 |
| }, |
| "cwe_77": { |
| "auroc": 0.6023809523809524, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9622761904761904 |
| }, |
| "cwe_287": { |
| "auroc": 0.5516773317820438, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9577764059128426 |
| }, |
| "cwe_269": { |
| "auroc": 0.583984375, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9359062499999999 |
| }, |
| "cwe_502": { |
| "auroc": 0.626219512195122, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9249288882166677 |
| }, |
| "cwe_200": { |
| "auroc": 0.5835611756664388, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9678353064470262 |
| }, |
| "cwe_863": { |
| "auroc": 0.5792331560283688, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9332989583333333 |
| }, |
| "cwe_918": { |
| "auroc": 0.5458826524118621, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9271448777030521 |
| }, |
| "cwe_119": { |
| "auroc": 0.5500685871056241, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9070571753786884 |
| }, |
| "cwe_476": { |
| "auroc": 0.5833333333333333, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9750698080279232 |
| }, |
| "cwe_798": { |
| "auroc": 0.6150793650793651, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9461112777944461 |
| }, |
| "cwe_190": { |
| "auroc": 0.588235294117647, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.915499671268902 |
| }, |
| "cwe_400": { |
| "auroc": 0.6271253264994332, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9349245469611863 |
| }, |
| "cwe_306": { |
| "auroc": 0.5940630797773656, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.855201883824746 |
| } |
| } |
| }, |
| "drift_vs_pytorch_bf16": { |
| "cells": 209896, |
| "mean_abs_dp": 0.0006609148113057017, |
| "p99_abs_dp": 0.010950522497296333, |
| "max_abs_dp": 0.08087354898452759, |
| "spearman": 0.9998371762592646, |
| "top500_overlap": 0.984, |
| "flag_agreement": 0.9996140945992301 |
| }, |
| "drift_vs_fp32": { |
| "cells": 209896, |
| "mean_abs_dp": 0.0005471020704135299, |
| "p99_abs_dp": 0.009149489924311638, |
| "max_abs_dp": 0.055769503116607666, |
| "spearman": 0.9998907859035402, |
| "top500_overlap": 0.984, |
| "flag_agreement": 0.9996903228265427 |
| } |
| }, |
| "model_q8_wasm.onnx": { |
| "timing": { |
| "provider": "CUDAExecutionProvider", |
| "seconds": 662.4384205341339, |
| "windows": 13060, |
| "tokens": 23609014 |
| }, |
| "metrics": { |
| "auroc": 0.5830254775134045, |
| "pre_above_post": 0.6674285714285715, |
| "auroc_vs_background": 0.9370247553098354, |
| "fixed_vs_background": 0.8998820382671503, |
| "ordinary_flagged": 0.007611493840507692, |
| "per_label": { |
| "cwe_79": { |
| "auroc": 0.5948952879581152, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9212701570680628 |
| }, |
| "cwe_787": { |
| "auroc": 0.5137117346938775, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9297323539344816 |
| }, |
| "cwe_89": { |
| "auroc": 0.6097744360902255, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9856433011722225 |
| }, |
| "cwe_352": { |
| "auroc": 0.6883942766295709, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9917594594594594 |
| }, |
| "cwe_22": { |
| "auroc": 0.6477179760517638, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9309739024368042 |
| }, |
| "cwe_125": { |
| "auroc": 0.5975274725274725, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.8966987504221547 |
| }, |
| "cwe_78": { |
| "auroc": 0.5615808823529412, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.871120159577285 |
| }, |
| "cwe_416": { |
| "auroc": 0.4666666666666666, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9810874704491725 |
| }, |
| "cwe_862": { |
| "auroc": 0.6651340996168582, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9464360229126362 |
| }, |
| "cwe_434": { |
| "auroc": 0.5784615384615385, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9945954595459546 |
| }, |
| "cwe_94": { |
| "auroc": 0.5328197226502311, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.8970284305630672 |
| }, |
| "cwe_20": { |
| "auroc": 0.5327133530599784, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9530277531064364 |
| }, |
| "cwe_77": { |
| "auroc": 0.6023809523809524, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9622857142857142 |
| }, |
| "cwe_287": { |
| "auroc": 0.5517604365771905, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9577868781642563 |
| }, |
| "cwe_269": { |
| "auroc": 0.583984375, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.93591875 |
| }, |
| "cwe_502": { |
| "auroc": 0.6268292682926828, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9249581623641802 |
| }, |
| "cwe_200": { |
| "auroc": 0.5834757347915243, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9678359647367368 |
| }, |
| "cwe_863": { |
| "auroc": 0.579011524822695, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.93331875 |
| }, |
| "cwe_918": { |
| "auroc": 0.5460099274532264, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9271077383034103 |
| }, |
| "cwe_119": { |
| "auroc": 0.5500685871056241, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9069696173715086 |
| }, |
| "cwe_476": { |
| "auroc": 0.5833333333333333, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9750785340314136 |
| }, |
| "cwe_798": { |
| "auroc": 0.6130952380952381, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.946094609460946 |
| }, |
| "cwe_190": { |
| "auroc": 0.588235294117647, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9155818540433924 |
| }, |
| "cwe_400": { |
| "auroc": 0.627051402099453, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9349250326895167 |
| }, |
| "cwe_306": { |
| "auroc": 0.5936920222634509, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.855158243097037 |
| } |
| } |
| }, |
| "drift_vs_pytorch_bf16": { |
| "cells": 209896, |
| "mean_abs_dp": 0.0006600198103114963, |
| "p99_abs_dp": 0.010931852273643017, |
| "max_abs_dp": 0.07931667566299438, |
| "spearman": 0.9998372665872358, |
| "top500_overlap": 0.984, |
| "flag_agreement": 0.9996093303350231 |
| }, |
| "drift_vs_fp32": { |
| "cells": 209896, |
| "mean_abs_dp": 0.0005467346054501832, |
| "p99_abs_dp": 0.009161485359072685, |
| "max_abs_dp": 0.05553334951400757, |
| "spearman": 0.9998910215686962, |
| "top500_overlap": 0.984, |
| "flag_agreement": 0.9996855585623357 |
| } |
| } |
| }, |
| "pytorch_bf16": { |
| "metrics": { |
| "auroc": 0.5829757808605576, |
| "pre_above_post": 0.656, |
| "auroc_vs_background": 0.9369072246717409, |
| "fixed_vs_background": 0.8995859071877126, |
| "ordinary_flagged": 0.007490291709289416, |
| "per_label": { |
| "cwe_79": { |
| "auroc": 0.5942977464147507, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9207507853403142 |
| }, |
| "cwe_787": { |
| "auroc": 0.5133928571428572, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9303655859506924 |
| }, |
| "cwe_89": { |
| "auroc": 0.6030075187969924, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9857380474889594 |
| }, |
| "cwe_352": { |
| "auroc": 0.686804451510334, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9919216216216217 |
| }, |
| "cwe_22": { |
| "auroc": 0.6474948558395518, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9307908894002394 |
| }, |
| "cwe_125": { |
| "auroc": 0.5947802197802198, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.897078689631881 |
| }, |
| "cwe_78": { |
| "auroc": 0.5615808823529411, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.8717497602221841 |
| }, |
| "cwe_416": { |
| "auroc": 0.4666666666666666, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9808904649330181 |
| }, |
| "cwe_862": { |
| "auroc": 0.6647509578544061, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.946167030496153 |
| }, |
| "cwe_434": { |
| "auroc": 0.5815384615384616, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9945794579457947 |
| }, |
| "cwe_94": { |
| "auroc": 0.5322033898305085, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.896212931676113 |
| }, |
| "cwe_20": { |
| "auroc": 0.5321027664271287, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9526475826390586 |
| }, |
| "cwe_77": { |
| "auroc": 0.6047619047619047, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9622666666666667 |
| }, |
| "cwe_287": { |
| "auroc": 0.5508185822321947, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9578093935047954 |
| }, |
| "cwe_269": { |
| "auroc": 0.583984375, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.93609375 |
| }, |
| "cwe_502": { |
| "auroc": 0.6292682926829267, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9242067925780277 |
| }, |
| "cwe_200": { |
| "auroc": 0.5839456596035544, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9679794718936626 |
| }, |
| "cwe_863": { |
| "auroc": 0.5787898936170213, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9333697916666667 |
| }, |
| "cwe_918": { |
| "auroc": 0.547537227949599, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9274120750504734 |
| }, |
| "cwe_119": { |
| "auroc": 0.5500685871056241, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.907932755450486 |
| }, |
| "cwe_476": { |
| "auroc": 0.5902777777777778, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9754712041884817 |
| }, |
| "cwe_798": { |
| "auroc": 0.6111111111111112, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9454320432043204 |
| }, |
| "cwe_190": { |
| "auroc": 0.5816993464052288, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.915318869165023 |
| }, |
| "cwe_400": { |
| "auroc": 0.6267557044995319, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.9346238811247912 |
| }, |
| "cwe_306": { |
| "auroc": 0.5929499072356216, |
| "pre_above_post": null, |
| "auroc_vs_background": 0.854812754002673 |
| } |
| } |
| } |
| } |
| } |
|
|
