{
  "split": "validation",
  "fix_windows": 3060,
  "background_windows": 10000,
  "graphs": {
    "model.onnx": {
      "timing": {
        "provider": "CUDAExecutionProvider",
        "seconds": 589.2025489807129,
        "windows": 13060,
        "tokens": 23609014
      },
      "metrics": {
        "auroc": 0.5831678804605528,
        "pre_above_post": 0.668,
        "auroc_vs_background": 0.9367932826047658,
        "fixed_vs_background": 0.899418856843914,
        "ordinary_flagged": 0.007495139794538148,
        "per_label": {
          "cwe_79": {
            "auroc": 0.5943831094923743,
            "pre_above_post": null,
            "auroc_vs_background": 0.9208015706806284
          },
          "cwe_787": {
            "auroc": 0.514030612244898,
            "pre_above_post": null,
            "auroc_vs_background": 0.9300489699425869
          },
          "cwe_89": {
            "auroc": 0.6052631578947368,
            "pre_above_post": null,
            "auroc_vs_background": 0.9858538485427489
          },
          "cwe_352": {
            "auroc": 0.686804451510334,
            "pre_above_post": null,
            "auroc_vs_background": 0.9919243243243243
          },
          "cwe_22": {
            "auroc": 0.6477427671864542,
            "pre_above_post": null,
            "auroc_vs_background": 0.9307166301838322
          },
          "cwe_125": {
            "auroc": 0.5975274725274725,
            "pre_above_post": null,
            "auroc_vs_background": 0.8971631205673758
          },
          "cwe_78": {
            "auroc": 0.5625,
            "pre_above_post": null,
            "auroc_vs_background": 0.8711819428181397
          },
          "cwe_416": {
            "auroc": 0.4666666666666666,
            "pre_above_post": null,
            "auroc_vs_background": 0.9810874704491725
          },
          "cwe_862": {
            "auroc": 0.6651340996168582,
            "pre_above_post": null,
            "auroc_vs_background": 0.9461515117028944
          },
          "cwe_434": {
            "auroc": 0.5784615384615385,
            "pre_above_post": null,
            "auroc_vs_background": 0.9945794579457945
          },
          "cwe_94": {
            "auroc": 0.5349768875192604,
            "pre_above_post": null,
            "auroc_vs_background": 0.895850110965493
          },
          "cwe_20": {
            "auroc": 0.5329481940726128,
            "pre_above_post": null,
            "auroc_vs_background": 0.9524488872728332
          },
          "cwe_77": {
            "auroc": 0.6023809523809524,
            "pre_above_post": null,
            "auroc_vs_background": 0.962347619047619
          },
          "cwe_287": {
            "auroc": 0.5519266461674839,
            "pre_above_post": null,
            "auroc_vs_background": 0.9576837264878321
          },
          "cwe_269": {
            "auroc": 0.5869140625,
            "pre_above_post": null,
            "auroc_vs_background": 0.93596875
          },
          "cwe_502": {
            "auroc": 0.626219512195122,
            "pre_above_post": null,
            "auroc_vs_background": 0.9243799979508097
          },
          "cwe_200": {
            "auroc": 0.584201982228298,
            "pre_above_post": null,
            "auroc_vs_background": 0.9678985022592503
          },
          "cwe_863": {
            "auroc": 0.5786790780141844,
            "pre_above_post": null,
            "auroc_vs_background": 0.9332364583333334
          },
          "cwe_918": {
            "auroc": 0.5463917525773195,
            "pre_above_post": null,
            "auroc_vs_background": 0.9270726622037488
          },
          "cwe_119": {
            "auroc": 0.5555555555555556,
            "pre_above_post": null,
            "auroc_vs_background": 0.9072760703966378
          },
          "cwe_476": {
            "auroc": 0.5833333333333333,
            "pre_above_post": null,
            "auroc_vs_background": 0.9756195462478184
          },
          "cwe_798": {
            "auroc": 0.615079365079365,
            "pre_above_post": null,
            "auroc_vs_background": 0.9451111777844451
          },
          "cwe_190": {
            "auroc": 0.588235294117647,
            "pre_above_post": null,
            "auroc_vs_background": 0.9148750821827746
          },
          "cwe_400": {
            "auroc": 0.6270021191661328,
            "pre_above_post": null,
            "auroc_vs_background": 0.9346607964778867
          },
          "cwe_306": {
            "auroc": 0.593692022263451,
            "pre_above_post": null,
            "auroc_vs_background": 0.8543763467255817
          }
        }
      },
      "drift_vs_pytorch_bf16": {
        "cells": 209896,
        "mean_abs_dp": 0.00034063943894580007,
        "p99_abs_dp": 0.005703164730221033,
        "max_abs_dp": 0.060594260692596436,
        "spearman": 0.9999457143488498,
        "top500_overlap": 0.998,
        "flag_agreement": 0.9997903723748904
      }
    },
    "model_q8.onnx": {
      "timing": {
        "provider": "CUDAExecutionProvider",
        "seconds": 606.5846574306488,
        "windows": 13060,
        "tokens": 23609014
      },
      "metrics": {
        "auroc": 0.5830152950187844,
        "pre_above_post": 0.6691428571428572,
        "auroc_vs_background": 0.9370235201270853,
        "fixed_vs_background": 0.8998818862870871,
        "ordinary_flagged": 0.007611493840507692,
        "per_label": {
          "cwe_79": {
            "auroc": 0.5948952879581152,
            "pre_above_post": null,
            "auroc_vs_background": 0.9212685863874346
          },
          "cwe_787": {
            "auroc": 0.5137117346938775,
            "pre_above_post": null,
            "auroc_vs_background": 0.9297323539344816
          },
          "cwe_89": {
            "auroc": 0.6097744360902255,
            "pre_above_post": null,
            "auroc_vs_background": 0.9856511966986172
          },
          "cwe_352": {
            "auroc": 0.6883942766295709,
            "pre_above_post": null,
            "auroc_vs_background": 0.9917486486486486
          },
          "cwe_22": {
            "auroc": 0.6476931849170737,
            "pre_above_post": null,
            "auroc_vs_background": 0.9309643206024291
          },
          "cwe_125": {
            "auroc": 0.5975274725274725,
            "pre_above_post": null,
            "auroc_vs_background": 0.89661431948666
          },
          "cwe_78": {
            "auroc": 0.5625,
            "pre_above_post": null,
            "auroc_vs_background": 0.8711407539909033
          },
          "cwe_416": {
            "auroc": 0.4666666666666666,
            "pre_above_post": null,
            "auroc_vs_background": 0.9812844759653271
          },
          "cwe_862": {
            "auroc": 0.6655172413793103,
            "pre_above_post": null,
            "auroc_vs_background": 0.9464498173955327
          },
          "cwe_434": {
            "auroc": 0.5784615384615385,
            "pre_above_post": null,
            "auroc_vs_background": 0.9946034603460346
          },
          "cwe_94": {
            "auroc": 0.5328197226502311,
            "pre_above_post": null,
            "auroc_vs_background": 0.8970097809003719
          },
          "cwe_20": {
            "auroc": 0.5325254802498708,
            "pre_above_post": null,
            "auroc_vs_background": 0.9530237791991119
          },
          "cwe_77": {
            "auroc": 0.6023809523809524,
            "pre_above_post": null,
            "auroc_vs_background": 0.9622761904761904
          },
          "cwe_287": {
            "auroc": 0.5516773317820438,
            "pre_above_post": null,
            "auroc_vs_background": 0.9577764059128426
          },
          "cwe_269": {
            "auroc": 0.583984375,
            "pre_above_post": null,
            "auroc_vs_background": 0.9359062499999999
          },
          "cwe_502": {
            "auroc": 0.626219512195122,
            "pre_above_post": null,
            "auroc_vs_background": 0.9249288882166677
          },
          "cwe_200": {
            "auroc": 0.5835611756664388,
            "pre_above_post": null,
            "auroc_vs_background": 0.9678353064470262
          },
          "cwe_863": {
            "auroc": 0.5792331560283688,
            "pre_above_post": null,
            "auroc_vs_background": 0.9332989583333333
          },
          "cwe_918": {
            "auroc": 0.5458826524118621,
            "pre_above_post": null,
            "auroc_vs_background": 0.9271448777030521
          },
          "cwe_119": {
            "auroc": 0.5500685871056241,
            "pre_above_post": null,
            "auroc_vs_background": 0.9070571753786884
          },
          "cwe_476": {
            "auroc": 0.5833333333333333,
            "pre_above_post": null,
            "auroc_vs_background": 0.9750698080279232
          },
          "cwe_798": {
            "auroc": 0.6150793650793651,
            "pre_above_post": null,
            "auroc_vs_background": 0.9461112777944461
          },
          "cwe_190": {
            "auroc": 0.588235294117647,
            "pre_above_post": null,
            "auroc_vs_background": 0.915499671268902
          },
          "cwe_400": {
            "auroc": 0.6271253264994332,
            "pre_above_post": null,
            "auroc_vs_background": 0.9349245469611863
          },
          "cwe_306": {
            "auroc": 0.5940630797773656,
            "pre_above_post": null,
            "auroc_vs_background": 0.855201883824746
          }
        }
      },
      "drift_vs_pytorch_bf16": {
        "cells": 209896,
        "mean_abs_dp": 0.0006609148113057017,
        "p99_abs_dp": 0.010950522497296333,
        "max_abs_dp": 0.08087354898452759,
        "spearman": 0.9998371762592646,
        "top500_overlap": 0.984,
        "flag_agreement": 0.9996140945992301
      },
      "drift_vs_fp32": {
        "cells": 209896,
        "mean_abs_dp": 0.0005471020704135299,
        "p99_abs_dp": 0.009149489924311638,
        "max_abs_dp": 0.055769503116607666,
        "spearman": 0.9998907859035402,
        "top500_overlap": 0.984,
        "flag_agreement": 0.9996903228265427
      }
    },
    "model_q8_wasm.onnx": {
      "timing": {
        "provider": "CUDAExecutionProvider",
        "seconds": 662.4384205341339,
        "windows": 13060,
        "tokens": 23609014
      },
      "metrics": {
        "auroc": 0.5830254775134045,
        "pre_above_post": 0.6674285714285715,
        "auroc_vs_background": 0.9370247553098354,
        "fixed_vs_background": 0.8998820382671503,
        "ordinary_flagged": 0.007611493840507692,
        "per_label": {
          "cwe_79": {
            "auroc": 0.5948952879581152,
            "pre_above_post": null,
            "auroc_vs_background": 0.9212701570680628
          },
          "cwe_787": {
            "auroc": 0.5137117346938775,
            "pre_above_post": null,
            "auroc_vs_background": 0.9297323539344816
          },
          "cwe_89": {
            "auroc": 0.6097744360902255,
            "pre_above_post": null,
            "auroc_vs_background": 0.9856433011722225
          },
          "cwe_352": {
            "auroc": 0.6883942766295709,
            "pre_above_post": null,
            "auroc_vs_background": 0.9917594594594594
          },
          "cwe_22": {
            "auroc": 0.6477179760517638,
            "pre_above_post": null,
            "auroc_vs_background": 0.9309739024368042
          },
          "cwe_125": {
            "auroc": 0.5975274725274725,
            "pre_above_post": null,
            "auroc_vs_background": 0.8966987504221547
          },
          "cwe_78": {
            "auroc": 0.5615808823529412,
            "pre_above_post": null,
            "auroc_vs_background": 0.871120159577285
          },
          "cwe_416": {
            "auroc": 0.4666666666666666,
            "pre_above_post": null,
            "auroc_vs_background": 0.9810874704491725
          },
          "cwe_862": {
            "auroc": 0.6651340996168582,
            "pre_above_post": null,
            "auroc_vs_background": 0.9464360229126362
          },
          "cwe_434": {
            "auroc": 0.5784615384615385,
            "pre_above_post": null,
            "auroc_vs_background": 0.9945954595459546
          },
          "cwe_94": {
            "auroc": 0.5328197226502311,
            "pre_above_post": null,
            "auroc_vs_background": 0.8970284305630672
          },
          "cwe_20": {
            "auroc": 0.5327133530599784,
            "pre_above_post": null,
            "auroc_vs_background": 0.9530277531064364
          },
          "cwe_77": {
            "auroc": 0.6023809523809524,
            "pre_above_post": null,
            "auroc_vs_background": 0.9622857142857142
          },
          "cwe_287": {
            "auroc": 0.5517604365771905,
            "pre_above_post": null,
            "auroc_vs_background": 0.9577868781642563
          },
          "cwe_269": {
            "auroc": 0.583984375,
            "pre_above_post": null,
            "auroc_vs_background": 0.93591875
          },
          "cwe_502": {
            "auroc": 0.6268292682926828,
            "pre_above_post": null,
            "auroc_vs_background": 0.9249581623641802
          },
          "cwe_200": {
            "auroc": 0.5834757347915243,
            "pre_above_post": null,
            "auroc_vs_background": 0.9678359647367368
          },
          "cwe_863": {
            "auroc": 0.579011524822695,
            "pre_above_post": null,
            "auroc_vs_background": 0.93331875
          },
          "cwe_918": {
            "auroc": 0.5460099274532264,
            "pre_above_post": null,
            "auroc_vs_background": 0.9271077383034103
          },
          "cwe_119": {
            "auroc": 0.5500685871056241,
            "pre_above_post": null,
            "auroc_vs_background": 0.9069696173715086
          },
          "cwe_476": {
            "auroc": 0.5833333333333333,
            "pre_above_post": null,
            "auroc_vs_background": 0.9750785340314136
          },
          "cwe_798": {
            "auroc": 0.6130952380952381,
            "pre_above_post": null,
            "auroc_vs_background": 0.946094609460946
          },
          "cwe_190": {
            "auroc": 0.588235294117647,
            "pre_above_post": null,
            "auroc_vs_background": 0.9155818540433924
          },
          "cwe_400": {
            "auroc": 0.627051402099453,
            "pre_above_post": null,
            "auroc_vs_background": 0.9349250326895167
          },
          "cwe_306": {
            "auroc": 0.5936920222634509,
            "pre_above_post": null,
            "auroc_vs_background": 0.855158243097037
          }
        }
      },
      "drift_vs_pytorch_bf16": {
        "cells": 209896,
        "mean_abs_dp": 0.0006600198103114963,
        "p99_abs_dp": 0.010931852273643017,
        "max_abs_dp": 0.07931667566299438,
        "spearman": 0.9998372665872358,
        "top500_overlap": 0.984,
        "flag_agreement": 0.9996093303350231
      },
      "drift_vs_fp32": {
        "cells": 209896,
        "mean_abs_dp": 0.0005467346054501832,
        "p99_abs_dp": 0.009161485359072685,
        "max_abs_dp": 0.05553334951400757,
        "spearman": 0.9998910215686962,
        "top500_overlap": 0.984,
        "flag_agreement": 0.9996855585623357
      }
    }
  },
  "pytorch_bf16": {
    "metrics": {
      "auroc": 0.5829757808605576,
      "pre_above_post": 0.656,
      "auroc_vs_background": 0.9369072246717409,
      "fixed_vs_background": 0.8995859071877126,
      "ordinary_flagged": 0.007490291709289416,
      "per_label": {
        "cwe_79": {
          "auroc": 0.5942977464147507,
          "pre_above_post": null,
          "auroc_vs_background": 0.9207507853403142
        },
        "cwe_787": {
          "auroc": 0.5133928571428572,
          "pre_above_post": null,
          "auroc_vs_background": 0.9303655859506924
        },
        "cwe_89": {
          "auroc": 0.6030075187969924,
          "pre_above_post": null,
          "auroc_vs_background": 0.9857380474889594
        },
        "cwe_352": {
          "auroc": 0.686804451510334,
          "pre_above_post": null,
          "auroc_vs_background": 0.9919216216216217
        },
        "cwe_22": {
          "auroc": 0.6474948558395518,
          "pre_above_post": null,
          "auroc_vs_background": 0.9307908894002394
        },
        "cwe_125": {
          "auroc": 0.5947802197802198,
          "pre_above_post": null,
          "auroc_vs_background": 0.897078689631881
        },
        "cwe_78": {
          "auroc": 0.5615808823529411,
          "pre_above_post": null,
          "auroc_vs_background": 0.8717497602221841
        },
        "cwe_416": {
          "auroc": 0.4666666666666666,
          "pre_above_post": null,
          "auroc_vs_background": 0.9808904649330181
        },
        "cwe_862": {
          "auroc": 0.6647509578544061,
          "pre_above_post": null,
          "auroc_vs_background": 0.946167030496153
        },
        "cwe_434": {
          "auroc": 0.5815384615384616,
          "pre_above_post": null,
          "auroc_vs_background": 0.9945794579457947
        },
        "cwe_94": {
          "auroc": 0.5322033898305085,
          "pre_above_post": null,
          "auroc_vs_background": 0.896212931676113
        },
        "cwe_20": {
          "auroc": 0.5321027664271287,
          "pre_above_post": null,
          "auroc_vs_background": 0.9526475826390586
        },
        "cwe_77": {
          "auroc": 0.6047619047619047,
          "pre_above_post": null,
          "auroc_vs_background": 0.9622666666666667
        },
        "cwe_287": {
          "auroc": 0.5508185822321947,
          "pre_above_post": null,
          "auroc_vs_background": 0.9578093935047954
        },
        "cwe_269": {
          "auroc": 0.583984375,
          "pre_above_post": null,
          "auroc_vs_background": 0.93609375
        },
        "cwe_502": {
          "auroc": 0.6292682926829267,
          "pre_above_post": null,
          "auroc_vs_background": 0.9242067925780277
        },
        "cwe_200": {
          "auroc": 0.5839456596035544,
          "pre_above_post": null,
          "auroc_vs_background": 0.9679794718936626
        },
        "cwe_863": {
          "auroc": 0.5787898936170213,
          "pre_above_post": null,
          "auroc_vs_background": 0.9333697916666667
        },
        "cwe_918": {
          "auroc": 0.547537227949599,
          "pre_above_post": null,
          "auroc_vs_background": 0.9274120750504734
        },
        "cwe_119": {
          "auroc": 0.5500685871056241,
          "pre_above_post": null,
          "auroc_vs_background": 0.907932755450486
        },
        "cwe_476": {
          "auroc": 0.5902777777777778,
          "pre_above_post": null,
          "auroc_vs_background": 0.9754712041884817
        },
        "cwe_798": {
          "auroc": 0.6111111111111112,
          "pre_above_post": null,
          "auroc_vs_background": 0.9454320432043204
        },
        "cwe_190": {
          "auroc": 0.5816993464052288,
          "pre_above_post": null,
          "auroc_vs_background": 0.915318869165023
        },
        "cwe_400": {
          "auroc": 0.6267557044995319,
          "pre_above_post": null,
          "auroc_vs_background": 0.9346238811247912
        },
        "cwe_306": {
          "auroc": 0.5929499072356216,
          "pre_above_post": null,
          "auroc_vs_background": 0.854812754002673
        }
      }
    }
  }
}