evaluation/export_check.json

{
  "source": "results_v5/models/legalbert",
  "quantisation": {
    "weight_type": "QInt8",
    "per_channel": true,
    "reduce_range": false
  },
  "variants_tried": [
    {
      "weight_type": "QInt8",
      "per_channel": true,
      "reduce_range": false,
      "val_agreement": 0.9796,
      "mb": 35.6
    },
    {
      "weight_type": "QInt8",
      "per_channel": true,
      "reduce_range": true,
      "val_agreement": 0.9592,
      "mb": 35.6
    },
    {
      "weight_type": "QInt8",
      "per_channel": false,
      "reduce_range": false,
      "val_agreement": 0.9728,
      "mb": 35.5
    },
    {
      "weight_type": "QInt8",
      "per_channel": false,
      "reduce_range": true,
      "val_agreement": 0.9252,
      "mb": 35.5
    },
    {
      "weight_type": "QUInt8",
      "per_channel": true,
      "reduce_range": false,
      "val_agreement": 0.9592,
      "mb": 35.6
    },
    {
      "weight_type": "QUInt8",
      "per_channel": true,
      "reduce_range": true,
      "val_agreement": 0.9388,
      "mb": 35.6
    },
    {
      "weight_type": "QUInt8",
      "per_channel": false,
      "reduce_range": false,
      "val_agreement": 0.9456,
      "mb": 35.5
    },
    {
      "weight_type": "QUInt8",
      "per_channel": false,
      "reduce_range": true,
      "val_agreement": 0.9456,
      "mb": 35.5
    }
  ],
  "int8_mb": 35.6,
  "fp32_mb": 140.4,
  "agreement": 0.9848,
  "n_rows": 198,
  "disagreements": [
    {
      "text": "Our client wants to deploy a political chatbot conversing with voters to persuade them toward a specific candidate.",
      "true": "high_risk",
      "pytorch": "limited_risk",
      "onnx_int8": "prohibited"
    },
    {
      "text": "A startup is building a stadium's live biometric camera system identifying any attendee in real time for general crowd management.",
      "true": "high_risk",
      "pytorch": "minimal_risk",
      "onnx_int8": "high_risk"
    },
    {
      "text": "We are developing software analysing a city council's call-volume data to plan how many staff to put on the phones.",
      "true": "minimal_risk",
      "pytorch": "minimal_risk",
      "onnx_int8": "high_risk"
    }
  ],
  "test_accuracy_pytorch": 0.7929,
  "test_accuracy_onnx_int8": 0.7929
}