File size: 2,041 Bytes
7c594f6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | {
"source": "results_v6/models/legalbert",
"quantisation": {
"weight_type": "QUInt8",
"per_channel": false,
"reduce_range": false
},
"variants_tried": [
{
"weight_type": "QInt8",
"per_channel": true,
"reduce_range": false,
"val_agreement": 0.9506,
"mb": 35.6
},
{
"weight_type": "QInt8",
"per_channel": true,
"reduce_range": true,
"val_agreement": 0.9753,
"mb": 35.6
},
{
"weight_type": "QInt8",
"per_channel": false,
"reduce_range": false,
"val_agreement": 0.963,
"mb": 35.5
},
{
"weight_type": "QInt8",
"per_channel": false,
"reduce_range": true,
"val_agreement": 0.9506,
"mb": 35.5
},
{
"weight_type": "QUInt8",
"per_channel": true,
"reduce_range": false,
"val_agreement": 0.9753,
"mb": 35.6
},
{
"weight_type": "QUInt8",
"per_channel": true,
"reduce_range": true,
"val_agreement": 0.9691,
"mb": 35.6
},
{
"weight_type": "QUInt8",
"per_channel": false,
"reduce_range": false,
"val_agreement": 0.9753,
"mb": 35.5
},
{
"weight_type": "QUInt8",
"per_channel": false,
"reduce_range": true,
"val_agreement": 0.9568,
"mb": 35.5
}
],
"int8_mb": 35.5,
"fp32_mb": 140.4,
"agreement": 0.9905,
"n_rows": 210,
"disagreements": [
{
"text": "Proposed use case: a financial newsletter auto-generating market commentary articles published directly to subscribers.",
"true": "limited_risk",
"pytorch": "limited_risk",
"onnx_int8": "minimal_risk"
},
{
"text": "Our client wants to deploy a language-learning app's AI that decides a learner's official proficiency level, steering their curriculum path.",
"true": "high_risk",
"pytorch": "minimal_risk",
"onnx_int8": "high_risk"
}
],
"test_accuracy_pytorch": 0.8333,
"test_accuracy_onnx_int8": 0.8333
} |