quant_benchmark.json

{
  "conversions": {
    "mnn_vocos_fp16": "/Users/abu/qefro/indicf5-benchmark/mnn/indicf5_vocos_fp16.mnn",
    "mnn_vocos_fp16_mb": 29.34,
    "mnn_vocos_int8wq": "/Users/abu/qefro/indicf5-benchmark/mnn/indicf5_vocos_int8wq.mnn",
    "mnn_vocos_int8wq_mb": 15.05,
    "mnn_vocos_fp16_int8wq": "/Users/abu/qefro/indicf5-benchmark/mnn/indicf5_vocos_fp16_int8wq.mnn",
    "mnn_vocos_fp16_int8wq_mb": 29.34,
    "mnn_dit_fp16": "/Users/abu/qefro/indicf5-benchmark/mnn/indicf5_dit_fp16.mnn",
    "mnn_dit_fp16_mb": 676.66,
    "mnn_dit_int8wq": "/Users/abu/qefro/indicf5-benchmark/mnn/indicf5_dit_int8wq.mnn",
    "mnn_dit_int8wq_mb": 346.52,
    "mnn_dit_fp16_int8wq": "/Users/abu/qefro/indicf5-benchmark/mnn/indicf5_dit_fp16_int8wq.mnn",
    "mnn_dit_fp16_int8wq_mb": 676.66,
    "litert_vocos_fp16": "/Users/abu/qefro/indicf5-benchmark/litert/vocos/indicf5_vocos_fp16.tflite",
    "litert_vocos_fp16_mb": 29.29,
    "litert_vocos_drq": "/Users/abu/qefro/indicf5-benchmark/litert/vocos_quant/indicf5_vocos_dynamic_range_quant.tflite",
    "litert_vocos_drq_mb": 57.23,
    "litert_dit_drq": null
  },
  "benchmarks": [
    {
      "name": "mnn_vocos_fp32",
      "ok": true,
      "mean_s": 0.01844218359997285,
      "std_s": 0.00017029317445372552,
      "min_s": 0.0182243340000241,
      "max_s": 0.01870145899988529,
      "out_shape": [
        1,
        1,
        32512
      ],
      "size_mb": 58.51,
      "format": "mnn"
    },
    {
      "name": "mnn_dit_fp32",
      "ok": true,
      "mean_s": 0.3114288501999454,
      "std_s": 0.03828318255141011,
      "min_s": 0.24100124999995387,
      "max_s": 0.34994941700006166,
      "out_shape": [
        1,
        128,
        100
      ],
      "size_mb": 1350.4,
      "format": "mnn"
    },
    {
      "name": "litert_vocos_fp32",
      "ok": true,
      "mean_s": 0.023510274599948387,
      "std_s": 0.00024219493729862353,
      "min_s": 0.023173624999799358,
      "max_s": 0.02386070799980189,
      "out_shape": [
        1,
        1,
        32512
      ],
      "size_mb": 58.52,
      "format": "litert"
    },
    {
      "name": "mnn_vocos_fp16",
      "ok": true,
      "mean_s": 0.01884079139995265,
      "std_s": 0.0003826578183155768,
      "min_s": 0.018479916999922352,
      "max_s": 0.01943533300004674,
      "out_shape": [
        1,
        1,
        32512
      ],
      "size_mb": 29.34,
      "format": "mnn"
    },
    {
      "name": "mnn_vocos_int8wq",
      "ok": true,
      "mean_s": 0.018562100000008287,
      "std_s": 0.0001635790577244952,
      "min_s": 0.01843074999987948,
      "max_s": 0.01886475000014798,
      "out_shape": [
        1,
        1,
        32512
      ],
      "size_mb": 15.05,
      "format": "mnn"
    },
    {
      "name": "mnn_vocos_fp16_int8wq",
      "ok": true,
      "mean_s": 0.021306441400111,
      "std_s": 0.0028035859396329985,
      "min_s": 0.01785941600019214,
      "max_s": 0.024554208000154176,
      "out_shape": [
        1,
        1,
        32512
      ],
      "size_mb": 29.34,
      "format": "mnn"
    },
    {
      "name": "mnn_dit_fp16",
      "ok": true,
      "mean_s": 0.48576437519996035,
      "std_s": 0.1993106179657564,
      "min_s": 0.26494529199999306,
      "max_s": 0.7715467499999704,
      "out_shape": [
        1,
        128,
        100
      ],
      "size_mb": 676.66,
      "format": "mnn"
    },
    {
      "name": "mnn_dit_int8wq",
      "ok": true,
      "mean_s": 0.2773376831999485,
      "std_s": 0.03149785049287247,
      "min_s": 0.23587449999990895,
      "max_s": 0.3237904999998591,
      "out_shape": [
        1,
        128,
        100
      ],
      "size_mb": 346.52,
      "format": "mnn"
    },
    {
      "name": "mnn_dit_fp16_int8wq",
      "ok": true,
      "mean_s": 0.2746680413999911,
      "std_s": 0.03585345627868485,
      "min_s": 0.22788883299995177,
      "max_s": 0.33330849999993006,
      "out_shape": [
        1,
        128,
        100
      ],
      "size_mb": 676.66,
      "format": "mnn"
    },
    {
      "name": "litert_vocos_fp16",
      "ok": false,
      "error": "tflite/kernels/conv.cc:360 input_type == kTfLiteFloat32 || input_type == kTfLiteUInt8 || input_type == kTfLiteInt8 || input_type == kTfLiteInt16 was not true.Node number 3 (CONV_2D) failed to prepare.",
      "size_mb": 29.29
    },
    {
      "name": "litert_vocos_drq",
      "ok": true,
      "mean_s": 0.02443846679998387,
      "std_s": 0.00026067359768839086,
      "min_s": 0.024193834000016068,
      "max_s": 0.024878499999886117,
      "out_shape": [
        1,
        1,
        32512
      ],
      "size_mb": 57.23,
      "format": "litert"
    },
    {
      "name": "litert_dit_drq",
      "ok": false,
      "error": "missing"
    }
  ],
  "quantization": [
    {
      "model": "mnn_vocos_fp16",
      "ok": true,
      "size_mb": 29.34
    },
    {
      "model": "mnn_vocos_int8wq",
      "ok": true,
      "size_mb": 15.05
    },
    {
      "model": "mnn_vocos_fp16_int8wq",
      "ok": true,
      "size_mb": 29.34
    },
    {
      "model": "mnn_dit_fp16",
      "ok": true,
      "size_mb": 676.66
    },
    {
      "model": "mnn_dit_int8wq",
      "ok": true,
      "size_mb": 346.52
    },
    {
      "model": "mnn_dit_fp16_int8wq",
      "ok": true,
      "size_mb": 676.66
    },
    {
      "model": "litert_vocos_fp16",
      "ok": true,
      "size_mb": 29.29
    },
    {
      "model": "litert_vocos_drq",
      "ok": true,
      "size_mb": 57.23
    },
    {
      "model": "litert_dit_drq",
      "ok": false,
      "error": "Skipped: DiT LiteRT float is not runnable (ONNX_CONV); DRQ reconvert would be ~1.3GB+ and same blocker."
    }
  ]
}