quant_config.json

{
  "format": 3,
  "bits": 4,
  "audio_bits": 4,
  "int8_layers": {
    "language_model": [
      "other",
      23,
      11,
      22,
      10,
      14
    ],
    "vision_tower": [
      12,
      13,
      14,
      15
    ]
  },
  "group": 16,
  "position_rows": 2520,
  "embed_scale": {
    "language_model.embed_tokens": 22.625
  },
  "meta": "uint8 affine per row with fp16 lo and hi",
  "audio": "tower block linears int4; convolutions, norms and embed_audio bf16",
  "text": "linears and token table int4",
  "vision": "block linears int4",
  "kept_bf16": "patch input projection, position table, embed_vision, norms"
}