Add MLX int4 variant

.gitattributes

@@ -36,3 +36,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 1_8b/xnnpack/hy_mt2_1_8b_xnnpack_8da4w.pte filter=lfs diff=lfs merge=lfs -text
 1_8b/xnnpack/hy_mt2_1_8b_xnnpack_8da8w.pte filter=lfs diff=lfs merge=lfs -text
 1_8b/mlx/hy_mt2_1_8b_mlx_int8.pte filter=lfs diff=lfs merge=lfs -text
+1_8b/mlx/hy_mt2_1_8b_mlx_int4.pte filter=lfs diff=lfs merge=lfs -text

1_8b/mlx/config.json

@@ -60,6 +60,55 @@
         "use_kv_cache": true,
         "use_sdpa_with_kv_cache": true
       }
+    },
+    {
+      "file": "hy_mt2_1_8b_mlx_int4.pte",
+      "precision": "int4",
+      "quantized": true,
+      "default": false,
+      "methods": {
+        "enable_dynamic_shape": true,
+        "forward": {
+          "inputs": [
+            {
+              "shape": [
+                1,
+                {
+                  "min": 1,
+                  "max": 1023
+                }
+              ],
+              "dtype": "int64"
+            },
+            {
+              "shape": [
+                1
+              ],
+              "dtype": "int64"
+            }
+          ],
+          "outputs": [
+            {
+              "shape": [
+                1,
+                120818
+              ],
+              "dtype": "float32"
+            }
+          ]
+        },
+        "get_bos_id": 120000,
+        "get_eos_ids": [
+          120020,
+          120001
+        ],
+        "get_max_context_len": 1024,
+        "get_max_seq_len": 1024,
+        "get_n_layers": 32,
+        "get_vocab_size": 120818,
+        "use_kv_cache": true,
+        "use_sdpa_with_kv_cache": true
+      }
     }
   ]
 }

1_8b/mlx/hy_mt2_1_8b_mlx_int4.pte

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5131e3ded8a26bdea6c39f9d920185ae96f2d041f46be5d9074951a31e71805
+size 952007552

README.md

@@ -20,11 +20,13 @@ Upstream model: [tencent/Hy-MT2-1.8B](https://huggingface.co/tencent/Hy-MT2-1.8B
 | `1_8b/xnnpack/hy_mt2_1_8b_xnnpack_8da4w.pte` | xnnpack | 8da4w |
 | `1_8b/xnnpack/hy_mt2_1_8b_xnnpack_8da8w.pte` | xnnpack | 8da8w |
 | `1_8b/mlx/hy_mt2_1_8b_mlx_int8.pte` | mlx | int8 |
+| `1_8b/mlx/hy_mt2_1_8b_mlx_int4.pte` | mlx | int4 |
 
 ## Repository structure
 
 ```
 1_8b/mlx/config.json                         1.5 kB
+1_8b/mlx/hy_mt2_1_8b_mlx_int4.pte            952.0 MB
 1_8b/mlx/hy_mt2_1_8b_mlx_int8.pte            1.8 GB
 1_8b/tokenizer.json                          5.2 MB
 1_8b/tokenizer_config.json                   166.5 kB
@@ -52,7 +54,7 @@ first.
 
 XNNPACK uses dynamic int8 activations with int4 weights (group 32, the default)
 or per-channel int8 weights; 4-bit costs 0.2-1.5 BLEU for 43% less size and
-1.5x faster decode. MLX uses int8 weight-only. Prompt it as the model card does, with no
+1.5x faster decode. MLX uses int8 (default) or int4 (group 64) weight-only; int4 is ~35% faster on an iPhone 16 but loses 3.4 BLEU on en->zh. Prompt it as the model card does, with no
 system prompt:
 
 ```