Add XNNPACK int8 export

.gitattributes

@@ -41,3 +41,4 @@ coreml/all_minilm_l6_v2_coreml_fp32.pte filter=lfs diff=lfs merge=lfs -text
 vulkan/all_minilm_l6_v2_vulkan_fp32.pte filter=lfs diff=lfs merge=lfs -text
 vulkan/all_minilm_l6_v2_vulkan_fp16.pte filter=lfs diff=lfs merge=lfs -text
 coreml/all_minilm_l6_v2_coreml_int8.pte filter=lfs diff=lfs merge=lfs -text
+xnnpack/all_minilm_l6_v2_xnnpack_int8.pte filter=lfs diff=lfs merge=lfs -text

README.md

@@ -17,21 +17,36 @@ Upstream model: [all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/
 | Path | Backend | Precision |
 | --- | --- | --- |
 | `coreml/all_minilm_l6_v2_coreml_fp16.pte` | coreml | fp16 |
+| `coreml/all_minilm_l6_v2_coreml_int8.pte` | coreml | int8 |
 | `vulkan/all_minilm_l6_v2_vulkan_fp16.pte` | vulkan | fp16 |
 | `xnnpack/all_minilm_l6_v2_xnnpack_fp32.pte` | xnnpack | fp32 |
+| `xnnpack/all_minilm_l6_v2_xnnpack_int8.pte` | xnnpack | int8 |
+
+## XNNPACK int8
+
+`xnnpack/all_minilm_l6_v2_xnnpack_int8.pte` is 8da8w (int8 dynamic activations, int8 per-channel
+weights) on the Linear layers, with an int8 word-embedding table. SQuAD v1.1 dev passage
+retrieval (2000 questions against their 315 paragraphs), dot product:
+
+| | top-1 | MRR |
+| --- | --- | --- |
+| fp32 | 57.70 | 0.7012 |
+| int8 | 58.05 | 0.7024 |
 
 ## Repository structure
 
 ```
 config.json                                38 B
 coreml/all_minilm_l6_v2_coreml_fp16.pte    43.4 MB
-coreml/config.json                         929 B
+coreml/all_minilm_l6_v2_coreml_int8.pte    21.8 MB
+coreml/config.json                         1.7 kB
 tokenizer.json                             695 kB
 tokenizer_config.json                      350 B
 vulkan/all_minilm_l6_v2_vulkan_fp16.pte    43.1 MB
 vulkan/config.json                         929 B
 xnnpack/all_minilm_l6_v2_xnnpack_fp32.pte  86.2 MB
-xnnpack/config.json                        931 B
+xnnpack/all_minilm_l6_v2_xnnpack_int8.pte  21.9 MB
+xnnpack/config.json                        1.5 kB
 ```
 
 ## Compatibility

xnnpack/all_minilm_l6_v2_xnnpack_int8.pte

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1769173f272a41e4371fa2c078795b66a932efbbc25881749e2571ad44cffb1
+size 23010688

xnnpack/config.json

@@ -40,6 +40,39 @@
           ]
         }
       }
+    },
+    {
+      "file": "all_minilm_l6_v2_xnnpack_int8.pte",
+      "precision": "int8",
+      "methods": {
+        "forward": {
+          "inputs": [
+            {
+              "shape": [
+                1,
+                254
+              ],
+              "dtype": "int64"
+            },
+            {
+              "shape": [
+                1,
+                254
+              ],
+              "dtype": "int64"
+            }
+          ],
+          "outputs": [
+            {
+              "shape": [
+                1,
+                384
+              ],
+              "dtype": "float32"
+            }
+          ]
+        }
+      }
     }
   ]
 }