aceOfDiamonds commited on
Commit
fb0186a
·
1 Parent(s): 7bbfab9

Add a Vulkan fp16 variant

Browse files

Lowered with eager attention and rank-3 attention (the singleton batch folded
into the head dim), mean pooling written with keepdim so the reduction stays on
the GPU, and the token embedding kept off texture storage, without which fp16
returns silently wrong values (pytorch/executorch#22333).

Verified against the CPU reference at cosine >= 0.99997 on an Adreno 840 and a
Mali-G76.

Requires the ExecuTorch 1.4.1 Vulkan runtime from the v0.10.0-libs-1.4.1
artifacts.

.gitattributes CHANGED
@@ -38,3 +38,4 @@ multi-qa-MiniLM-L6-cos-v1_xnnpack.pte filter=lfs diff=lfs merge=lfs -text
38
  xnnpack/multi_qa_minilm_l6_cos_v1_xnnpack_fp32.pte filter=lfs diff=lfs merge=lfs -text
39
  coreml/multi_qa_minilm_l6_cos_v1_coreml_fp16.pte filter=lfs diff=lfs merge=lfs -text
40
  coreml/multi_qa_minilm_l6_cos_v1_coreml_fp32.pte filter=lfs diff=lfs merge=lfs -text
 
 
38
  xnnpack/multi_qa_minilm_l6_cos_v1_xnnpack_fp32.pte filter=lfs diff=lfs merge=lfs -text
39
  coreml/multi_qa_minilm_l6_cos_v1_coreml_fp16.pte filter=lfs diff=lfs merge=lfs -text
40
  coreml/multi_qa_minilm_l6_cos_v1_coreml_fp32.pte filter=lfs diff=lfs merge=lfs -text
41
+ vulkan/multi_qa_minilm_l6_cos_v1_vulkan_fp16.pte filter=lfs diff=lfs merge=lfs -text
vulkan/config.json ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "$schema": "https://huggingface.co/software-mansion/react-native-executorch-spec/resolve/main/config.schema.json",
3
+ "model": "multi_qa_minilm_l6_cos_v1",
4
+ "family": "sbert",
5
+ "capabilities": [
6
+ "text-embedding"
7
+ ],
8
+ "backend": "vulkan",
9
+ "license": "apache-2.0",
10
+ "variants": [
11
+ {
12
+ "file": "multi_qa_minilm_l6_cos_v1_vulkan_fp16.pte",
13
+ "precision": "fp16",
14
+ "quantized": false,
15
+ "default": true,
16
+ "methods": {
17
+ "forward": {
18
+ "inputs": [
19
+ {
20
+ "shape": [
21
+ 1,
22
+ 510
23
+ ],
24
+ "dtype": "int64"
25
+ },
26
+ {
27
+ "shape": [
28
+ 1,
29
+ 510
30
+ ],
31
+ "dtype": "int64"
32
+ }
33
+ ],
34
+ "outputs": [
35
+ {
36
+ "shape": [
37
+ 1,
38
+ 384
39
+ ],
40
+ "dtype": "float32"
41
+ }
42
+ ]
43
+ }
44
+ }
45
+ }
46
+ ]
47
+ }
vulkan/multi_qa_minilm_l6_cos_v1_vulkan_fp16.pte ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9da2740169a6f3628a5f8cef092235e9d15f4918e6e9545240166a5201633505
3
+ size 45208066