tarekziade HF Staff commited on
Commit
401bd5e
·
verified ·
1 Parent(s): 006df2d

Review cloud MIC baselines arm64-MT5

Browse files

Measured and independently validated on CPU and CUDA. Review reasons: {"cpu/prototypes/MT5/mt5-small-mic/bfloat16": {"reason": "New or unchanged measured reference", "sha256": "69700508cdc16150a9b3d8dfd9b100307cbb1f88c4531c0bc00440ff46789b23", "verdict": "new"}, "cpu/prototypes/MT5/mt5-small-mic/default": {"reason": "New or unchanged measured reference", "sha256": "195bb16c9af7acaa7adcad8bffa72fe91f085f8f0b42c99aed55364abaab20e8", "verdict": "new"}, "cpu/prototypes/MT5/mt5-small-mic/eager": {"reason": "New or unchanged measured reference", "sha256": "ff4e3f49a9ad3d4ee2cf27606528cf5cdacbce487a64059381094a96e9b869f3", "verdict": "new"}}

integrations/transformers/expectations/cpu-arm64/mt5-small-mic.bfloat16.json ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/mt5-small-mic",
4
+ "contract_name": "MT5",
5
+ "contract_revision": "006df2dcc1aaf836b2324d064381029e2f6f2784",
6
+ "framework": "transformers",
7
+ "variant": "bfloat16",
8
+ "target": "mt5-small-mic",
9
+ "kind": "fixture",
10
+ "revision": "006df2dcc1aaf836b2324d064381029e2f6f2784",
11
+ "source_model": "google/mt5-small",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/mt5-small-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 4.33,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "bfloat16",
48
+ "attn_implementation": "sdpa"
49
+ },
50
+ "decoded_text": [
51
+ "\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a",
52
+ "Win Win Win Win Win Win Win Win"
53
+ ],
54
+ "probe_logits": [
55
+ [
56
+ 31.875,
57
+ 4.9375,
58
+ 0.48046875,
59
+ 13.125,
60
+ 1.921875
61
+ ],
62
+ [
63
+ 31.25,
64
+ 4.84375,
65
+ 2.1875,
66
+ 12.875,
67
+ -1.3046875
68
+ ]
69
+ ],
70
+ "training": {
71
+ "loss": 42.25,
72
+ "gradients": {
73
+ "encoder.block.0.layer.0.SelfAttention.q.weight": true,
74
+ "decoder.block.0.layer.0.SelfAttention.q.weight": true,
75
+ "decoder.block.0.layer.1.EncDecAttention.q.weight": true
76
+ },
77
+ "updated": true
78
+ }
79
+ }
80
+ }
integrations/transformers/expectations/cpu-arm64/mt5-small-mic.eager.json ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/mt5-small-mic",
4
+ "contract_name": "MT5",
5
+ "contract_revision": "006df2dcc1aaf836b2324d064381029e2f6f2784",
6
+ "framework": "transformers",
7
+ "variant": "eager",
8
+ "target": "mt5-small-mic",
9
+ "kind": "fixture",
10
+ "revision": "006df2dcc1aaf836b2324d064381029e2f6f2784",
11
+ "source_model": "google/mt5-small",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/mt5-small-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.42,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "eager"
49
+ },
50
+ "decoded_text": [
51
+ "\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a",
52
+ "Win Win Win Win Win Win Win Win"
53
+ ],
54
+ "probe_logits": [
55
+ [
56
+ 31.92408561706543,
57
+ 4.906943321228027,
58
+ 0.616291344165802,
59
+ 13.151479721069336,
60
+ 2.0583295822143555
61
+ ],
62
+ [
63
+ 31.296791076660156,
64
+ 4.67181921005249,
65
+ 2.257627248764038,
66
+ 12.908451080322266,
67
+ -1.2190333604812622
68
+ ]
69
+ ],
70
+ "training": {
71
+ "loss": 41.97232437133789,
72
+ "gradients": {
73
+ "encoder.block.0.layer.0.SelfAttention.q.weight": true,
74
+ "decoder.block.0.layer.0.SelfAttention.q.weight": true,
75
+ "decoder.block.0.layer.1.EncDecAttention.q.weight": true
76
+ },
77
+ "updated": true
78
+ }
79
+ }
80
+ }
integrations/transformers/expectations/cpu-arm64/mt5-small-mic.json ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/mt5-small-mic",
4
+ "contract_name": "MT5",
5
+ "contract_revision": "006df2dcc1aaf836b2324d064381029e2f6f2784",
6
+ "framework": "transformers",
7
+ "variant": "default",
8
+ "target": "mt5-small-mic",
9
+ "kind": "fixture",
10
+ "revision": "006df2dcc1aaf836b2324d064381029e2f6f2784",
11
+ "source_model": "google/mt5-small",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/mt5-small-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.72,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "sdpa"
49
+ },
50
+ "decoded_text": [
51
+ "\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a\u0e01\u0e23\u0e2d\u0e1a",
52
+ "Win Win Win Win Win Win Win Win"
53
+ ],
54
+ "probe_logits": [
55
+ [
56
+ 31.92408561706543,
57
+ 4.906942844390869,
58
+ 0.6162952184677124,
59
+ 13.15147876739502,
60
+ 2.0583250522613525
61
+ ],
62
+ [
63
+ 31.29678726196289,
64
+ 4.671821117401123,
65
+ 2.257624864578247,
66
+ 12.908451080322266,
67
+ -1.2190276384353638
68
+ ]
69
+ ],
70
+ "training": {
71
+ "loss": 41.97232437133789,
72
+ "gradients": {
73
+ "encoder.block.0.layer.0.SelfAttention.q.weight": true,
74
+ "decoder.block.0.layer.0.SelfAttention.q.weight": true,
75
+ "decoder.block.0.layer.1.EncDecAttention.q.weight": true
76
+ },
77
+ "updated": true
78
+ }
79
+ }
80
+ }