Review cloud MIC baselines arm64-RoBERTa

#1
by tarekziade HF Staff - opened
integrations/transformers/expectations/cpu-arm64/roberta-base-mic.bfloat16.json ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/roberta-base-mic",
4
+ "contract_name": "RoBERTa",
5
+ "contract_revision": "7581d460c53ed2a56bc936b5577438d26b36b73f",
6
+ "framework": "transformers",
7
+ "variant": "bfloat16",
8
+ "target": "roberta-base-mic",
9
+ "kind": "fixture",
10
+ "revision": "7581d460c53ed2a56bc936b5577438d26b36b73f",
11
+ "source_model": "FacebookAI/roberta-base",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/roberta-base-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.33,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "bfloat16",
48
+ "attn_implementation": "sdpa"
49
+ },
50
+ "predicted_tokens": [
51
+ 33326,
52
+ 33326,
53
+ 33326,
54
+ 5069
55
+ ],
56
+ "predicted_text": [
57
+ "teenth",
58
+ "teenth",
59
+ "teenth",
60
+ "ains"
61
+ ],
62
+ "answer_logits": [
63
+ [
64
+ -0.279296875,
65
+ 0.0712890625,
66
+ 0.0712890625,
67
+ 0.031005859375
68
+ ],
69
+ [
70
+ -0.279296875,
71
+ 0.0703125,
72
+ 0.0703125,
73
+ 0.031494140625
74
+ ],
75
+ [
76
+ -0.279296875,
77
+ 0.0693359375,
78
+ 0.0693359375,
79
+ 0.0302734375
80
+ ],
81
+ [
82
+ -0.166015625,
83
+ 0.037841796875,
84
+ 0.037841796875,
85
+ 0.0150146484375
86
+ ]
87
+ ],
88
+ "training": {
89
+ "loss": 10.8125,
90
+ "gradients": {
91
+ "roberta.encoder.layer.0.attention.self.query.weight": true,
92
+ "roberta.encoder.layer.0.intermediate.dense.weight": true
93
+ },
94
+ "updated": true
95
+ }
96
+ }
97
+ }
integrations/transformers/expectations/cpu-arm64/roberta-base-mic.eager.json ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/roberta-base-mic",
4
+ "contract_name": "RoBERTa",
5
+ "contract_revision": "7581d460c53ed2a56bc936b5577438d26b36b73f",
6
+ "framework": "transformers",
7
+ "variant": "eager",
8
+ "target": "roberta-base-mic",
9
+ "kind": "fixture",
10
+ "revision": "7581d460c53ed2a56bc936b5577438d26b36b73f",
11
+ "source_model": "FacebookAI/roberta-base",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/roberta-base-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.18,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "eager"
49
+ },
50
+ "predicted_tokens": [
51
+ 33326,
52
+ 33326,
53
+ 33326,
54
+ 5069
55
+ ],
56
+ "predicted_text": [
57
+ "teenth",
58
+ "teenth",
59
+ "teenth",
60
+ "ains"
61
+ ],
62
+ "answer_logits": [
63
+ [
64
+ -0.2789422869682312,
65
+ 0.07029280066490173,
66
+ 0.07029280066490173,
67
+ 0.031082630157470703
68
+ ],
69
+ [
70
+ -0.27898338437080383,
71
+ 0.07068229466676712,
72
+ 0.07068229466676712,
73
+ 0.030745994299650192
74
+ ],
75
+ [
76
+ -0.2788701355457306,
77
+ 0.07043246179819107,
78
+ 0.07043246179819107,
79
+ 0.03071236051619053
80
+ ],
81
+ [
82
+ -0.16440105438232422,
83
+ 0.03672809153795242,
84
+ 0.03672809153795242,
85
+ 0.014085035771131516
86
+ ]
87
+ ],
88
+ "training": {
89
+ "loss": 10.869680404663086,
90
+ "gradients": {
91
+ "roberta.encoder.layer.0.attention.self.query.weight": true,
92
+ "roberta.encoder.layer.0.intermediate.dense.weight": true
93
+ },
94
+ "updated": true
95
+ }
96
+ }
97
+ }
integrations/transformers/expectations/cpu-arm64/roberta-base-mic.json ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/roberta-base-mic",
4
+ "contract_name": "RoBERTa",
5
+ "contract_revision": "7581d460c53ed2a56bc936b5577438d26b36b73f",
6
+ "framework": "transformers",
7
+ "variant": "default",
8
+ "target": "roberta-base-mic",
9
+ "kind": "fixture",
10
+ "revision": "7581d460c53ed2a56bc936b5577438d26b36b73f",
11
+ "source_model": "FacebookAI/roberta-base",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/roberta-base-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.1,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "sdpa"
49
+ },
50
+ "predicted_tokens": [
51
+ 33326,
52
+ 33326,
53
+ 33326,
54
+ 5069
55
+ ],
56
+ "predicted_text": [
57
+ "teenth",
58
+ "teenth",
59
+ "teenth",
60
+ "ains"
61
+ ],
62
+ "answer_logits": [
63
+ [
64
+ -0.2789422273635864,
65
+ 0.07029279321432114,
66
+ 0.07029279321432114,
67
+ 0.031082630157470703
68
+ ],
69
+ [
70
+ -0.27898335456848145,
71
+ 0.07068227976560593,
72
+ 0.07068227976560593,
73
+ 0.030745983123779297
74
+ ],
75
+ [
76
+ -0.2788701355457306,
77
+ 0.07043245434761047,
78
+ 0.07043245434761047,
79
+ 0.030712353065609932
80
+ ],
81
+ [
82
+ -0.164401113986969,
83
+ 0.036728084087371826,
84
+ 0.036728084087371826,
85
+ 0.014085037633776665
86
+ ]
87
+ ],
88
+ "training": {
89
+ "loss": 10.869680404663086,
90
+ "gradients": {
91
+ "roberta.encoder.layer.0.attention.self.query.weight": true,
92
+ "roberta.encoder.layer.0.intermediate.dense.weight": true
93
+ },
94
+ "updated": true
95
+ }
96
+ }
97
+ }