Review cloud MIC baselines arm64-LUKE

#2
by tarekziade HF Staff - opened
integrations/transformers/expectations/cpu-arm64/luke-base-mic.bfloat16.json ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/luke-base-mic",
4
+ "contract_name": "LUKE",
5
+ "contract_revision": "9d8ddf8bc533d409a7ea31ed871c9bfec0a26c05",
6
+ "framework": "transformers",
7
+ "variant": "bfloat16",
8
+ "target": "luke-base-mic",
9
+ "kind": "fixture",
10
+ "revision": "9d8ddf8bc533d409a7ea31ed871c9bfec0a26c05",
11
+ "source_model": "studio-ousia/luke-base",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/luke-base-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.44,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "bfloat16",
48
+ "attn_implementation": "eager"
49
+ },
50
+ "predicted_tokens": [
51
+ 46570,
52
+ 46570,
53
+ 46570,
54
+ 30827
55
+ ],
56
+ "answer_logits": [
57
+ [
58
+ 0.1689453125,
59
+ 0.1689453125,
60
+ -0.130859375,
61
+ 0.1689453125
62
+ ],
63
+ [
64
+ 0.169921875,
65
+ 0.169921875,
66
+ -0.130859375,
67
+ 0.169921875
68
+ ],
69
+ [
70
+ 0.1689453125,
71
+ 0.1689453125,
72
+ -0.1318359375,
73
+ 0.1689453125
74
+ ],
75
+ [
76
+ 0.15625,
77
+ 0.15625,
78
+ -0.15234375,
79
+ 0.15625
80
+ ]
81
+ ],
82
+ "entity_logits": [
83
+ [
84
+ 0.0,
85
+ -0.036865234375,
86
+ -0.0361328125,
87
+ -0.039306640625,
88
+ 0.037109375
89
+ ],
90
+ [
91
+ 0.0,
92
+ -0.03662109375,
93
+ -0.035888671875,
94
+ -0.039306640625,
95
+ 0.037109375
96
+ ],
97
+ [
98
+ 0.0,
99
+ -0.036865234375,
100
+ -0.035888671875,
101
+ -0.039306640625,
102
+ 0.037353515625
103
+ ],
104
+ [
105
+ 0.0,
106
+ -0.03662109375,
107
+ -0.0361328125,
108
+ -0.03955078125,
109
+ 0.037109375
110
+ ]
111
+ ],
112
+ "training": {
113
+ "loss": 10.75,
114
+ "gradients": {
115
+ "luke.encoder.layer.0.attention.self.query.weight": true,
116
+ "luke.encoder.layer.0.intermediate.dense.weight": true,
117
+ "lm_head.dense.weight": true
118
+ },
119
+ "updated": true
120
+ }
121
+ }
122
+ }
integrations/transformers/expectations/cpu-arm64/luke-base-mic.eager.json ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/luke-base-mic",
4
+ "contract_name": "LUKE",
5
+ "contract_revision": "9d8ddf8bc533d409a7ea31ed871c9bfec0a26c05",
6
+ "framework": "transformers",
7
+ "variant": "eager",
8
+ "target": "luke-base-mic",
9
+ "kind": "fixture",
10
+ "revision": "9d8ddf8bc533d409a7ea31ed871c9bfec0a26c05",
11
+ "source_model": "studio-ousia/luke-base",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/luke-base-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.34,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "eager"
49
+ },
50
+ "predicted_tokens": [
51
+ 46570,
52
+ 46570,
53
+ 46570,
54
+ 30827
55
+ ],
56
+ "answer_logits": [
57
+ [
58
+ 0.17008844017982483,
59
+ 0.17008844017982483,
60
+ -0.13127025961875916,
61
+ 0.17008844017982483
62
+ ],
63
+ [
64
+ 0.1697533279657364,
65
+ 0.1697533279657364,
66
+ -0.13180765509605408,
67
+ 0.1697533279657364
68
+ ],
69
+ [
70
+ 0.17020536959171295,
71
+ 0.17020536959171295,
72
+ -0.13180400431156158,
73
+ 0.17020536959171295
74
+ ],
75
+ [
76
+ 0.156118705868721,
77
+ 0.156118705868721,
78
+ -0.15173213183879852,
79
+ 0.156118705868721
80
+ ]
81
+ ],
82
+ "entity_logits": [
83
+ [
84
+ 0.0,
85
+ -0.03656845912337303,
86
+ -0.03587094694375992,
87
+ -0.0396362729370594,
88
+ 0.03710060939192772
89
+ ],
90
+ [
91
+ 0.0,
92
+ -0.03661423176527023,
93
+ -0.035875409841537476,
94
+ -0.03963891789317131,
95
+ 0.0371009036898613
96
+ ],
97
+ [
98
+ 0.0,
99
+ -0.036702170968055725,
100
+ -0.03590266779065132,
101
+ -0.039533648639917374,
102
+ 0.03722726181149483
103
+ ],
104
+ [
105
+ 0.0,
106
+ -0.03655447065830231,
107
+ -0.03584036976099014,
108
+ -0.0396711602807045,
109
+ 0.0366782620549202
110
+ ]
111
+ ],
112
+ "training": {
113
+ "loss": 10.73832893371582,
114
+ "gradients": {
115
+ "luke.encoder.layer.0.attention.self.query.weight": true,
116
+ "luke.encoder.layer.0.intermediate.dense.weight": true,
117
+ "lm_head.dense.weight": true
118
+ },
119
+ "updated": true
120
+ }
121
+ }
122
+ }
integrations/transformers/expectations/cpu-arm64/luke-base-mic.json ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/luke-base-mic",
4
+ "contract_name": "LUKE",
5
+ "contract_revision": "9d8ddf8bc533d409a7ea31ed871c9bfec0a26c05",
6
+ "framework": "transformers",
7
+ "variant": "default",
8
+ "target": "luke-base-mic",
9
+ "kind": "fixture",
10
+ "revision": "9d8ddf8bc533d409a7ea31ed871c9bfec0a26c05",
11
+ "source_model": "studio-ousia/luke-base",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/luke-base-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.18,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "eager"
49
+ },
50
+ "predicted_tokens": [
51
+ 40205,
52
+ 40205,
53
+ 40205,
54
+ 9329
55
+ ],
56
+ "answer_logits": [
57
+ [
58
+ 0.03272596746683121,
59
+ 0.03272596746683121,
60
+ -0.25672200322151184,
61
+ 0.03272596746683121
62
+ ],
63
+ [
64
+ 0.03263312578201294,
65
+ 0.03263312578201294,
66
+ -0.25619956851005554,
67
+ 0.03263312578201294
68
+ ],
69
+ [
70
+ 0.032636649906635284,
71
+ 0.032636649906635284,
72
+ -0.25608131289482117,
73
+ 0.032636649906635284
74
+ ],
75
+ [
76
+ -0.05772646889090538,
77
+ -0.05772646889090538,
78
+ -0.1246420368552208,
79
+ -0.05772646889090538
80
+ ]
81
+ ],
82
+ "entity_logits": [
83
+ [
84
+ 0.0,
85
+ -0.03656845912337303,
86
+ -0.03587094694375992,
87
+ -0.0396362729370594,
88
+ 0.03710060939192772
89
+ ],
90
+ [
91
+ 0.0,
92
+ -0.03661423176527023,
93
+ -0.035875409841537476,
94
+ -0.03963891789317131,
95
+ 0.0371009036898613
96
+ ],
97
+ [
98
+ 0.0,
99
+ -0.036702170968055725,
100
+ -0.03590266779065132,
101
+ -0.039533648639917374,
102
+ 0.03722726181149483
103
+ ],
104
+ [
105
+ 0.0,
106
+ -0.03655447065830231,
107
+ -0.03584036976099014,
108
+ -0.0396711602807045,
109
+ 0.0366782620549202
110
+ ]
111
+ ],
112
+ "training": {
113
+ "loss": 10.895441055297852,
114
+ "gradients": {
115
+ "luke.encoder.layer.0.attention.self.query.weight": true,
116
+ "luke.encoder.layer.0.intermediate.dense.weight": true,
117
+ "lm_head.dense.weight": true
118
+ },
119
+ "updated": true
120
+ }
121
+ }
122
+ }