Review cloud MIC baselines arm64-Flint

#1
by tarekziade HF Staff - opened
integrations/transformers/expectations/cpu-arm64/flint-mic.bfloat16.json ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/flint-mic",
4
+ "contract_name": "Flint",
5
+ "contract_revision": "0ef1e43061d0ad8938cba0541525c1640401c143",
6
+ "framework": "transformers",
7
+ "variant": "bfloat16",
8
+ "target": "flint-mic",
9
+ "kind": "fixture",
10
+ "revision": "0ef1e43061d0ad8938cba0541525c1640401c143",
11
+ "source_model": "RecursiveSI/flint",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/flint-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.64,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "bfloat16",
48
+ "attn_implementation": "eager"
49
+ },
50
+ "generated_tokens": [
51
+ [
52
+ 90614,
53
+ 131281,
54
+ 32668,
55
+ 131930
56
+ ],
57
+ [
58
+ 26200,
59
+ 52029,
60
+ 49170,
61
+ 63634
62
+ ],
63
+ [
64
+ 138649,
65
+ 54698,
66
+ 551,
67
+ 144794
68
+ ],
69
+ [
70
+ 141782,
71
+ 75510,
72
+ 32929,
73
+ 33195
74
+ ]
75
+ ],
76
+ "generated_text": [
77
+ " kinases\u5b89\u846c\u30f3\u30b9\u307b\u3046",
78
+ "Identitygrowth <<< plum",
79
+ " \u0639\u0636 /.ide TIcon",
80
+ " Jansen \u0438\u043c\u043f(',_callback"
81
+ ],
82
+ "next_token_logits": [
83
+ [
84
+ -0.068359375,
85
+ -0.056396484375,
86
+ 0.435546875,
87
+ 0.1357421875
88
+ ],
89
+ [
90
+ -0.02001953125,
91
+ -0.0263671875,
92
+ 0.27734375,
93
+ 0.044189453125
94
+ ],
95
+ [
96
+ 0.0106201171875,
97
+ -0.171875,
98
+ 0.32421875,
99
+ -0.10205078125
100
+ ],
101
+ [
102
+ 0.02197265625,
103
+ -0.154296875,
104
+ 0.314453125,
105
+ -0.055908203125
106
+ ]
107
+ ],
108
+ "training": {
109
+ "loss": 11.915054321289062,
110
+ "gradients": {
111
+ "model.layers.0.self_attn.q_proj.weight": true,
112
+ "model.layers.1.mlp.experts.gate_up_proj": true,
113
+ "model.layers.1.mlp.gate.weight": true,
114
+ "model.layers.1.mlp.shared_expert.down_proj.weight": true,
115
+ "lm_head.weight": true
116
+ },
117
+ "updated": true
118
+ }
119
+ }
120
+ }
integrations/transformers/expectations/cpu-arm64/flint-mic.eager.json ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/flint-mic",
4
+ "contract_name": "Flint",
5
+ "contract_revision": "0ef1e43061d0ad8938cba0541525c1640401c143",
6
+ "framework": "transformers",
7
+ "variant": "eager",
8
+ "target": "flint-mic",
9
+ "kind": "fixture",
10
+ "revision": "0ef1e43061d0ad8938cba0541525c1640401c143",
11
+ "source_model": "RecursiveSI/flint",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/flint-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.26,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "eager"
49
+ },
50
+ "generated_tokens": [
51
+ [
52
+ 90614,
53
+ 131281,
54
+ 32668,
55
+ 131930
56
+ ],
57
+ [
58
+ 26200,
59
+ 52029,
60
+ 49170,
61
+ 63634
62
+ ],
63
+ [
64
+ 138649,
65
+ 54698,
66
+ 551,
67
+ 144794
68
+ ],
69
+ [
70
+ 141782,
71
+ 75510,
72
+ 32929,
73
+ 33195
74
+ ]
75
+ ],
76
+ "generated_text": [
77
+ " kinases\u5b89\u846c\u30f3\u30b9\u307b\u3046",
78
+ "Identitygrowth <<< plum",
79
+ " \u0639\u0636 /.ide TIcon",
80
+ " Jansen \u0438\u043c\u043f(',_callback"
81
+ ],
82
+ "next_token_logits": [
83
+ [
84
+ -0.0706963837146759,
85
+ -0.056407880038022995,
86
+ 0.43850913643836975,
87
+ 0.1358330398797989
88
+ ],
89
+ [
90
+ -0.019916284829378128,
91
+ -0.02684975415468216,
92
+ 0.27915921807289124,
93
+ 0.04557495936751366
94
+ ],
95
+ [
96
+ 0.010465828701853752,
97
+ -0.17284883558750153,
98
+ 0.3238558769226074,
99
+ -0.10293308645486832
100
+ ],
101
+ [
102
+ 0.022369472309947014,
103
+ -0.1533534824848175,
104
+ 0.3142346143722534,
105
+ -0.054519835859537125
106
+ ]
107
+ ],
108
+ "training": {
109
+ "loss": 11.914274215698242,
110
+ "gradients": {
111
+ "model.layers.0.self_attn.q_proj.weight": true,
112
+ "model.layers.1.mlp.experts.gate_up_proj": true,
113
+ "model.layers.1.mlp.gate.weight": true,
114
+ "model.layers.1.mlp.shared_expert.down_proj.weight": true,
115
+ "lm_head.weight": true
116
+ },
117
+ "updated": true
118
+ }
119
+ }
120
+ }
integrations/transformers/expectations/cpu-arm64/flint-mic.json ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/flint-mic",
4
+ "contract_name": "Flint",
5
+ "contract_revision": "0ef1e43061d0ad8938cba0541525c1640401c143",
6
+ "framework": "transformers",
7
+ "variant": "default",
8
+ "target": "flint-mic",
9
+ "kind": "fixture",
10
+ "revision": "0ef1e43061d0ad8938cba0541525c1640401c143",
11
+ "source_model": "RecursiveSI/flint",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/flint-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.42,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "eager"
49
+ },
50
+ "generated_tokens": [
51
+ [
52
+ 90614,
53
+ 131281,
54
+ 32668,
55
+ 131930
56
+ ],
57
+ [
58
+ 26200,
59
+ 52029,
60
+ 49170,
61
+ 63634
62
+ ],
63
+ [
64
+ 138649,
65
+ 54698,
66
+ 551,
67
+ 144794
68
+ ],
69
+ [
70
+ 141782,
71
+ 75510,
72
+ 32929,
73
+ 33195
74
+ ]
75
+ ],
76
+ "generated_text": [
77
+ " kinases\u5b89\u846c\u30f3\u30b9\u307b\u3046",
78
+ "Identitygrowth <<< plum",
79
+ " \u0639\u0636 /.ide TIcon",
80
+ " Jansen \u0438\u043c\u043f(',_callback"
81
+ ],
82
+ "next_token_logits": [
83
+ [
84
+ -0.0706963837146759,
85
+ -0.056407880038022995,
86
+ 0.43850913643836975,
87
+ 0.1358330398797989
88
+ ],
89
+ [
90
+ -0.019916284829378128,
91
+ -0.02684975415468216,
92
+ 0.27915921807289124,
93
+ 0.04557495936751366
94
+ ],
95
+ [
96
+ 0.010465828701853752,
97
+ -0.17284883558750153,
98
+ 0.3238558769226074,
99
+ -0.10293308645486832
100
+ ],
101
+ [
102
+ 0.022369472309947014,
103
+ -0.1533534824848175,
104
+ 0.3142346143722534,
105
+ -0.054519835859537125
106
+ ]
107
+ ],
108
+ "training": {
109
+ "loss": 11.914274215698242,
110
+ "gradients": {
111
+ "model.layers.0.self_attn.q_proj.weight": true,
112
+ "model.layers.1.mlp.experts.gate_up_proj": true,
113
+ "model.layers.1.mlp.gate.weight": true,
114
+ "model.layers.1.mlp.shared_expert.down_proj.weight": true,
115
+ "lm_head.weight": true
116
+ },
117
+ "updated": true
118
+ }
119
+ }
120
+ }