Review cloud MIC baselines arm64-DeepSeek-Coder-V2-Lite-Instruct

#1
by tarekziade HF Staff - opened
integrations/transformers/expectations/cpu-arm64/DeepSeek-Coder-V2-Lite-Instruct-mic.bfloat16.json ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/DeepSeek-Coder-V2-Lite-Instruct-mic",
4
+ "contract_name": "DeepSeek-Coder-V2-Lite-Instruct",
5
+ "contract_revision": "947b5968d33b8ab8f9079bf7d9bca75a4a788293",
6
+ "framework": "transformers",
7
+ "variant": "bfloat16",
8
+ "target": "DeepSeek-Coder-V2-Lite-Instruct-mic",
9
+ "kind": "fixture",
10
+ "revision": "947b5968d33b8ab8f9079bf7d9bca75a4a788293",
11
+ "source_model": "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/DeepSeek-Coder-V2-Lite-Instruct-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.45,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "bfloat16",
48
+ "attn_implementation": "sdpa"
49
+ },
50
+ "logits": {
51
+ "probe": [
52
+ [
53
+ 0.0255126953125,
54
+ -0.03125,
55
+ 0.05712890625,
56
+ -0.1123046875,
57
+ -0.0205078125,
58
+ 0.0751953125,
59
+ -0.050537109375,
60
+ 0.025390625,
61
+ 0.1044921875,
62
+ -0.1982421875,
63
+ 0.341796875,
64
+ 0.0625,
65
+ 0.0341796875,
66
+ 0.16796875,
67
+ -0.13671875,
68
+ 0.036376953125
69
+ ],
70
+ [
71
+ 0.055419921875,
72
+ -0.06884765625,
73
+ 0.056884765625,
74
+ -0.076171875,
75
+ -0.054931640625,
76
+ 0.0791015625,
77
+ -0.037353515625,
78
+ -0.00183868408203125,
79
+ 0.142578125,
80
+ -0.205078125,
81
+ 0.341796875,
82
+ 0.0308837890625,
83
+ 0.0634765625,
84
+ 0.2197265625,
85
+ -0.140625,
86
+ 0.043212890625
87
+ ]
88
+ ],
89
+ "position_norms": [
90
+ [
91
+ 51.137691497802734,
92
+ 51.1424560546875,
93
+ 51.079036712646484,
94
+ 51.16875076293945,
95
+ 51.22607421875,
96
+ 51.19965362548828,
97
+ 51.206512451171875,
98
+ 51.19367218017578,
99
+ 51.27151870727539,
100
+ 51.205684661865234,
101
+ 51.081077575683594,
102
+ 51.17299270629883,
103
+ 51.15104675292969,
104
+ 51.142147064208984,
105
+ 50.99774932861328,
106
+ 51.07828903198242
107
+ ],
108
+ [
109
+ 51.137691497802734,
110
+ 51.1424560546875,
111
+ 51.079036712646484,
112
+ 51.0666618347168,
113
+ 51.15766143798828,
114
+ 50.99543380737305,
115
+ 51.161102294921875,
116
+ 51.24142074584961,
117
+ 51.307193756103516,
118
+ 51.1064338684082,
119
+ 51.03031539916992,
120
+ 51.1072883605957,
121
+ 51.155517578125,
122
+ 51.149009704589844,
123
+ 51.05406188964844,
124
+ 51.063514709472656
125
+ ]
126
+ ]
127
+ },
128
+ "generated_text": [
129
+ " testim amountsomachrels ALTERAppl Recruitscanf",
130
+ " testim amounts\u51fa\u56fd surrendered Cartacustomers espacio\u0441\u0442\u044a"
131
+ ],
132
+ "training": {
133
+ "loss": 11.563977241516113,
134
+ "gradients": {
135
+ "model.layers.0.self_attn.q_proj.weight": true,
136
+ "model.layers.0.mlp.gate_proj.weight": true
137
+ },
138
+ "updated": true
139
+ }
140
+ }
141
+ }
integrations/transformers/expectations/cpu-arm64/DeepSeek-Coder-V2-Lite-Instruct-mic.eager.json ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/DeepSeek-Coder-V2-Lite-Instruct-mic",
4
+ "contract_name": "DeepSeek-Coder-V2-Lite-Instruct",
5
+ "contract_revision": "947b5968d33b8ab8f9079bf7d9bca75a4a788293",
6
+ "framework": "transformers",
7
+ "variant": "eager",
8
+ "target": "DeepSeek-Coder-V2-Lite-Instruct-mic",
9
+ "kind": "fixture",
10
+ "revision": "947b5968d33b8ab8f9079bf7d9bca75a4a788293",
11
+ "source_model": "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/DeepSeek-Coder-V2-Lite-Instruct-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.34,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "eager"
49
+ },
50
+ "logits": {
51
+ "probe": [
52
+ [
53
+ 0.025437403470277786,
54
+ -0.03097386471927166,
55
+ 0.057485003024339676,
56
+ -0.11214236915111542,
57
+ -0.02039257250726223,
58
+ 0.07542132586240768,
59
+ -0.04993953928351402,
60
+ 0.025900324806571007,
61
+ 0.1052633672952652,
62
+ -0.19907252490520477,
63
+ 0.3424341082572937,
64
+ 0.06264027208089828,
65
+ 0.03478936851024628,
66
+ 0.16950979828834534,
67
+ -0.13501432538032532,
68
+ 0.03707234933972359
69
+ ],
70
+ [
71
+ 0.05516336113214493,
72
+ -0.06818839907646179,
73
+ 0.057154491543769836,
74
+ -0.07518450915813446,
75
+ -0.054752059280872345,
76
+ 0.08047668635845184,
77
+ -0.03771958127617836,
78
+ -0.0021044586319476366,
79
+ 0.14207547903060913,
80
+ -0.2046119123697281,
81
+ 0.34193214774131775,
82
+ 0.03144151344895363,
83
+ 0.06359387189149857,
84
+ 0.22085553407669067,
85
+ -0.14136837422847748,
86
+ 0.0427250899374485
87
+ ]
88
+ ],
89
+ "position_norms": [
90
+ [
91
+ 51.13037872314453,
92
+ 51.13823699951172,
93
+ 51.08026885986328,
94
+ 51.15916442871094,
95
+ 51.22240447998047,
96
+ 51.217140197753906,
97
+ 51.22309875488281,
98
+ 51.217041015625,
99
+ 51.26316452026367,
100
+ 51.18793487548828,
101
+ 51.0853385925293,
102
+ 51.13606262207031,
103
+ 51.149906158447266,
104
+ 51.14849090576172,
105
+ 51.00004196166992,
106
+ 51.09321594238281
107
+ ],
108
+ [
109
+ 51.13037872314453,
110
+ 51.13823699951172,
111
+ 51.08026885986328,
112
+ 51.05168533325195,
113
+ 51.138710021972656,
114
+ 50.996341705322266,
115
+ 51.15065002441406,
116
+ 51.25377655029297,
117
+ 51.27322006225586,
118
+ 51.111854553222656,
119
+ 51.00624465942383,
120
+ 51.08974838256836,
121
+ 51.11851119995117,
122
+ 51.12657928466797,
123
+ 51.07209396362305,
124
+ 51.052490234375
125
+ ]
126
+ ]
127
+ },
128
+ "generated_text": [
129
+ " testim amountsomachrels ALTERAppl Recruitscanf",
130
+ " testim amounts\u51fa\u56fd surrendered Cartacustomers espacio\u0441\u0442\u044a"
131
+ ],
132
+ "training": {
133
+ "loss": 11.563955307006836,
134
+ "gradients": {
135
+ "model.layers.0.self_attn.q_proj.weight": true,
136
+ "model.layers.0.mlp.gate_proj.weight": true
137
+ },
138
+ "updated": true
139
+ }
140
+ }
141
+ }
integrations/transformers/expectations/cpu-arm64/DeepSeek-Coder-V2-Lite-Instruct-mic.json ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/DeepSeek-Coder-V2-Lite-Instruct-mic",
4
+ "contract_name": "DeepSeek-Coder-V2-Lite-Instruct",
5
+ "contract_revision": "947b5968d33b8ab8f9079bf7d9bca75a4a788293",
6
+ "framework": "transformers",
7
+ "variant": "default",
8
+ "target": "DeepSeek-Coder-V2-Lite-Instruct-mic",
9
+ "kind": "fixture",
10
+ "revision": "947b5968d33b8ab8f9079bf7d9bca75a4a788293",
11
+ "source_model": "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/DeepSeek-Coder-V2-Lite-Instruct-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.22,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "sdpa"
49
+ },
50
+ "logits": {
51
+ "probe": [
52
+ [
53
+ 0.02543741464614868,
54
+ -0.03097386285662651,
55
+ 0.057485029101371765,
56
+ -0.11214237660169601,
57
+ -0.020392565056681633,
58
+ 0.07542132586240768,
59
+ -0.049939535558223724,
60
+ 0.025900335982441902,
61
+ 0.1052633747458458,
62
+ -0.19907255470752716,
63
+ 0.3424341082572937,
64
+ 0.06264027208089828,
65
+ 0.03478936478495598,
66
+ 0.16950979828834534,
67
+ -0.13501432538032532,
68
+ 0.03707233443856239
69
+ ],
70
+ [
71
+ 0.055163364857435226,
72
+ -0.06818839907646179,
73
+ 0.05715448409318924,
74
+ -0.07518450170755386,
75
+ -0.05475204437971115,
76
+ 0.08047667145729065,
77
+ -0.03771958500146866,
78
+ -0.0021044653840363026,
79
+ 0.1420755684375763,
80
+ -0.20461192727088928,
81
+ 0.34193217754364014,
82
+ 0.03144150599837303,
83
+ 0.06359387189149857,
84
+ 0.22085557878017426,
85
+ -0.14136840403079987,
86
+ 0.042725104838609695
87
+ ]
88
+ ],
89
+ "position_norms": [
90
+ [
91
+ 51.13037872314453,
92
+ 51.13823699951172,
93
+ 51.08026885986328,
94
+ 51.15916061401367,
95
+ 51.222408294677734,
96
+ 51.217140197753906,
97
+ 51.22309875488281,
98
+ 51.217041015625,
99
+ 51.26316452026367,
100
+ 51.18793487548828,
101
+ 51.08534240722656,
102
+ 51.13606262207031,
103
+ 51.149906158447266,
104
+ 51.14849090576172,
105
+ 51.00004196166992,
106
+ 51.09321594238281
107
+ ],
108
+ [
109
+ 51.13037872314453,
110
+ 51.13823699951172,
111
+ 51.08026885986328,
112
+ 51.05168533325195,
113
+ 51.138702392578125,
114
+ 50.996341705322266,
115
+ 51.1506462097168,
116
+ 51.25376892089844,
117
+ 51.27322006225586,
118
+ 51.11185836791992,
119
+ 51.00624084472656,
120
+ 51.089752197265625,
121
+ 51.11851119995117,
122
+ 51.1265754699707,
123
+ 51.07209396362305,
124
+ 51.05249786376953
125
+ ]
126
+ ]
127
+ },
128
+ "generated_text": [
129
+ " testim amountsomachrels ALTERAppl Recruitscanf",
130
+ " testim amounts\u51fa\u56fd surrendered Cartacustomers espacio\u0441\u0442\u044a"
131
+ ],
132
+ "training": {
133
+ "loss": 11.563955307006836,
134
+ "gradients": {
135
+ "model.layers.0.self_attn.q_proj.weight": true,
136
+ "model.layers.0.mlp.gate_proj.weight": true
137
+ },
138
+ "updated": true
139
+ }
140
+ }
141
+ }