tarekziade HF Staff commited on
Commit
371eafe
·
verified ·
1 Parent(s): 2cc72c0

Review cloud MIC baselines arm64-DeepSeek-R1

Browse files

Measured and independently validated on CPU and CUDA. Review reasons: {"cpu/prototypes/DeepSeek-R1/DeepSeek-R1-mic/bfloat16": {"reason": "New or unchanged measured reference", "sha256": "f33cde739ee3d1ceda21a9f30f37f5ddee6898dc1f40f7df5b35bdcebf615fb7", "verdict": "new"}, "cpu/prototypes/DeepSeek-R1/DeepSeek-R1-mic/default": {"reason": "New or unchanged measured reference", "sha256": "b7fb13f94b0bbc0f04e59be84c42129d2332607c55e51b4b6e688cdce62cb4de", "verdict": "new"}, "cpu/prototypes/DeepSeek-R1/DeepSeek-R1-mic/eager": {"reason": "New or unchanged measured reference", "sha256": "8a254d8d3beb35c7232480585bb90ce4c4046f90f582495d5d497fb97e9f465e", "verdict": "new"}}

integrations/transformers/expectations/cpu-arm64/DeepSeek-R1-mic.bfloat16.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/DeepSeek-R1-mic",
4
+ "contract_name": "DeepSeek-R1",
5
+ "contract_revision": "2cc72c0556b8bf70aa8fd98e4200c2b01190ddc8",
6
+ "framework": "transformers",
7
+ "variant": "bfloat16",
8
+ "target": "DeepSeek-R1-mic",
9
+ "kind": "fixture",
10
+ "revision": "2cc72c0556b8bf70aa8fd98e4200c2b01190ddc8",
11
+ "source_model": "deepseek-ai/DeepSeek-R1",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/DeepSeek-R1-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.37,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "bfloat16",
48
+ "attn_implementation": "sdpa"
49
+ },
50
+ "next_token_ids": [
51
+ 119704,
52
+ 5475
53
+ ],
54
+ "probe_logits": [
55
+ [
56
+ -0.1796875,
57
+ -0.10888671875,
58
+ -0.10986328125,
59
+ -0.1669921875,
60
+ 0.205078125,
61
+ -0.1767578125,
62
+ 0.10400390625,
63
+ 0.05224609375,
64
+ -0.00946044921875,
65
+ 0.1162109375,
66
+ -0.06298828125,
67
+ -0.03759765625,
68
+ -0.09326171875,
69
+ 0.2275390625,
70
+ -0.1005859375,
71
+ 0.068359375
72
+ ],
73
+ [
74
+ -0.2119140625,
75
+ -0.07568359375,
76
+ -0.0966796875,
77
+ -0.14453125,
78
+ 0.21875,
79
+ -0.26953125,
80
+ 0.1435546875,
81
+ 0.043701171875,
82
+ 0.0037841796875,
83
+ 0.12890625,
84
+ -0.140625,
85
+ 0.0140380859375,
86
+ -0.140625,
87
+ 0.203125,
88
+ -0.12255859375,
89
+ 0.09033203125
90
+ ]
91
+ ],
92
+ "position_norms": [
93
+ [
94
+ 57.44831085205078,
95
+ 57.26078796386719,
96
+ 57.43337631225586,
97
+ 57.489166259765625,
98
+ 57.36770248413086
99
+ ],
100
+ [
101
+ 57.44831085205078,
102
+ 57.26078796386719,
103
+ 57.43337631225586,
104
+ 57.506778717041016,
105
+ 57.35624313354492
106
+ ]
107
+ ],
108
+ "training": {
109
+ "loss": 11.722225189208984,
110
+ "gradients": {
111
+ "model.layers.0.self_attn.q_a_proj.weight": true,
112
+ "model.layers.0.mlp.gate_proj.weight": true
113
+ },
114
+ "updated": true
115
+ }
116
+ }
117
+ }
integrations/transformers/expectations/cpu-arm64/DeepSeek-R1-mic.eager.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/DeepSeek-R1-mic",
4
+ "contract_name": "DeepSeek-R1",
5
+ "contract_revision": "2cc72c0556b8bf70aa8fd98e4200c2b01190ddc8",
6
+ "framework": "transformers",
7
+ "variant": "eager",
8
+ "target": "DeepSeek-R1-mic",
9
+ "kind": "fixture",
10
+ "revision": "2cc72c0556b8bf70aa8fd98e4200c2b01190ddc8",
11
+ "source_model": "deepseek-ai/DeepSeek-R1",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/DeepSeek-R1-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.32,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "bfloat16",
48
+ "attn_implementation": "eager"
49
+ },
50
+ "next_token_ids": [
51
+ 119704,
52
+ 5475
53
+ ],
54
+ "probe_logits": [
55
+ [
56
+ -0.1787109375,
57
+ -0.10888671875,
58
+ -0.109375,
59
+ -0.16796875,
60
+ 0.203125,
61
+ -0.1767578125,
62
+ 0.103515625,
63
+ 0.052734375,
64
+ -0.0089111328125,
65
+ 0.11669921875,
66
+ -0.0625,
67
+ -0.03662109375,
68
+ -0.09326171875,
69
+ 0.2275390625,
70
+ -0.10205078125,
71
+ 0.06787109375
72
+ ],
73
+ [
74
+ -0.2119140625,
75
+ -0.07666015625,
76
+ -0.09716796875,
77
+ -0.14453125,
78
+ 0.2177734375,
79
+ -0.267578125,
80
+ 0.14453125,
81
+ 0.044189453125,
82
+ 0.00494384765625,
83
+ 0.12890625,
84
+ -0.1416015625,
85
+ 0.0155029296875,
86
+ -0.1416015625,
87
+ 0.203125,
88
+ -0.123046875,
89
+ 0.08984375
90
+ ]
91
+ ],
92
+ "position_norms": [
93
+ [
94
+ 57.44831085205078,
95
+ 57.248531341552734,
96
+ 57.43209457397461,
97
+ 57.5039176940918,
98
+ 57.39615249633789
99
+ ],
100
+ [
101
+ 57.44831085205078,
102
+ 57.248531341552734,
103
+ 57.43209457397461,
104
+ 57.499412536621094,
105
+ 57.37013244628906
106
+ ]
107
+ ],
108
+ "training": {
109
+ "loss": 11.722163200378418,
110
+ "gradients": {
111
+ "model.layers.0.self_attn.q_a_proj.weight": true,
112
+ "model.layers.0.mlp.gate_proj.weight": true
113
+ },
114
+ "updated": true
115
+ }
116
+ }
117
+ }
integrations/transformers/expectations/cpu-arm64/DeepSeek-R1-mic.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/DeepSeek-R1-mic",
4
+ "contract_name": "DeepSeek-R1",
5
+ "contract_revision": "2cc72c0556b8bf70aa8fd98e4200c2b01190ddc8",
6
+ "framework": "transformers",
7
+ "variant": "default",
8
+ "target": "DeepSeek-R1-mic",
9
+ "kind": "fixture",
10
+ "revision": "2cc72c0556b8bf70aa8fd98e4200c2b01190ddc8",
11
+ "source_model": "deepseek-ai/DeepSeek-R1",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/DeepSeek-R1-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.35,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "bfloat16",
48
+ "attn_implementation": "sdpa"
49
+ },
50
+ "next_token_ids": [
51
+ 119704,
52
+ 5475
53
+ ],
54
+ "probe_logits": [
55
+ [
56
+ -0.1796875,
57
+ -0.10888671875,
58
+ -0.10986328125,
59
+ -0.1669921875,
60
+ 0.205078125,
61
+ -0.1767578125,
62
+ 0.10400390625,
63
+ 0.05224609375,
64
+ -0.00946044921875,
65
+ 0.1162109375,
66
+ -0.06298828125,
67
+ -0.03759765625,
68
+ -0.09326171875,
69
+ 0.2275390625,
70
+ -0.1005859375,
71
+ 0.068359375
72
+ ],
73
+ [
74
+ -0.2119140625,
75
+ -0.07568359375,
76
+ -0.0966796875,
77
+ -0.14453125,
78
+ 0.21875,
79
+ -0.26953125,
80
+ 0.1435546875,
81
+ 0.043701171875,
82
+ 0.0037841796875,
83
+ 0.12890625,
84
+ -0.140625,
85
+ 0.0140380859375,
86
+ -0.140625,
87
+ 0.203125,
88
+ -0.12255859375,
89
+ 0.09033203125
90
+ ]
91
+ ],
92
+ "position_norms": [
93
+ [
94
+ 57.44831085205078,
95
+ 57.26078796386719,
96
+ 57.43337631225586,
97
+ 57.489166259765625,
98
+ 57.36770248413086
99
+ ],
100
+ [
101
+ 57.44831085205078,
102
+ 57.26078796386719,
103
+ 57.43337631225586,
104
+ 57.506778717041016,
105
+ 57.35624313354492
106
+ ]
107
+ ],
108
+ "training": {
109
+ "loss": 11.722225189208984,
110
+ "gradients": {
111
+ "model.layers.0.self_attn.q_a_proj.weight": true,
112
+ "model.layers.0.mlp.gate_proj.weight": true
113
+ },
114
+ "updated": true
115
+ }
116
+ }
117
+ }