tarekziade HF Staff commited on
Commit
0c7f58a
·
verified ·
1 Parent(s): f158c55

Review cloud MIC baselines arm64-StarCoder2

Browse files

Measured and independently validated on CPU and CUDA. Review reasons: {"cpu/prototypes/StarCoder2/starcoder2-3b-mic/bfloat16": {"reason": "New or unchanged measured reference", "sha256": "4650c65dce24935d84f03d2be1d32c879c6e5db4cd4bd99f74e38b317f316299", "verdict": "new"}, "cpu/prototypes/StarCoder2/starcoder2-3b-mic/default": {"reason": "New or unchanged measured reference", "sha256": "c863fa9221f031543e5bf0d8bb335be7b11e3e476aa03d2d61fcdf96b4abc984", "verdict": "new"}, "cpu/prototypes/StarCoder2/starcoder2-3b-mic/eager": {"reason": "New or unchanged measured reference", "sha256": "bd87b783b88b723f359824273bfb7e079f4af25deeecc96538762f51ef8811d3", "verdict": "new"}}

integrations/transformers/expectations/cpu-arm64/starcoder2-3b-mic.bfloat16.json ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/starcoder2-3b-mic",
4
+ "contract_name": "StarCoder2",
5
+ "contract_revision": "f158c5574331920c1b49c8447d5bf4a4864df079",
6
+ "framework": "transformers",
7
+ "variant": "bfloat16",
8
+ "target": "starcoder2-3b-mic",
9
+ "kind": "fixture",
10
+ "revision": "f158c5574331920c1b49c8447d5bf4a4864df079",
11
+ "source_model": "bigcode/starcoder2-3b",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/starcoder2-3b-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.27,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "bfloat16",
48
+ "attn_implementation": "sdpa"
49
+ },
50
+ "next_token_ids": [
51
+ 32340,
52
+ 37963
53
+ ],
54
+ "next_token_text": [
55
+ "'(",
56
+ "\ud68c"
57
+ ],
58
+ "probe_logits": [
59
+ [
60
+ -0.10986328125,
61
+ -0.0849609375,
62
+ -0.142578125,
63
+ -0.1435546875,
64
+ -0.17578125
65
+ ],
66
+ [
67
+ -0.14453125,
68
+ -0.1435546875,
69
+ 0.1796875,
70
+ -0.09130859375,
71
+ -0.1904296875
72
+ ]
73
+ ],
74
+ "training": {
75
+ "loss": 10.828603744506836,
76
+ "gradients": {
77
+ "model.layers.0.self_attn.q_proj.weight": true,
78
+ "model.layers.0.mlp.c_fc.weight": true
79
+ },
80
+ "updated": true
81
+ }
82
+ }
83
+ }
integrations/transformers/expectations/cpu-arm64/starcoder2-3b-mic.eager.json ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/starcoder2-3b-mic",
4
+ "contract_name": "StarCoder2",
5
+ "contract_revision": "f158c5574331920c1b49c8447d5bf4a4864df079",
6
+ "framework": "transformers",
7
+ "variant": "eager",
8
+ "target": "starcoder2-3b-mic",
9
+ "kind": "fixture",
10
+ "revision": "f158c5574331920c1b49c8447d5bf4a4864df079",
11
+ "source_model": "bigcode/starcoder2-3b",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/starcoder2-3b-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.12,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "eager"
49
+ },
50
+ "next_token_ids": [
51
+ 32340,
52
+ 37963
53
+ ],
54
+ "next_token_text": [
55
+ "'(",
56
+ "\ud68c"
57
+ ],
58
+ "probe_logits": [
59
+ [
60
+ -0.10958767682313919,
61
+ -0.08578762412071228,
62
+ -0.14188803732395172,
63
+ -0.14319221675395966,
64
+ -0.17468278110027313
65
+ ],
66
+ [
67
+ -0.14452381432056427,
68
+ -0.14295387268066406,
69
+ 0.17886754870414734,
70
+ -0.09121350198984146,
71
+ -0.19060996174812317
72
+ ]
73
+ ],
74
+ "training": {
75
+ "loss": 10.830249786376953,
76
+ "gradients": {
77
+ "model.layers.0.self_attn.q_proj.weight": true,
78
+ "model.layers.0.mlp.c_fc.weight": true
79
+ },
80
+ "updated": true
81
+ }
82
+ }
83
+ }
integrations/transformers/expectations/cpu-arm64/starcoder2-3b-mic.json ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/starcoder2-3b-mic",
4
+ "contract_name": "StarCoder2",
5
+ "contract_revision": "f158c5574331920c1b49c8447d5bf4a4864df079",
6
+ "framework": "transformers",
7
+ "variant": "default",
8
+ "target": "starcoder2-3b-mic",
9
+ "kind": "fixture",
10
+ "revision": "f158c5574331920c1b49c8447d5bf4a4864df079",
11
+ "source_model": "bigcode/starcoder2-3b",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/starcoder2-3b-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.12,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "sdpa"
49
+ },
50
+ "next_token_ids": [
51
+ 32340,
52
+ 37963
53
+ ],
54
+ "next_token_text": [
55
+ "'(",
56
+ "\ud68c"
57
+ ],
58
+ "probe_logits": [
59
+ [
60
+ -0.10958768427371979,
61
+ -0.08578766882419586,
62
+ -0.14188802242279053,
63
+ -0.14319227635860443,
64
+ -0.1746828258037567
65
+ ],
66
+ [
67
+ -0.14452384412288666,
68
+ -0.14295388758182526,
69
+ 0.17886756360530853,
70
+ -0.09121348708868027,
71
+ -0.19060996174812317
72
+ ]
73
+ ],
74
+ "training": {
75
+ "loss": 10.828756332397461,
76
+ "gradients": {
77
+ "model.layers.0.self_attn.q_proj.weight": true,
78
+ "model.layers.0.mlp.c_fc.weight": true
79
+ },
80
+ "updated": true
81
+ }
82
+ }
83
+ }