Review cloud MIC baselines arm64-Inkling

#1
by tarekziade HF Staff - opened
integrations/transformers/expectations/cpu-arm64/Inkling-mic.bfloat16.json ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/Inkling-mic",
4
+ "contract_name": "Inkling",
5
+ "contract_revision": "06ce3d54ff614c8b427f9bc8f28b677c75875b2c",
6
+ "framework": "transformers",
7
+ "variant": "bfloat16",
8
+ "target": "Inkling-mic",
9
+ "kind": "fixture",
10
+ "revision": "06ce3d54ff614c8b427f9bc8f28b677c75875b2c",
11
+ "source_model": "thinkingmachines/Inkling",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/Inkling-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 3.19,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "bfloat16",
48
+ "attn_implementation": "sdpa"
49
+ },
50
+ "generated_ids": [
51
+ [
52
+ 61443,
53
+ 290,
54
+ 11180,
55
+ 306,
56
+ 1001,
57
+ 21872,
58
+ 13,
59
+ 82403,
60
+ 136293,
61
+ 83424,
62
+ 147813,
63
+ 353,
64
+ 100962,
65
+ 193230,
66
+ 155848
67
+ ],
68
+ [
69
+ 4827,
70
+ 7351,
71
+ 382,
72
+ 306,
73
+ 290,
74
+ 19925,
75
+ 30,
76
+ 1323,
77
+ 6261,
78
+ 149312,
79
+ 8504,
80
+ 184078,
81
+ 170383,
82
+ 139854,
83
+ 133321
84
+ ]
85
+ ],
86
+ "logits_probe": [
87
+ [
88
+ -0.00098419189453125,
89
+ 0.003631591796875,
90
+ -0.005615234375,
91
+ -0.0035552978515625,
92
+ 0.00078582763671875
93
+ ],
94
+ [
95
+ 0.0037384033203125,
96
+ 0.0081787109375,
97
+ -0.00135040283203125,
98
+ 0.00457763671875,
99
+ -0.00506591796875
100
+ ]
101
+ ],
102
+ "training": {
103
+ "loss": 12.205510139465332,
104
+ "gradients": {
105
+ "model.language_model.layers.0.self_attn.q_proj.weight": true,
106
+ "model.language_model.layers.0.mlp.up_proj.weight": true,
107
+ "lm_head.weight": true
108
+ },
109
+ "updated": true
110
+ }
111
+ }
112
+ }
integrations/transformers/expectations/cpu-arm64/Inkling-mic.eager.json ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/Inkling-mic",
4
+ "contract_name": "Inkling",
5
+ "contract_revision": "06ce3d54ff614c8b427f9bc8f28b677c75875b2c",
6
+ "framework": "transformers",
7
+ "variant": "eager",
8
+ "target": "Inkling-mic",
9
+ "kind": "fixture",
10
+ "revision": "06ce3d54ff614c8b427f9bc8f28b677c75875b2c",
11
+ "source_model": "thinkingmachines/Inkling",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/Inkling-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.87,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "eager"
49
+ },
50
+ "generated_ids": [
51
+ [
52
+ 61443,
53
+ 290,
54
+ 11180,
55
+ 306,
56
+ 1001,
57
+ 21872,
58
+ 13,
59
+ 82403,
60
+ 136293,
61
+ 83424,
62
+ 147813,
63
+ 353,
64
+ 100962,
65
+ 193230,
66
+ 155848
67
+ ],
68
+ [
69
+ 4827,
70
+ 7351,
71
+ 382,
72
+ 306,
73
+ 290,
74
+ 19925,
75
+ 30,
76
+ 1323,
77
+ 110635,
78
+ 142845,
79
+ 180914,
80
+ 166086,
81
+ 46238,
82
+ 116598,
83
+ 75536
84
+ ]
85
+ ],
86
+ "logits_probe": [
87
+ [
88
+ -0.0009258222416974604,
89
+ 0.003660770831629634,
90
+ -0.005591022316366434,
91
+ -0.003639644244685769,
92
+ 0.0007681713323108852
93
+ ],
94
+ [
95
+ 0.0037533966824412346,
96
+ 0.00815436989068985,
97
+ -0.00134138239081949,
98
+ 0.004615881480276585,
99
+ -0.005021501332521439
100
+ ]
101
+ ],
102
+ "training": {
103
+ "loss": 12.205512046813965,
104
+ "gradients": {
105
+ "model.language_model.layers.0.self_attn.q_proj.weight": true,
106
+ "model.language_model.layers.0.mlp.up_proj.weight": true,
107
+ "lm_head.weight": true
108
+ },
109
+ "updated": true
110
+ }
111
+ }
112
+ }
integrations/transformers/expectations/cpu-arm64/Inkling-mic.json ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "recorded_with": {
3
+ "contract": "huggingface-mic/Inkling-mic",
4
+ "contract_name": "Inkling",
5
+ "contract_revision": "06ce3d54ff614c8b427f9bc8f28b677c75875b2c",
6
+ "framework": "transformers",
7
+ "variant": "default",
8
+ "target": "Inkling-mic",
9
+ "kind": "fixture",
10
+ "revision": "06ce3d54ff614c8b427f9bc8f28b677c75875b2c",
11
+ "source_model": "thinkingmachines/Inkling",
12
+ "weights": "random",
13
+ "fixture_repo": "huggingface-mic/Inkling-mic",
14
+ "environment": {
15
+ "python": "3.11.14",
16
+ "transformers": "5.23.0.dev0",
17
+ "sentence-transformers": null,
18
+ "torch": "2.15.0",
19
+ "torchvision": "0.29.2",
20
+ "torchaudio": "2.11.1",
21
+ "tokenizers": "0.23.3",
22
+ "numpy": "2.3.5",
23
+ "pillow": "13.0.0",
24
+ "accelerate": "1.15.0",
25
+ "safetensors": "0.8.0",
26
+ "huggingface-hub": "2.4.0",
27
+ "sentencepiece": "0.2.2",
28
+ "protobuf": "7.36.2",
29
+ "librosa": "0.11.0",
30
+ "soundfile": "0.14.0",
31
+ "scipy": "1.17.1"
32
+ },
33
+ "seconds": 2.93,
34
+ "environment_differs": {
35
+ "torch": "2.15.0 (pinned 2.14.0)",
36
+ "torchvision": "0.29.2 (pinned 0.29.1)",
37
+ "torchaudio": "2.11.1 (pinned 2.11.0)",
38
+ "pillow": "13.0.0 (pinned 12.3.0)",
39
+ "huggingface-hub": "2.4.0 (pinned 2.3.0)"
40
+ },
41
+ "platform": "cpu-arm64",
42
+ "template_date": "2026-01-01"
43
+ },
44
+ "outputs": {
45
+ "runtime": {
46
+ "device": "cpu",
47
+ "dtype": "float32",
48
+ "attn_implementation": "sdpa"
49
+ },
50
+ "generated_ids": [
51
+ [
52
+ 61443,
53
+ 290,
54
+ 11180,
55
+ 306,
56
+ 1001,
57
+ 21872,
58
+ 13,
59
+ 82403,
60
+ 136293,
61
+ 83424,
62
+ 147813,
63
+ 353,
64
+ 100962,
65
+ 193230,
66
+ 155848
67
+ ],
68
+ [
69
+ 4827,
70
+ 7351,
71
+ 382,
72
+ 306,
73
+ 290,
74
+ 19925,
75
+ 30,
76
+ 1323,
77
+ 110635,
78
+ 142845,
79
+ 180914,
80
+ 166086,
81
+ 46238,
82
+ 116598,
83
+ 75536
84
+ ]
85
+ ],
86
+ "logits_probe": [
87
+ [
88
+ -0.0009258222416974604,
89
+ 0.003660770831629634,
90
+ -0.005591022316366434,
91
+ -0.003639644244685769,
92
+ 0.0007681713323108852
93
+ ],
94
+ [
95
+ 0.0037533966824412346,
96
+ 0.00815436989068985,
97
+ -0.00134138239081949,
98
+ 0.004615881480276585,
99
+ -0.005021501332521439
100
+ ]
101
+ ],
102
+ "training": {
103
+ "loss": 12.205512046813965,
104
+ "gradients": {
105
+ "model.language_model.layers.0.self_attn.q_proj.weight": true,
106
+ "model.language_model.layers.0.mlp.up_proj.weight": true,
107
+ "lm_head.weight": true
108
+ },
109
+ "updated": true
110
+ }
111
+ }
112
+ }