File size: 9,607 Bytes
78710b2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
layer,module,loss,samples,damp,time
0,linear_attn.in_proj_qkv,0.0001062057,0.05000,1.061
0,linear_attn.in_proj_z,0.0000624540,0.05000,0.364
0,linear_attn.out_proj,0.0000000647,0.05000,0.582
0,mlp.up_proj,0.0000038046,0.05000,1.441
0,mlp.gate_proj,0.0000043560,0.05000,1.430
0,mlp.down_proj,0.0000000069,0.05000,1.366
1,linear_attn.in_proj_qkv,0.0000309942,0.05000,0.906
1,linear_attn.in_proj_z,0.0000158869,0.05000,0.492
1,linear_attn.out_proj,0.0000000166,0.05000,0.578
1,mlp.up_proj,0.0000080042,0.05000,1.391
1,mlp.gate_proj,0.0000093789,0.05000,1.399
1,mlp.down_proj,0.0000000203,0.05000,1.353
2,linear_attn.in_proj_qkv,0.0000507535,0.05000,0.968
2,linear_attn.in_proj_z,0.0000259517,0.05000,0.435
2,linear_attn.out_proj,0.0000000304,0.05000,0.579
2,mlp.up_proj,0.0000108939,0.05000,1.365
2,mlp.gate_proj,0.0000146012,0.05000,1.372
2,mlp.down_proj,0.0000000359,0.05000,1.316
3,self_attn.v_proj,0.0000160838,0.05000,3.799
3,self_attn.k_proj,0.0000148501,0.05000,3.826
3,self_attn.q_proj,0.0001720952,0.05000,3.829
3,self_attn.o_proj,0.0000000644,0.05000,0.577
3,mlp.gate_proj,0.0000179513,0.05000,1.362
3,mlp.up_proj,0.0000151137,0.05000,1.369
3,mlp.down_proj,0.0000000559,0.05000,1.315
4,linear_attn.in_proj_qkv,0.0000823915,0.05000,0.814
4,linear_attn.in_proj_z,0.0000438157,0.05000,0.387
4,linear_attn.out_proj,0.0000000666,0.05000,0.569
4,mlp.up_proj,0.0000227899,0.05000,1.379
4,mlp.gate_proj,0.0000272993,0.05000,1.381
4,mlp.down_proj,0.0000000906,0.05000,1.328
5,linear_attn.in_proj_qkv,0.0001031028,0.05000,0.793
5,linear_attn.in_proj_z,0.0000539819,0.05000,0.418
5,linear_attn.out_proj,0.0000001276,0.05000,0.571
5,mlp.up_proj,0.0000293930,0.05000,1.275
5,mlp.gate_proj,0.0000382961,0.05000,1.283
5,mlp.down_proj,0.0000001699,0.05000,1.298
6,linear_attn.in_proj_qkv,0.0001076786,0.05000,0.778
6,linear_attn.in_proj_z,0.0000463039,0.05000,0.416
6,linear_attn.out_proj,0.0000002121,0.05000,0.564
6,mlp.up_proj,0.0000342711,0.05000,1.138
6,mlp.gate_proj,0.0000486841,0.05000,1.145
6,mlp.down_proj,0.0000002166,0.05000,1.309
7,self_attn.v_proj,0.0000125313,0.05000,3.237
7,self_attn.k_proj,0.0000123067,0.05000,3.288
7,self_attn.q_proj,0.0001181358,0.05000,3.298
7,self_attn.o_proj,0.0000003714,0.05000,0.565
7,mlp.up_proj,0.0000372502,0.05000,1.216
7,mlp.gate_proj,0.0000561754,0.05000,1.225
7,mlp.down_proj,0.0000002302,0.05000,1.300
8,linear_attn.in_proj_qkv,0.0001202234,0.05000,0.865
8,linear_attn.in_proj_z,0.0000546115,0.05000,0.350
8,linear_attn.out_proj,0.0000002335,0.05000,0.563
8,mlp.gate_proj,0.0000536431,0.05000,1.141
8,mlp.up_proj,0.0000384764,0.05000,1.147
8,mlp.down_proj,0.0000002216,0.05000,1.306
9,linear_attn.in_proj_qkv,0.0001260519,0.05000,0.766
9,linear_attn.in_proj_z,0.0000552407,0.05000,0.381
9,linear_attn.out_proj,0.0000002220,0.05000,0.571
9,mlp.up_proj,0.0000399238,0.05000,1.182
9,mlp.gate_proj,0.0000504698,0.05000,1.190
9,mlp.down_proj,0.0000002394,0.05000,1.298
10,linear_attn.in_proj_qkv,0.0001189474,0.05000,0.788
10,linear_attn.in_proj_z,0.0000511090,0.05000,0.352
10,linear_attn.out_proj,0.0000002361,0.05000,0.563
10,mlp.up_proj,0.0000417634,0.05000,1.142
10,mlp.gate_proj,0.0000469903,0.05000,1.146
10,mlp.down_proj,0.0000002582,0.05000,1.299
11,self_attn.k_proj,0.0000110828,0.05000,3.111
11,self_attn.v_proj,0.0000119973,0.05000,3.167
11,self_attn.q_proj,0.0001007621,0.05000,3.159
11,self_attn.o_proj,0.0000005383,0.05000,0.558
11,mlp.gate_proj,0.0000488222,0.05000,1.147
11,mlp.up_proj,0.0000459833,0.05000,1.154
11,mlp.down_proj,0.0000003192,0.05000,1.306
12,linear_attn.in_proj_qkv,0.0001304546,0.05000,0.820
12,linear_attn.in_proj_z,0.0000497158,0.05000,0.351
12,linear_attn.out_proj,0.0000003005,0.05000,0.567
12,mlp.up_proj,0.0000468588,0.05000,1.210
12,mlp.gate_proj,0.0000491226,0.05000,1.216
12,mlp.down_proj,0.0000003481,0.05000,1.330
13,linear_attn.in_proj_qkv,0.0001503557,0.05000,0.762
13,linear_attn.in_proj_z,0.0000632949,0.05000,0.403
13,linear_attn.out_proj,0.0000002876,0.05000,0.565
13,mlp.gate_proj,0.0000527054,0.05000,1.201
13,mlp.up_proj,0.0000498222,0.05000,1.208
13,mlp.down_proj,0.0000003851,0.05000,1.318
14,linear_attn.in_proj_qkv,0.0001460825,0.05000,0.747
14,linear_attn.in_proj_z,0.0000584706,0.05000,0.415
14,linear_attn.out_proj,0.0000003919,0.05000,0.564
14,mlp.gate_proj,0.0000511832,0.05000,1.202
14,mlp.up_proj,0.0000509041,0.05000,1.207
14,mlp.down_proj,0.0000004046,0.05000,1.315
15,self_attn.k_proj,0.0000126020,0.05000,3.257
15,self_attn.v_proj,0.0000171143,0.05000,3.285
15,self_attn.q_proj,0.0001148908,0.05000,3.292
15,self_attn.o_proj,0.0000007863,0.05000,0.562
15,mlp.gate_proj,0.0000599797,0.05000,1.140
15,mlp.up_proj,0.0000603494,0.05000,1.144
15,mlp.down_proj,0.0000006153,0.05000,1.306
16,linear_attn.in_proj_qkv,0.0001585772,0.05000,0.812
16,linear_attn.in_proj_z,0.0000649396,0.05000,0.350
16,linear_attn.out_proj,0.0000005524,0.05000,0.561
16,mlp.up_proj,0.0000652249,0.05000,1.204
16,mlp.gate_proj,0.0000679017,0.05000,1.209
16,mlp.down_proj,0.0000007760,0.05000,1.310
17,linear_attn.in_proj_qkv,0.0002137600,0.05000,0.785
17,linear_attn.in_proj_z,0.0000729942,0.05000,0.419
17,linear_attn.out_proj,0.0000006821,0.05000,0.586
17,mlp.up_proj,0.0000701895,0.05000,1.258
17,mlp.gate_proj,0.0000713158,0.05000,1.264
17,mlp.down_proj,0.0000009175,0.05000,1.314
18,linear_attn.in_proj_qkv,0.0001936378,0.05000,0.799
18,linear_attn.in_proj_z,0.0000768392,0.05000,0.401
18,linear_attn.out_proj,0.0000009189,0.05000,0.560
18,mlp.up_proj,0.0000767932,0.05000,1.166
18,mlp.gate_proj,0.0000834938,0.05000,1.172
18,mlp.down_proj,0.0000015128,0.05000,1.297
19,self_attn.k_proj,0.0000178808,0.05000,3.661
19,self_attn.q_proj,0.0001621967,0.05000,3.671
19,self_attn.v_proj,0.0000336217,0.05000,3.681
19,self_attn.o_proj,0.0000024908,0.05000,0.572
19,mlp.up_proj,0.0000974410,0.05000,1.280
19,mlp.gate_proj,0.0001058582,0.05000,1.283
19,mlp.down_proj,0.0000020646,0.05000,1.336
20,linear_attn.in_proj_qkv,0.0003080268,0.05000,0.816
20,linear_attn.in_proj_z,0.0001266365,0.05000,0.392
20,linear_attn.out_proj,0.0000010720,0.05000,0.569
20,mlp.gate_proj,0.0001360129,0.05000,1.416
20,mlp.up_proj,0.0000936365,0.05000,1.422
20,mlp.down_proj,0.0000018920,0.05000,1.343
21,linear_attn.in_proj_qkv,0.0003111902,0.05000,0.832
21,linear_attn.in_proj_z,0.0001392197,0.05000,0.374
21,linear_attn.out_proj,0.0000012129,0.05000,0.566
21,mlp.up_proj,0.0000973126,0.05000,1.193
21,mlp.gate_proj,0.0001503969,0.05000,1.201
21,mlp.down_proj,0.0000022277,0.05000,1.310
22,linear_attn.in_proj_qkv,0.0002915294,0.05000,0.770
22,linear_attn.in_proj_z,0.0001620547,0.05000,0.423
22,linear_attn.out_proj,0.0000013246,0.05000,0.558
22,mlp.gate_proj,0.0002069766,0.05000,1.145
22,mlp.up_proj,0.0001170133,0.05000,1.151
22,mlp.down_proj,0.0000033070,0.05000,1.306
23,self_attn.k_proj,0.0000193849,0.05000,3.592
23,self_attn.v_proj,0.0000385716,0.05000,3.610
23,self_attn.q_proj,0.0001964796,0.05000,3.613
23,self_attn.o_proj,0.0000024868,0.05000,0.565
23,mlp.gate_proj,0.0001874615,0.05000,1.199
23,mlp.up_proj,0.0001190553,0.05000,1.204
23,mlp.down_proj,0.0000027940,0.05000,1.327
24,linear_attn.in_proj_qkv,0.0003445098,0.05000,0.910
24,linear_attn.in_proj_z,0.0001590758,0.05000,0.431
24,linear_attn.out_proj,0.0000014111,0.05000,0.571
24,mlp.gate_proj,0.0001981594,0.05000,1.324
24,mlp.up_proj,0.0001155888,0.05000,1.332
24,mlp.down_proj,0.0000023850,0.05000,1.307
25,linear_attn.in_proj_qkv,0.0003647602,0.05000,0.877
25,linear_attn.in_proj_z,0.0001737609,0.05000,0.432
25,linear_attn.out_proj,0.0000014427,0.05000,0.565
25,mlp.gate_proj,0.0002334497,0.05000,1.368
25,mlp.up_proj,0.0001262979,0.05000,1.373
25,mlp.down_proj,0.0000025405,0.05000,1.322
26,linear_attn.in_proj_qkv,0.0003223225,0.05000,0.954
26,linear_attn.in_proj_z,0.0001861227,0.05000,0.452
26,linear_attn.out_proj,0.0000017613,0.05000,0.579
26,mlp.up_proj,0.0001477981,0.05000,1.330
26,mlp.gate_proj,0.0002823460,0.05000,1.337
26,mlp.down_proj,0.0000037028,0.05000,1.308
27,self_attn.k_proj,0.0000215237,0.05000,3.941
27,self_attn.q_proj,0.0001924323,0.05000,3.980
27,self_attn.v_proj,0.0000818426,0.05000,4.015
27,self_attn.o_proj,0.0000055045,0.05000,0.559
27,mlp.up_proj,0.0001504280,0.05000,1.406
27,mlp.gate_proj,0.0002794680,0.05000,1.414
27,mlp.down_proj,0.0000048705,0.05000,1.327
28,linear_attn.in_proj_qkv,0.0003944376,0.05000,0.845
28,linear_attn.in_proj_z,0.0001802195,0.05000,0.379
28,linear_attn.out_proj,0.0000038920,0.05000,0.562
28,mlp.up_proj,0.0001510296,0.05000,1.179
28,mlp.gate_proj,0.0002722793,0.05000,1.186
28,mlp.down_proj,0.0000059651,0.05000,1.307
29,linear_attn.in_proj_qkv,0.0002864954,0.05000,0.880
29,linear_attn.in_proj_z,0.0001608498,0.05000,0.433
29,linear_attn.out_proj,0.0000039612,0.05000,0.576
29,mlp.up_proj,0.0001665302,0.05000,1.413
29,mlp.gate_proj,0.0002985448,0.05000,1.421
29,mlp.down_proj,0.0000078739,0.05000,1.322
30,linear_attn.in_proj_qkv,0.0003061922,0.05000,0.968
30,linear_attn.in_proj_z,0.0001411075,0.05000,0.361
30,linear_attn.out_proj,0.0000104969,0.05000,0.575
30,mlp.up_proj,0.0001648950,0.05000,1.408
30,mlp.gate_proj,0.0002782126,0.05000,1.409
30,mlp.down_proj,0.0000105565,0.05000,1.337
31,self_attn.v_proj,0.0000769979,0.05000,4.278
31,self_attn.k_proj,0.0000222255,0.05000,4.300
31,self_attn.q_proj,0.0002227224,0.05000,4.318
31,self_attn.o_proj,0.0000141060,0.05000,0.566
31,mlp.up_proj,0.0001564013,0.05000,1.214
31,mlp.gate_proj,0.0002541700,0.05000,1.221
31,mlp.down_proj,0.0000204349,0.05000,1.304