File size: 11,667 Bytes
9e1c36a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
layer,module,loss,samples,damp,time
0,self_attn.k_proj,0.0000000000,0.01000,1.664
0,self_attn.v_proj,0.0000000000,0.01000,1.557
0,self_attn.q_proj,0.0000000001,0.01000,1.610
0,self_attn.o_proj,0.0000000001,0.01000,1.505
0,mlp.gate_proj,0.0000000080,0.01000,1.580
0,mlp.up_proj,0.0000000068,0.01000,1.611
0,mlp.down_proj,0.0000000023,0.01000,4.940
1,self_attn.k_proj,0.0000000001,0.01000,1.493
1,self_attn.v_proj,0.0000000001,0.01000,1.533
1,self_attn.q_proj,0.0000000002,0.01000,1.601
1,self_attn.o_proj,0.0000000003,0.01000,1.553
1,mlp.gate_proj,0.0000002378,0.01000,1.564
1,mlp.up_proj,0.0000000745,0.01000,1.616
1,mlp.down_proj,0.0000000021,0.01000,4.933
2,self_attn.k_proj,0.0000000001,0.01000,1.508
2,self_attn.v_proj,0.0000000001,0.01000,1.539
2,self_attn.q_proj,0.0000000005,0.01000,1.616
2,self_attn.o_proj,0.0000000003,0.01000,1.500
2,mlp.gate_proj,0.0000003614,0.01000,1.568
2,mlp.up_proj,0.0000002598,0.01000,1.602
2,mlp.down_proj,0.0000000021,0.01000,4.974
3,self_attn.k_proj,0.0000000002,0.01000,1.508
3,self_attn.v_proj,0.0000000003,0.01000,1.554
3,self_attn.q_proj,0.0000000009,0.01000,1.610
3,self_attn.o_proj,0.0000000005,0.01000,1.522
3,mlp.gate_proj,0.0000003603,0.01000,1.606
3,mlp.up_proj,0.0000001621,0.01000,1.621
3,mlp.down_proj,0.0000000057,0.01000,4.931
4,self_attn.k_proj,0.0000000005,0.01000,1.501
4,self_attn.v_proj,0.0000000005,0.01000,1.533
4,self_attn.q_proj,0.0000000018,0.01000,1.641
4,self_attn.o_proj,0.0000000009,0.01000,1.514
4,mlp.gate_proj,0.0000005436,0.01000,1.661
4,mlp.up_proj,0.0000001941,0.01000,1.621
4,mlp.down_proj,0.0000000103,0.01000,4.976
5,self_attn.k_proj,0.0000000005,0.01000,1.514
5,self_attn.v_proj,0.0000000006,0.01000,1.540
5,self_attn.q_proj,0.0000000020,0.01000,1.615
5,self_attn.o_proj,0.0000000014,0.01000,1.529
5,mlp.gate_proj,0.0000002218,0.01000,1.574
5,mlp.up_proj,0.0000000955,0.01000,1.607
5,mlp.down_proj,0.0000000143,0.01000,4.988
6,self_attn.k_proj,0.0000000010,0.01000,1.512
6,self_attn.v_proj,0.0000000012,0.01000,1.545
6,self_attn.q_proj,0.0000000038,0.01000,1.626
6,self_attn.o_proj,0.0000000035,0.01000,1.520
6,mlp.gate_proj,0.0000002918,0.01000,1.548
6,mlp.up_proj,0.0000001269,0.01000,1.617
6,mlp.down_proj,0.0000001219,0.01000,4.987
7,self_attn.k_proj,0.0000000025,0.01000,1.524
7,self_attn.v_proj,0.0000000029,0.01000,1.562
7,self_attn.q_proj,0.0000000091,0.01000,1.606
7,self_attn.o_proj,0.0000000036,0.01000,1.533
7,mlp.gate_proj,0.0000003621,0.01000,1.574
7,mlp.up_proj,0.0000001570,0.01000,1.594
7,mlp.down_proj,0.0000000216,0.01000,4.959
8,self_attn.k_proj,0.0000000039,0.01000,1.510
8,self_attn.v_proj,0.0000000044,0.01000,1.544
8,self_attn.q_proj,0.0000000145,0.01000,1.618
8,self_attn.o_proj,0.0000000054,0.01000,1.522
8,mlp.gate_proj,0.0000001515,0.01000,1.564
8,mlp.up_proj,0.0000001310,0.01000,1.599
8,mlp.down_proj,0.0000000345,0.01000,4.941
9,self_attn.k_proj,0.0000000054,0.01000,1.494
9,self_attn.v_proj,0.0000000056,0.01000,1.540
9,self_attn.q_proj,0.0000000183,0.01000,1.613
9,self_attn.o_proj,0.0000000058,0.01000,1.518
9,mlp.gate_proj,0.0000002240,0.01000,1.563
9,mlp.up_proj,0.0000001633,0.01000,1.616
9,mlp.down_proj,0.0000000367,0.01000,5.004
10,self_attn.k_proj,0.0000000089,0.01000,1.506
10,self_attn.v_proj,0.0000000102,0.01000,1.543
10,self_attn.q_proj,0.0000000331,0.01000,1.610
10,self_attn.o_proj,0.0000000094,0.01000,1.524
10,mlp.gate_proj,0.0000001939,0.01000,1.565
10,mlp.up_proj,0.0000001494,0.01000,1.612
10,mlp.down_proj,0.0000000352,0.01000,4.938
11,self_attn.k_proj,0.0000000048,0.01000,1.528
11,self_attn.v_proj,0.0000000048,0.01000,1.541
11,self_attn.q_proj,0.0000000161,0.01000,1.621
11,self_attn.o_proj,0.0000000080,0.01000,1.522
11,mlp.gate_proj,0.0000001699,0.01000,1.564
11,mlp.up_proj,0.0000001435,0.01000,1.614
11,mlp.down_proj,0.0000000350,0.01000,4.944
12,self_attn.k_proj,0.0000000050,0.01000,1.503
12,self_attn.v_proj,0.0000000055,0.01000,1.546
12,self_attn.q_proj,0.0000000177,0.01000,1.626
12,self_attn.o_proj,0.0000000090,0.01000,1.524
12,mlp.gate_proj,0.0000001580,0.01000,1.562
12,mlp.up_proj,0.0000001436,0.01000,1.611
12,mlp.down_proj,0.0000000346,0.01000,4.945
13,self_attn.k_proj,0.0000000039,0.01000,1.509
13,self_attn.v_proj,0.0000000041,0.01000,1.547
13,self_attn.q_proj,0.0000000142,0.01000,1.620
13,self_attn.o_proj,0.0000000064,0.01000,1.521
13,mlp.gate_proj,0.0000001509,0.01000,1.568
13,mlp.up_proj,0.0000001471,0.01000,1.633
13,mlp.down_proj,0.0000000391,0.01000,4.962
14,self_attn.k_proj,0.0000000063,0.01000,1.547
14,self_attn.v_proj,0.0000000068,0.01000,1.546
14,self_attn.q_proj,0.0000000233,0.01000,1.615
14,self_attn.o_proj,0.0000000128,0.01000,1.518
14,mlp.gate_proj,0.0000001488,0.01000,1.566
14,mlp.up_proj,0.0000001473,0.01000,1.607
14,mlp.down_proj,0.0000000366,0.01000,4.994
15,self_attn.k_proj,0.0000000066,0.01000,1.507
15,self_attn.v_proj,0.0000000068,0.01000,1.549
15,self_attn.q_proj,0.0000000239,0.01000,1.614
15,self_attn.o_proj,0.0000000100,0.01000,1.518
15,mlp.gate_proj,0.0000001411,0.01000,1.579
15,mlp.up_proj,0.0000001453,0.01000,1.599
15,mlp.down_proj,0.0000000370,0.01000,4.955
16,self_attn.k_proj,0.0000000111,0.01000,1.511
16,self_attn.v_proj,0.0000000130,0.01000,1.546
16,self_attn.q_proj,0.0000000429,0.01000,1.631
16,self_attn.o_proj,0.0000000132,0.01000,1.513
16,mlp.gate_proj,0.0000001543,0.01000,1.574
16,mlp.up_proj,0.0000001530,0.01000,1.622
16,mlp.down_proj,0.0000000761,0.01000,4.931
17,self_attn.k_proj,0.0000000097,0.01000,1.511
17,self_attn.v_proj,0.0000000108,0.01000,1.539
17,self_attn.q_proj,0.0000000381,0.01000,1.622
17,self_attn.o_proj,0.0000000133,0.01000,1.520
17,mlp.gate_proj,0.0000001423,0.01000,1.569
17,mlp.up_proj,0.0000001452,0.01000,1.620
17,mlp.down_proj,0.0000000410,0.01000,4.928
18,self_attn.k_proj,0.0000000114,0.01000,1.500
18,self_attn.v_proj,0.0000000130,0.01000,1.548
18,self_attn.q_proj,0.0000000424,0.01000,1.608
18,self_attn.o_proj,0.0000000169,0.01000,1.530
18,mlp.gate_proj,0.0000001623,0.01000,1.563
18,mlp.up_proj,0.0000001655,0.01000,1.620
18,mlp.down_proj,0.0000000542,0.01000,4.953
19,self_attn.k_proj,0.0000000248,0.01000,1.518
19,self_attn.v_proj,0.0000000260,0.01000,1.546
19,self_attn.q_proj,0.0000000911,0.01000,1.614
19,self_attn.o_proj,0.0000000216,0.01000,1.512
19,mlp.gate_proj,0.0000001851,0.01000,1.558
19,mlp.up_proj,0.0000001915,0.01000,1.608
19,mlp.down_proj,0.0000000750,0.01000,4.947
20,self_attn.k_proj,0.0000000190,0.01000,1.523
20,self_attn.v_proj,0.0000000241,0.01000,1.555
20,self_attn.q_proj,0.0000000818,0.01000,1.615
20,self_attn.o_proj,0.0000000236,0.01000,1.522
20,mlp.gate_proj,0.0000002010,0.01000,1.581
20,mlp.up_proj,0.0000002100,0.01000,1.637
20,mlp.down_proj,0.0000000899,0.01000,4.963
21,self_attn.k_proj,0.0000000268,0.01000,1.503
21,self_attn.v_proj,0.0000000365,0.01000,1.549
21,self_attn.q_proj,0.0000001119,0.01000,1.624
21,self_attn.o_proj,0.0000000307,0.01000,1.521
21,mlp.gate_proj,0.0000002521,0.01000,1.565
21,mlp.up_proj,0.0000002601,0.01000,1.600
21,mlp.down_proj,0.0000001285,0.01000,4.954
22,self_attn.k_proj,0.0000000581,0.01000,1.515
22,self_attn.v_proj,0.0000000762,0.01000,1.550
22,self_attn.q_proj,0.0000002398,0.01000,1.616
22,self_attn.o_proj,0.0000000610,0.01000,1.513
22,mlp.gate_proj,0.0000003000,0.01000,1.560
22,mlp.up_proj,0.0000003052,0.01000,1.608
22,mlp.down_proj,0.0000002181,0.01000,4.960
23,self_attn.k_proj,0.0000000582,0.01000,1.517
23,self_attn.v_proj,0.0000000861,0.01000,1.544
23,self_attn.q_proj,0.0000002539,0.01000,1.612
23,self_attn.o_proj,0.0000000826,0.01000,1.524
23,mlp.gate_proj,0.0000003659,0.01000,1.567
23,mlp.up_proj,0.0000003648,0.01000,1.605
23,mlp.down_proj,0.0000003286,0.01000,4.964
24,self_attn.k_proj,0.0000001104,0.01000,1.511
24,self_attn.v_proj,0.0000001788,0.01000,1.536
24,self_attn.q_proj,0.0000005036,0.01000,1.613
24,self_attn.o_proj,0.0000000917,0.01000,1.530
24,mlp.gate_proj,0.0000004154,0.01000,1.565
24,mlp.up_proj,0.0000004158,0.01000,1.600
24,mlp.down_proj,0.0000004310,0.01000,5.002
25,self_attn.k_proj,0.0000000878,0.01000,1.511
25,self_attn.v_proj,0.0000001228,0.01000,1.555
25,self_attn.q_proj,0.0000003635,0.01000,1.618
25,self_attn.o_proj,0.0000000785,0.01000,1.518
25,mlp.gate_proj,0.0000004925,0.01000,1.574
25,mlp.up_proj,0.0000004896,0.01000,1.602
25,mlp.down_proj,0.0000005587,0.01000,4.954
26,self_attn.k_proj,0.0000001446,0.01000,1.514
26,self_attn.v_proj,0.0000002027,0.01000,1.555
26,self_attn.q_proj,0.0000006273,0.01000,1.633
26,self_attn.o_proj,0.0000000881,0.01000,1.515
26,mlp.gate_proj,0.0000006090,0.01000,1.566
26,mlp.up_proj,0.0000006170,0.01000,1.615
26,mlp.down_proj,0.0000006918,0.01000,4.965
27,self_attn.k_proj,0.0000001715,0.01000,1.518
27,self_attn.v_proj,0.0000002550,0.01000,1.541
27,self_attn.q_proj,0.0000007693,0.01000,1.617
27,self_attn.o_proj,0.0000001073,0.01000,1.520
27,mlp.gate_proj,0.0000007085,0.01000,1.560
27,mlp.up_proj,0.0000007293,0.01000,1.605
27,mlp.down_proj,0.0000010438,0.01000,4.943
28,self_attn.k_proj,0.0000002032,0.01000,1.505
28,self_attn.v_proj,0.0000002683,0.01000,1.556
28,self_attn.q_proj,0.0000008412,0.01000,1.613
28,self_attn.o_proj,0.0000001484,0.01000,1.527
28,mlp.gate_proj,0.0000008297,0.01000,1.558
28,mlp.up_proj,0.0000008718,0.01000,1.600
28,mlp.down_proj,0.0000014877,0.01000,4.940
29,self_attn.k_proj,0.0000004508,0.01000,1.505
29,self_attn.v_proj,0.0000006929,0.01000,1.546
29,self_attn.q_proj,0.0000019936,0.01000,1.624
29,self_attn.o_proj,0.0000001428,0.01000,1.515
29,mlp.gate_proj,0.0000009466,0.01000,1.555
29,mlp.up_proj,0.0000010386,0.01000,1.608
29,mlp.down_proj,0.0000016402,0.01000,4.970
30,self_attn.k_proj,0.0000005650,0.01000,1.525
30,self_attn.v_proj,0.0000008719,0.01000,1.544
30,self_attn.q_proj,0.0000021832,0.01000,1.617
30,self_attn.o_proj,0.0000002277,0.01000,1.520
30,mlp.gate_proj,0.0000010097,0.01000,1.562
30,mlp.up_proj,0.0000011334,0.01000,1.603
30,mlp.down_proj,0.0000021914,0.01000,4.953
31,self_attn.k_proj,0.0000007390,0.01000,1.523
31,self_attn.v_proj,0.0000011882,0.01000,1.598
31,self_attn.q_proj,0.0000029089,0.01000,1.646
31,self_attn.o_proj,0.0000002836,0.01000,1.541
31,mlp.gate_proj,0.0000010394,0.01000,1.560
31,mlp.up_proj,0.0000012097,0.01000,1.615
31,mlp.down_proj,0.0000026372,0.01000,4.952
32,self_attn.k_proj,0.0000010328,0.01000,1.507
32,self_attn.v_proj,0.0000017336,0.01000,1.542
32,self_attn.q_proj,0.0000041525,0.01000,1.644
32,self_attn.o_proj,0.0000003640,0.01000,1.594
32,mlp.gate_proj,0.0000010954,0.01000,1.565
32,mlp.up_proj,0.0000012984,0.01000,1.605
32,mlp.down_proj,0.0000032852,0.01000,4.950
33,self_attn.k_proj,0.0000016540,0.01000,1.510
33,self_attn.v_proj,0.0000035005,0.01000,1.542
33,self_attn.q_proj,0.0000076394,0.01000,1.615
33,self_attn.o_proj,0.0000005438,0.01000,1.533
33,mlp.gate_proj,0.0000011671,0.01000,1.579
33,mlp.up_proj,0.0000014071,0.01000,1.631
33,mlp.down_proj,0.0000042296,0.01000,4.963
34,self_attn.k_proj,0.0000015274,0.01000,1.511
34,self_attn.v_proj,0.0000030016,0.01000,1.535
34,self_attn.q_proj,0.0000064193,0.01000,1.621
34,self_attn.o_proj,0.0000009888,0.01000,1.520
34,mlp.gate_proj,0.0000014060,0.01000,1.577
34,mlp.up_proj,0.0000016153,0.01000,1.619
34,mlp.down_proj,0.0000061315,0.01000,4.935
35,self_attn.k_proj,0.0000007103,0.01000,1.526
35,self_attn.v_proj,0.0000010391,0.01000,1.550
35,self_attn.q_proj,0.0000027734,0.01000,1.629
35,self_attn.o_proj,0.0000009104,0.01000,1.525
35,mlp.gate_proj,0.0000021520,0.01000,1.580
35,mlp.up_proj,0.0000023153,0.01000,1.618
35,mlp.down_proj,0.0000103650,0.01000,4.973