File size: 2,353 Bytes
ea3d322
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
{
  "base_model": "google/gemma-4-31B-it",
  "base_num_hidden_layers": 60,
  "num_hidden_layers": 72,
  "section_size": 6,
  "duplicated_sections": [
    -3,
    -2
  ],
  "new_layer_indices": [
    48,
    49,
    50,
    51,
    52,
    53,
    60,
    61,
    62,
    63,
    64,
    65
  ],
  "layer_map_new_to_old": [
    0,
    1,
    2,
    3,
    4,
    5,
    6,
    7,
    8,
    9,
    10,
    11,
    12,
    13,
    14,
    15,
    16,
    17,
    18,
    19,
    20,
    21,
    22,
    23,
    24,
    25,
    26,
    27,
    28,
    29,
    30,
    31,
    32,
    33,
    34,
    35,
    36,
    37,
    38,
    39,
    40,
    41,
    42,
    43,
    44,
    45,
    46,
    47,
    42,
    43,
    44,
    45,
    46,
    47,
    48,
    49,
    50,
    51,
    52,
    53,
    48,
    49,
    50,
    51,
    52,
    53,
    54,
    55,
    56,
    57,
    58,
    59
  ],
  "identity_norms": [
    "post_attention_layernorm",
    "post_feedforward_layernorm"
  ],
  "identity_norm_value": 0.0,
  "layer_scalar_set_to_one": true,
  "must_be_trainable": [
    "layers.48.post_attention_layernorm",
    "layers.48.post_feedforward_layernorm",
    "layers.49.post_attention_layernorm",
    "layers.49.post_feedforward_layernorm",
    "layers.50.post_attention_layernorm",
    "layers.50.post_feedforward_layernorm",
    "layers.51.post_attention_layernorm",
    "layers.51.post_feedforward_layernorm",
    "layers.52.post_attention_layernorm",
    "layers.52.post_feedforward_layernorm",
    "layers.53.post_attention_layernorm",
    "layers.53.post_feedforward_layernorm",
    "layers.60.post_attention_layernorm",
    "layers.60.post_feedforward_layernorm",
    "layers.61.post_attention_layernorm",
    "layers.61.post_feedforward_layernorm",
    "layers.62.post_attention_layernorm",
    "layers.62.post_feedforward_layernorm",
    "layers.63.post_attention_layernorm",
    "layers.63.post_feedforward_layernorm",
    "layers.64.post_attention_layernorm",
    "layers.64.post_feedforward_layernorm",
    "layers.65.post_attention_layernorm",
    "layers.65.post_feedforward_layernorm"
  ],
  "identity_norm_init": "source_norm x 0.03",
  "norm_source_map": {
    "48": 42,
    "49": 43,
    "50": 44,
    "51": 45,
    "52": 46,
    "53": 47,
    "60": 54,
    "61": 55,
    "62": 56,
    "63": 57,
    "64": 58,
    "65": 59
  }
}