File size: 2,353 Bytes
ea3d322 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 | {
"base_model": "google/gemma-4-31B-it",
"base_num_hidden_layers": 60,
"num_hidden_layers": 72,
"section_size": 6,
"duplicated_sections": [
-3,
-2
],
"new_layer_indices": [
48,
49,
50,
51,
52,
53,
60,
61,
62,
63,
64,
65
],
"layer_map_new_to_old": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35,
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
42,
43,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53,
48,
49,
50,
51,
52,
53,
54,
55,
56,
57,
58,
59
],
"identity_norms": [
"post_attention_layernorm",
"post_feedforward_layernorm"
],
"identity_norm_value": 0.0,
"layer_scalar_set_to_one": true,
"must_be_trainable": [
"layers.48.post_attention_layernorm",
"layers.48.post_feedforward_layernorm",
"layers.49.post_attention_layernorm",
"layers.49.post_feedforward_layernorm",
"layers.50.post_attention_layernorm",
"layers.50.post_feedforward_layernorm",
"layers.51.post_attention_layernorm",
"layers.51.post_feedforward_layernorm",
"layers.52.post_attention_layernorm",
"layers.52.post_feedforward_layernorm",
"layers.53.post_attention_layernorm",
"layers.53.post_feedforward_layernorm",
"layers.60.post_attention_layernorm",
"layers.60.post_feedforward_layernorm",
"layers.61.post_attention_layernorm",
"layers.61.post_feedforward_layernorm",
"layers.62.post_attention_layernorm",
"layers.62.post_feedforward_layernorm",
"layers.63.post_attention_layernorm",
"layers.63.post_feedforward_layernorm",
"layers.64.post_attention_layernorm",
"layers.64.post_feedforward_layernorm",
"layers.65.post_attention_layernorm",
"layers.65.post_feedforward_layernorm"
],
"identity_norm_init": "source_norm x 0.03",
"norm_source_map": {
"48": 42,
"49": 43,
"50": 44,
"51": 45,
"52": 46,
"53": 47,
"60": 54,
"61": 55,
"62": 56,
"63": 57,
"64": 58,
"65": 59
}
}
|