| { |
| "base_model": "google/gemma-4-31B-it", |
| "base_num_hidden_layers": 60, |
| "num_hidden_layers": 72, |
| "section_size": 6, |
| "duplicated_sections": [ |
| -3, |
| -2 |
| ], |
| "new_layer_indices": [ |
| 48, |
| 49, |
| 50, |
| 51, |
| 52, |
| 53, |
| 60, |
| 61, |
| 62, |
| 63, |
| 64, |
| 65 |
| ], |
| "layer_map_new_to_old": [ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| 5, |
| 6, |
| 7, |
| 8, |
| 9, |
| 10, |
| 11, |
| 12, |
| 13, |
| 14, |
| 15, |
| 16, |
| 17, |
| 18, |
| 19, |
| 20, |
| 21, |
| 22, |
| 23, |
| 24, |
| 25, |
| 26, |
| 27, |
| 28, |
| 29, |
| 30, |
| 31, |
| 32, |
| 33, |
| 34, |
| 35, |
| 36, |
| 37, |
| 38, |
| 39, |
| 40, |
| 41, |
| 42, |
| 43, |
| 44, |
| 45, |
| 46, |
| 47, |
| 42, |
| 43, |
| 44, |
| 45, |
| 46, |
| 47, |
| 48, |
| 49, |
| 50, |
| 51, |
| 52, |
| 53, |
| 48, |
| 49, |
| 50, |
| 51, |
| 52, |
| 53, |
| 54, |
| 55, |
| 56, |
| 57, |
| 58, |
| 59 |
| ], |
| "identity_norms": [ |
| "post_attention_layernorm", |
| "post_feedforward_layernorm" |
| ], |
| "identity_norm_value": 0.0, |
| "layer_scalar_set_to_one": true, |
| "must_be_trainable": [ |
| "layers.48.post_attention_layernorm", |
| "layers.48.post_feedforward_layernorm", |
| "layers.49.post_attention_layernorm", |
| "layers.49.post_feedforward_layernorm", |
| "layers.50.post_attention_layernorm", |
| "layers.50.post_feedforward_layernorm", |
| "layers.51.post_attention_layernorm", |
| "layers.51.post_feedforward_layernorm", |
| "layers.52.post_attention_layernorm", |
| "layers.52.post_feedforward_layernorm", |
| "layers.53.post_attention_layernorm", |
| "layers.53.post_feedforward_layernorm", |
| "layers.60.post_attention_layernorm", |
| "layers.60.post_feedforward_layernorm", |
| "layers.61.post_attention_layernorm", |
| "layers.61.post_feedforward_layernorm", |
| "layers.62.post_attention_layernorm", |
| "layers.62.post_feedforward_layernorm", |
| "layers.63.post_attention_layernorm", |
| "layers.63.post_feedforward_layernorm", |
| "layers.64.post_attention_layernorm", |
| "layers.64.post_feedforward_layernorm", |
| "layers.65.post_attention_layernorm", |
| "layers.65.post_feedforward_layernorm" |
| ], |
| "identity_norm_init": "source_norm x 0.03", |
| "norm_source_map": { |
| "48": 42, |
| "49": 43, |
| "50": 44, |
| "51": 45, |
| "52": 46, |
| "53": 47, |
| "60": 54, |
| "61": 55, |
| "62": 56, |
| "63": 57, |
| "64": 58, |
| "65": 59 |
| } |
| } |
|
|
