diff --git "a/config.json" "b/config.json" --- "a/config.json" +++ "b/config.json" @@ -198,10446 +198,178 @@ "model.visual.blocks.26.mlp.linear_fc2", "model.visual.merger.linear_fc1", "model.visual.merger.linear_fc2", - "model.language_model.layers.0.linear_attn", "model.language_model.layers.0.linear_attn.norm", "model.language_model.layers.0.linear_attn.in_proj_b", "model.language_model.layers.0.linear_attn.in_proj_a", "model.language_model.layers.0.mlp.gate", - "model.language_model.layers.0.mlp.experts.0", - "model.language_model.layers.0.mlp.experts.1", - "model.language_model.layers.0.mlp.experts.2", - "model.language_model.layers.0.mlp.experts.3", - "model.language_model.layers.0.mlp.experts.4", - "model.language_model.layers.0.mlp.experts.5", - "model.language_model.layers.0.mlp.experts.6", - "model.language_model.layers.0.mlp.experts.7", - "model.language_model.layers.0.mlp.experts.8", - "model.language_model.layers.0.mlp.experts.9", - "model.language_model.layers.0.mlp.experts.10", - "model.language_model.layers.0.mlp.experts.11", - "model.language_model.layers.0.mlp.experts.12", - "model.language_model.layers.0.mlp.experts.13", - "model.language_model.layers.0.mlp.experts.14", - "model.language_model.layers.0.mlp.experts.15", - "model.language_model.layers.0.mlp.experts.16", - "model.language_model.layers.0.mlp.experts.17", - "model.language_model.layers.0.mlp.experts.18", - "model.language_model.layers.0.mlp.experts.19", - "model.language_model.layers.0.mlp.experts.20", - "model.language_model.layers.0.mlp.experts.21", - "model.language_model.layers.0.mlp.experts.22", - "model.language_model.layers.0.mlp.experts.23", - "model.language_model.layers.0.mlp.experts.24", - "model.language_model.layers.0.mlp.experts.25", - "model.language_model.layers.0.mlp.experts.26", - "model.language_model.layers.0.mlp.experts.27", - "model.language_model.layers.0.mlp.experts.28", - "model.language_model.layers.0.mlp.experts.29", - "model.language_model.layers.0.mlp.experts.30", - "model.language_model.layers.0.mlp.experts.31", - "model.language_model.layers.0.mlp.experts.32", - "model.language_model.layers.0.mlp.experts.33", - "model.language_model.layers.0.mlp.experts.34", - "model.language_model.layers.0.mlp.experts.35", - "model.language_model.layers.0.mlp.experts.36", - "model.language_model.layers.0.mlp.experts.37", - "model.language_model.layers.0.mlp.experts.38", - "model.language_model.layers.0.mlp.experts.39", - "model.language_model.layers.0.mlp.experts.40", - "model.language_model.layers.0.mlp.experts.41", - "model.language_model.layers.0.mlp.experts.42", - "model.language_model.layers.0.mlp.experts.43", - "model.language_model.layers.0.mlp.experts.44", - "model.language_model.layers.0.mlp.experts.45", - "model.language_model.layers.0.mlp.experts.46", - "model.language_model.layers.0.mlp.experts.47", - "model.language_model.layers.0.mlp.experts.48", - "model.language_model.layers.0.mlp.experts.49", - "model.language_model.layers.0.mlp.experts.50", - "model.language_model.layers.0.mlp.experts.51", - "model.language_model.layers.0.mlp.experts.52", - "model.language_model.layers.0.mlp.experts.53", - "model.language_model.layers.0.mlp.experts.54", - "model.language_model.layers.0.mlp.experts.55", - "model.language_model.layers.0.mlp.experts.56", - "model.language_model.layers.0.mlp.experts.57", - "model.language_model.layers.0.mlp.experts.58", - "model.language_model.layers.0.mlp.experts.59", - "model.language_model.layers.0.mlp.experts.60", - "model.language_model.layers.0.mlp.experts.61", - "model.language_model.layers.0.mlp.experts.62", - "model.language_model.layers.0.mlp.experts.63", - "model.language_model.layers.0.mlp.experts.64", - "model.language_model.layers.0.mlp.experts.65", - "model.language_model.layers.0.mlp.experts.66", - "model.language_model.layers.0.mlp.experts.67", - "model.language_model.layers.0.mlp.experts.68", - "model.language_model.layers.0.mlp.experts.69", - "model.language_model.layers.0.mlp.experts.70", - "model.language_model.layers.0.mlp.experts.71", - "model.language_model.layers.0.mlp.experts.72", - "model.language_model.layers.0.mlp.experts.73", - "model.language_model.layers.0.mlp.experts.74", - "model.language_model.layers.0.mlp.experts.75", - "model.language_model.layers.0.mlp.experts.76", - "model.language_model.layers.0.mlp.experts.77", - "model.language_model.layers.0.mlp.experts.78", - "model.language_model.layers.0.mlp.experts.79", - "model.language_model.layers.0.mlp.experts.80", - "model.language_model.layers.0.mlp.experts.81", - "model.language_model.layers.0.mlp.experts.82", - "model.language_model.layers.0.mlp.experts.83", - "model.language_model.layers.0.mlp.experts.84", - "model.language_model.layers.0.mlp.experts.85", - "model.language_model.layers.0.mlp.experts.86", - "model.language_model.layers.0.mlp.experts.87", - "model.language_model.layers.0.mlp.experts.88", - "model.language_model.layers.0.mlp.experts.89", - "model.language_model.layers.0.mlp.experts.90", - "model.language_model.layers.0.mlp.experts.91", - "model.language_model.layers.0.mlp.experts.92", - "model.language_model.layers.0.mlp.experts.93", - "model.language_model.layers.0.mlp.experts.94", - "model.language_model.layers.0.mlp.experts.95", - "model.language_model.layers.0.mlp.experts.96", - "model.language_model.layers.0.mlp.experts.97", - "model.language_model.layers.0.mlp.experts.98", - "model.language_model.layers.0.mlp.experts.99", - "model.language_model.layers.0.mlp.experts.100", - "model.language_model.layers.0.mlp.experts.101", - "model.language_model.layers.0.mlp.experts.102", - "model.language_model.layers.0.mlp.experts.103", - "model.language_model.layers.0.mlp.experts.104", - "model.language_model.layers.0.mlp.experts.105", - "model.language_model.layers.0.mlp.experts.106", - "model.language_model.layers.0.mlp.experts.107", - "model.language_model.layers.0.mlp.experts.108", - "model.language_model.layers.0.mlp.experts.109", - "model.language_model.layers.0.mlp.experts.110", - "model.language_model.layers.0.mlp.experts.111", - "model.language_model.layers.0.mlp.experts.112", - "model.language_model.layers.0.mlp.experts.113", - "model.language_model.layers.0.mlp.experts.114", - "model.language_model.layers.0.mlp.experts.115", - "model.language_model.layers.0.mlp.experts.116", - "model.language_model.layers.0.mlp.experts.117", - "model.language_model.layers.0.mlp.experts.118", - "model.language_model.layers.0.mlp.experts.119", - "model.language_model.layers.0.mlp.experts.120", - "model.language_model.layers.0.mlp.experts.121", - "model.language_model.layers.0.mlp.experts.122", - "model.language_model.layers.0.mlp.experts.123", - "model.language_model.layers.0.mlp.experts.124", - "model.language_model.layers.0.mlp.experts.125", - "model.language_model.layers.0.mlp.experts.126", - "model.language_model.layers.0.mlp.experts.127", - "model.language_model.layers.0.mlp.experts.128", - "model.language_model.layers.0.mlp.experts.129", - "model.language_model.layers.0.mlp.experts.130", - "model.language_model.layers.0.mlp.experts.131", - "model.language_model.layers.0.mlp.experts.132", - "model.language_model.layers.0.mlp.experts.133", - "model.language_model.layers.0.mlp.experts.134", - "model.language_model.layers.0.mlp.experts.135", - "model.language_model.layers.0.mlp.experts.136", - "model.language_model.layers.0.mlp.experts.137", - "model.language_model.layers.0.mlp.experts.138", - "model.language_model.layers.0.mlp.experts.139", - "model.language_model.layers.0.mlp.experts.140", - "model.language_model.layers.0.mlp.experts.141", - "model.language_model.layers.0.mlp.experts.142", - "model.language_model.layers.0.mlp.experts.143", - "model.language_model.layers.0.mlp.experts.144", - "model.language_model.layers.0.mlp.experts.145", - "model.language_model.layers.0.mlp.experts.146", - "model.language_model.layers.0.mlp.experts.147", - "model.language_model.layers.0.mlp.experts.148", - "model.language_model.layers.0.mlp.experts.149", - "model.language_model.layers.0.mlp.experts.150", - "model.language_model.layers.0.mlp.experts.151", - "model.language_model.layers.0.mlp.experts.152", - "model.language_model.layers.0.mlp.experts.153", - "model.language_model.layers.0.mlp.experts.154", - "model.language_model.layers.0.mlp.experts.155", - "model.language_model.layers.0.mlp.experts.156", - "model.language_model.layers.0.mlp.experts.157", - "model.language_model.layers.0.mlp.experts.158", - "model.language_model.layers.0.mlp.experts.159", - "model.language_model.layers.0.mlp.experts.160", - "model.language_model.layers.0.mlp.experts.161", - "model.language_model.layers.0.mlp.experts.162", - "model.language_model.layers.0.mlp.experts.163", - "model.language_model.layers.0.mlp.experts.164", - "model.language_model.layers.0.mlp.experts.165", - "model.language_model.layers.0.mlp.experts.166", - "model.language_model.layers.0.mlp.experts.167", - "model.language_model.layers.0.mlp.experts.168", - "model.language_model.layers.0.mlp.experts.169", - "model.language_model.layers.0.mlp.experts.170", - "model.language_model.layers.0.mlp.experts.171", - "model.language_model.layers.0.mlp.experts.172", - "model.language_model.layers.0.mlp.experts.173", - "model.language_model.layers.0.mlp.experts.174", - "model.language_model.layers.0.mlp.experts.175", - "model.language_model.layers.0.mlp.experts.176", - "model.language_model.layers.0.mlp.experts.177", - "model.language_model.layers.0.mlp.experts.178", - "model.language_model.layers.0.mlp.experts.179", - "model.language_model.layers.0.mlp.experts.180", - "model.language_model.layers.0.mlp.experts.181", - "model.language_model.layers.0.mlp.experts.182", - "model.language_model.layers.0.mlp.experts.183", - "model.language_model.layers.0.mlp.experts.184", - "model.language_model.layers.0.mlp.experts.185", - "model.language_model.layers.0.mlp.experts.186", - "model.language_model.layers.0.mlp.experts.187", - "model.language_model.layers.0.mlp.experts.188", - "model.language_model.layers.0.mlp.experts.189", - "model.language_model.layers.0.mlp.experts.190", - "model.language_model.layers.0.mlp.experts.191", - "model.language_model.layers.0.mlp.experts.192", - "model.language_model.layers.0.mlp.experts.193", - "model.language_model.layers.0.mlp.experts.194", - "model.language_model.layers.0.mlp.experts.195", - "model.language_model.layers.0.mlp.experts.196", - "model.language_model.layers.0.mlp.experts.197", - "model.language_model.layers.0.mlp.experts.198", - "model.language_model.layers.0.mlp.experts.199", - "model.language_model.layers.0.mlp.experts.200", - "model.language_model.layers.0.mlp.experts.201", - "model.language_model.layers.0.mlp.experts.202", - "model.language_model.layers.0.mlp.experts.203", - "model.language_model.layers.0.mlp.experts.204", - "model.language_model.layers.0.mlp.experts.205", - "model.language_model.layers.0.mlp.experts.206", - "model.language_model.layers.0.mlp.experts.207", - "model.language_model.layers.0.mlp.experts.208", - "model.language_model.layers.0.mlp.experts.209", - "model.language_model.layers.0.mlp.experts.210", - "model.language_model.layers.0.mlp.experts.211", - "model.language_model.layers.0.mlp.experts.212", - "model.language_model.layers.0.mlp.experts.213", - "model.language_model.layers.0.mlp.experts.214", - "model.language_model.layers.0.mlp.experts.215", - "model.language_model.layers.0.mlp.experts.216", - "model.language_model.layers.0.mlp.experts.217", - "model.language_model.layers.0.mlp.experts.218", - "model.language_model.layers.0.mlp.experts.219", - "model.language_model.layers.0.mlp.experts.220", - "model.language_model.layers.0.mlp.experts.221", - "model.language_model.layers.0.mlp.experts.222", - "model.language_model.layers.0.mlp.experts.223", - "model.language_model.layers.0.mlp.experts.224", - "model.language_model.layers.0.mlp.experts.225", - "model.language_model.layers.0.mlp.experts.226", - "model.language_model.layers.0.mlp.experts.227", - "model.language_model.layers.0.mlp.experts.228", - "model.language_model.layers.0.mlp.experts.229", - "model.language_model.layers.0.mlp.experts.230", - "model.language_model.layers.0.mlp.experts.231", - "model.language_model.layers.0.mlp.experts.232", - "model.language_model.layers.0.mlp.experts.233", - "model.language_model.layers.0.mlp.experts.234", - "model.language_model.layers.0.mlp.experts.235", - "model.language_model.layers.0.mlp.experts.236", - "model.language_model.layers.0.mlp.experts.237", - "model.language_model.layers.0.mlp.experts.238", - "model.language_model.layers.0.mlp.experts.239", - "model.language_model.layers.0.mlp.experts.240", - "model.language_model.layers.0.mlp.experts.241", - "model.language_model.layers.0.mlp.experts.242", - "model.language_model.layers.0.mlp.experts.243", - "model.language_model.layers.0.mlp.experts.244", - "model.language_model.layers.0.mlp.experts.245", - "model.language_model.layers.0.mlp.experts.246", - "model.language_model.layers.0.mlp.experts.247", - "model.language_model.layers.0.mlp.experts.248", - "model.language_model.layers.0.mlp.experts.249", - "model.language_model.layers.0.mlp.experts.250", - "model.language_model.layers.0.mlp.experts.251", - "model.language_model.layers.0.mlp.experts.252", - "model.language_model.layers.0.mlp.experts.253", - "model.language_model.layers.0.mlp.experts.254", - "model.language_model.layers.0.mlp.experts.255", "model.language_model.layers.0.mlp.shared_expert_gate", - "model.language_model.layers.1.linear_attn", "model.language_model.layers.1.linear_attn.norm", "model.language_model.layers.1.linear_attn.in_proj_b", "model.language_model.layers.1.linear_attn.in_proj_a", "model.language_model.layers.1.mlp.gate", - "model.language_model.layers.1.mlp.experts.0", - "model.language_model.layers.1.mlp.experts.1", - "model.language_model.layers.1.mlp.experts.2", - "model.language_model.layers.1.mlp.experts.3", - "model.language_model.layers.1.mlp.experts.4", - "model.language_model.layers.1.mlp.experts.5", - "model.language_model.layers.1.mlp.experts.6", - "model.language_model.layers.1.mlp.experts.7", - "model.language_model.layers.1.mlp.experts.8", - "model.language_model.layers.1.mlp.experts.9", - "model.language_model.layers.1.mlp.experts.10", - "model.language_model.layers.1.mlp.experts.11", - "model.language_model.layers.1.mlp.experts.12", - "model.language_model.layers.1.mlp.experts.13", - "model.language_model.layers.1.mlp.experts.14", - "model.language_model.layers.1.mlp.experts.15", - "model.language_model.layers.1.mlp.experts.16", - "model.language_model.layers.1.mlp.experts.17", - "model.language_model.layers.1.mlp.experts.18", - "model.language_model.layers.1.mlp.experts.19", - "model.language_model.layers.1.mlp.experts.20", - "model.language_model.layers.1.mlp.experts.21", - "model.language_model.layers.1.mlp.experts.22", - "model.language_model.layers.1.mlp.experts.23", - "model.language_model.layers.1.mlp.experts.24", - "model.language_model.layers.1.mlp.experts.25", - "model.language_model.layers.1.mlp.experts.26", - "model.language_model.layers.1.mlp.experts.27", - "model.language_model.layers.1.mlp.experts.28", - "model.language_model.layers.1.mlp.experts.29", - "model.language_model.layers.1.mlp.experts.30", - "model.language_model.layers.1.mlp.experts.31", - "model.language_model.layers.1.mlp.experts.32", - "model.language_model.layers.1.mlp.experts.33", - "model.language_model.layers.1.mlp.experts.34", - "model.language_model.layers.1.mlp.experts.35", - "model.language_model.layers.1.mlp.experts.36", - "model.language_model.layers.1.mlp.experts.37", - "model.language_model.layers.1.mlp.experts.38", - "model.language_model.layers.1.mlp.experts.39", - "model.language_model.layers.1.mlp.experts.40", - "model.language_model.layers.1.mlp.experts.41", - "model.language_model.layers.1.mlp.experts.42", - "model.language_model.layers.1.mlp.experts.43", - "model.language_model.layers.1.mlp.experts.44", - "model.language_model.layers.1.mlp.experts.45", - "model.language_model.layers.1.mlp.experts.46", - "model.language_model.layers.1.mlp.experts.47", - "model.language_model.layers.1.mlp.experts.48", - "model.language_model.layers.1.mlp.experts.49", - "model.language_model.layers.1.mlp.experts.50", - "model.language_model.layers.1.mlp.experts.51", - "model.language_model.layers.1.mlp.experts.52", - "model.language_model.layers.1.mlp.experts.53", - "model.language_model.layers.1.mlp.experts.54", - "model.language_model.layers.1.mlp.experts.55", - "model.language_model.layers.1.mlp.experts.56", - "model.language_model.layers.1.mlp.experts.57", - "model.language_model.layers.1.mlp.experts.58", - "model.language_model.layers.1.mlp.experts.59", - "model.language_model.layers.1.mlp.experts.60", - "model.language_model.layers.1.mlp.experts.61", - "model.language_model.layers.1.mlp.experts.62", - "model.language_model.layers.1.mlp.experts.63", - "model.language_model.layers.1.mlp.experts.64", - "model.language_model.layers.1.mlp.experts.65", - "model.language_model.layers.1.mlp.experts.66", - "model.language_model.layers.1.mlp.experts.67", - "model.language_model.layers.1.mlp.experts.68", - "model.language_model.layers.1.mlp.experts.69", - "model.language_model.layers.1.mlp.experts.70", - "model.language_model.layers.1.mlp.experts.71", - "model.language_model.layers.1.mlp.experts.72", - "model.language_model.layers.1.mlp.experts.73", - "model.language_model.layers.1.mlp.experts.74", - "model.language_model.layers.1.mlp.experts.75", - "model.language_model.layers.1.mlp.experts.76", - "model.language_model.layers.1.mlp.experts.77", - "model.language_model.layers.1.mlp.experts.78", - "model.language_model.layers.1.mlp.experts.79", - "model.language_model.layers.1.mlp.experts.80", - "model.language_model.layers.1.mlp.experts.81", - "model.language_model.layers.1.mlp.experts.82", - "model.language_model.layers.1.mlp.experts.83", - "model.language_model.layers.1.mlp.experts.84", - "model.language_model.layers.1.mlp.experts.85", - "model.language_model.layers.1.mlp.experts.86", - "model.language_model.layers.1.mlp.experts.87", - "model.language_model.layers.1.mlp.experts.88", - "model.language_model.layers.1.mlp.experts.89", - "model.language_model.layers.1.mlp.experts.90", - "model.language_model.layers.1.mlp.experts.91", - "model.language_model.layers.1.mlp.experts.92", - "model.language_model.layers.1.mlp.experts.93", - "model.language_model.layers.1.mlp.experts.94", - "model.language_model.layers.1.mlp.experts.95", - "model.language_model.layers.1.mlp.experts.96", - "model.language_model.layers.1.mlp.experts.97", - "model.language_model.layers.1.mlp.experts.98", - "model.language_model.layers.1.mlp.experts.99", - "model.language_model.layers.1.mlp.experts.100", - "model.language_model.layers.1.mlp.experts.101", - "model.language_model.layers.1.mlp.experts.102", - "model.language_model.layers.1.mlp.experts.103", - "model.language_model.layers.1.mlp.experts.104", - "model.language_model.layers.1.mlp.experts.105", - "model.language_model.layers.1.mlp.experts.106", - "model.language_model.layers.1.mlp.experts.107", - "model.language_model.layers.1.mlp.experts.108", - "model.language_model.layers.1.mlp.experts.109", - "model.language_model.layers.1.mlp.experts.110", - "model.language_model.layers.1.mlp.experts.111", - "model.language_model.layers.1.mlp.experts.112", - "model.language_model.layers.1.mlp.experts.113", - "model.language_model.layers.1.mlp.experts.114", - "model.language_model.layers.1.mlp.experts.115", - "model.language_model.layers.1.mlp.experts.116", - "model.language_model.layers.1.mlp.experts.117", - "model.language_model.layers.1.mlp.experts.118", - "model.language_model.layers.1.mlp.experts.119", - "model.language_model.layers.1.mlp.experts.120", - "model.language_model.layers.1.mlp.experts.121", - "model.language_model.layers.1.mlp.experts.122", - "model.language_model.layers.1.mlp.experts.123", - "model.language_model.layers.1.mlp.experts.124", - "model.language_model.layers.1.mlp.experts.125", - "model.language_model.layers.1.mlp.experts.126", - "model.language_model.layers.1.mlp.experts.127", - "model.language_model.layers.1.mlp.experts.128", - "model.language_model.layers.1.mlp.experts.129", - "model.language_model.layers.1.mlp.experts.130", - "model.language_model.layers.1.mlp.experts.131", - "model.language_model.layers.1.mlp.experts.132", - "model.language_model.layers.1.mlp.experts.133", - "model.language_model.layers.1.mlp.experts.134", - "model.language_model.layers.1.mlp.experts.135", - "model.language_model.layers.1.mlp.experts.136", - "model.language_model.layers.1.mlp.experts.137", - "model.language_model.layers.1.mlp.experts.138", - "model.language_model.layers.1.mlp.experts.139", - "model.language_model.layers.1.mlp.experts.140", - "model.language_model.layers.1.mlp.experts.141", - "model.language_model.layers.1.mlp.experts.142", - "model.language_model.layers.1.mlp.experts.143", - "model.language_model.layers.1.mlp.experts.144", - "model.language_model.layers.1.mlp.experts.145", - "model.language_model.layers.1.mlp.experts.146", - "model.language_model.layers.1.mlp.experts.147", - "model.language_model.layers.1.mlp.experts.148", - "model.language_model.layers.1.mlp.experts.149", - "model.language_model.layers.1.mlp.experts.150", - "model.language_model.layers.1.mlp.experts.151", - "model.language_model.layers.1.mlp.experts.152", - "model.language_model.layers.1.mlp.experts.153", - "model.language_model.layers.1.mlp.experts.154", - "model.language_model.layers.1.mlp.experts.155", - "model.language_model.layers.1.mlp.experts.156", - "model.language_model.layers.1.mlp.experts.157", - "model.language_model.layers.1.mlp.experts.158", - "model.language_model.layers.1.mlp.experts.159", - "model.language_model.layers.1.mlp.experts.160", - "model.language_model.layers.1.mlp.experts.161", - "model.language_model.layers.1.mlp.experts.162", - "model.language_model.layers.1.mlp.experts.163", - "model.language_model.layers.1.mlp.experts.164", - "model.language_model.layers.1.mlp.experts.165", - "model.language_model.layers.1.mlp.experts.166", - "model.language_model.layers.1.mlp.experts.167", - "model.language_model.layers.1.mlp.experts.168", - "model.language_model.layers.1.mlp.experts.169", - "model.language_model.layers.1.mlp.experts.170", - "model.language_model.layers.1.mlp.experts.171", - "model.language_model.layers.1.mlp.experts.172", - "model.language_model.layers.1.mlp.experts.173", - "model.language_model.layers.1.mlp.experts.174", - "model.language_model.layers.1.mlp.experts.175", - "model.language_model.layers.1.mlp.experts.176", - "model.language_model.layers.1.mlp.experts.177", - "model.language_model.layers.1.mlp.experts.178", - "model.language_model.layers.1.mlp.experts.179", - "model.language_model.layers.1.mlp.experts.180", - "model.language_model.layers.1.mlp.experts.181", - "model.language_model.layers.1.mlp.experts.182", - "model.language_model.layers.1.mlp.experts.183", - "model.language_model.layers.1.mlp.experts.184", - "model.language_model.layers.1.mlp.experts.185", - "model.language_model.layers.1.mlp.experts.186", - "model.language_model.layers.1.mlp.experts.187", - "model.language_model.layers.1.mlp.experts.188", - "model.language_model.layers.1.mlp.experts.189", - "model.language_model.layers.1.mlp.experts.190", - "model.language_model.layers.1.mlp.experts.191", - "model.language_model.layers.1.mlp.experts.192", - "model.language_model.layers.1.mlp.experts.193", - "model.language_model.layers.1.mlp.experts.194", - "model.language_model.layers.1.mlp.experts.195", - "model.language_model.layers.1.mlp.experts.196", - "model.language_model.layers.1.mlp.experts.197", - "model.language_model.layers.1.mlp.experts.198", - "model.language_model.layers.1.mlp.experts.199", - "model.language_model.layers.1.mlp.experts.200", - "model.language_model.layers.1.mlp.experts.201", - "model.language_model.layers.1.mlp.experts.202", - "model.language_model.layers.1.mlp.experts.203", - "model.language_model.layers.1.mlp.experts.204", - "model.language_model.layers.1.mlp.experts.205", - "model.language_model.layers.1.mlp.experts.206", - "model.language_model.layers.1.mlp.experts.207", - "model.language_model.layers.1.mlp.experts.208", - "model.language_model.layers.1.mlp.experts.209", - "model.language_model.layers.1.mlp.experts.210", - "model.language_model.layers.1.mlp.experts.211", - "model.language_model.layers.1.mlp.experts.212", - "model.language_model.layers.1.mlp.experts.213", - "model.language_model.layers.1.mlp.experts.214", - "model.language_model.layers.1.mlp.experts.215", - "model.language_model.layers.1.mlp.experts.216", - "model.language_model.layers.1.mlp.experts.217", - "model.language_model.layers.1.mlp.experts.218", - "model.language_model.layers.1.mlp.experts.219", - "model.language_model.layers.1.mlp.experts.220", - "model.language_model.layers.1.mlp.experts.221", - "model.language_model.layers.1.mlp.experts.222", - "model.language_model.layers.1.mlp.experts.223", - "model.language_model.layers.1.mlp.experts.224", - "model.language_model.layers.1.mlp.experts.225", - "model.language_model.layers.1.mlp.experts.226", - "model.language_model.layers.1.mlp.experts.227", - "model.language_model.layers.1.mlp.experts.228", - "model.language_model.layers.1.mlp.experts.229", - "model.language_model.layers.1.mlp.experts.230", - "model.language_model.layers.1.mlp.experts.231", - "model.language_model.layers.1.mlp.experts.232", - "model.language_model.layers.1.mlp.experts.233", - "model.language_model.layers.1.mlp.experts.234", - "model.language_model.layers.1.mlp.experts.235", - "model.language_model.layers.1.mlp.experts.236", - "model.language_model.layers.1.mlp.experts.237", - "model.language_model.layers.1.mlp.experts.238", - "model.language_model.layers.1.mlp.experts.239", - "model.language_model.layers.1.mlp.experts.240", - "model.language_model.layers.1.mlp.experts.241", - "model.language_model.layers.1.mlp.experts.242", - "model.language_model.layers.1.mlp.experts.243", - "model.language_model.layers.1.mlp.experts.244", - "model.language_model.layers.1.mlp.experts.245", - "model.language_model.layers.1.mlp.experts.246", - "model.language_model.layers.1.mlp.experts.247", - "model.language_model.layers.1.mlp.experts.248", - "model.language_model.layers.1.mlp.experts.249", - "model.language_model.layers.1.mlp.experts.250", - "model.language_model.layers.1.mlp.experts.251", - "model.language_model.layers.1.mlp.experts.252", - "model.language_model.layers.1.mlp.experts.253", - "model.language_model.layers.1.mlp.experts.254", - "model.language_model.layers.1.mlp.experts.255", "model.language_model.layers.1.mlp.shared_expert_gate", - "model.language_model.layers.2.linear_attn", "model.language_model.layers.2.linear_attn.norm", "model.language_model.layers.2.linear_attn.in_proj_b", "model.language_model.layers.2.linear_attn.in_proj_a", "model.language_model.layers.2.mlp.gate", - "model.language_model.layers.2.mlp.experts.0", - "model.language_model.layers.2.mlp.experts.1", - "model.language_model.layers.2.mlp.experts.2", - "model.language_model.layers.2.mlp.experts.3", - "model.language_model.layers.2.mlp.experts.4", - "model.language_model.layers.2.mlp.experts.5", - "model.language_model.layers.2.mlp.experts.6", - "model.language_model.layers.2.mlp.experts.7", - "model.language_model.layers.2.mlp.experts.8", - "model.language_model.layers.2.mlp.experts.9", - "model.language_model.layers.2.mlp.experts.10", - "model.language_model.layers.2.mlp.experts.11", - "model.language_model.layers.2.mlp.experts.12", - "model.language_model.layers.2.mlp.experts.13", - "model.language_model.layers.2.mlp.experts.14", - "model.language_model.layers.2.mlp.experts.15", - "model.language_model.layers.2.mlp.experts.16", - "model.language_model.layers.2.mlp.experts.17", - "model.language_model.layers.2.mlp.experts.18", - "model.language_model.layers.2.mlp.experts.19", - "model.language_model.layers.2.mlp.experts.20", - "model.language_model.layers.2.mlp.experts.21", - "model.language_model.layers.2.mlp.experts.22", - "model.language_model.layers.2.mlp.experts.23", - "model.language_model.layers.2.mlp.experts.24", - "model.language_model.layers.2.mlp.experts.25", - "model.language_model.layers.2.mlp.experts.26", - "model.language_model.layers.2.mlp.experts.27", - "model.language_model.layers.2.mlp.experts.28", - "model.language_model.layers.2.mlp.experts.29", - "model.language_model.layers.2.mlp.experts.30", - "model.language_model.layers.2.mlp.experts.31", - "model.language_model.layers.2.mlp.experts.32", - "model.language_model.layers.2.mlp.experts.33", - "model.language_model.layers.2.mlp.experts.34", - "model.language_model.layers.2.mlp.experts.35", - "model.language_model.layers.2.mlp.experts.36", - "model.language_model.layers.2.mlp.experts.37", - "model.language_model.layers.2.mlp.experts.38", - "model.language_model.layers.2.mlp.experts.39", - "model.language_model.layers.2.mlp.experts.40", - "model.language_model.layers.2.mlp.experts.41", - "model.language_model.layers.2.mlp.experts.42", - "model.language_model.layers.2.mlp.experts.43", - "model.language_model.layers.2.mlp.experts.44", - "model.language_model.layers.2.mlp.experts.45", - "model.language_model.layers.2.mlp.experts.46", - "model.language_model.layers.2.mlp.experts.47", - "model.language_model.layers.2.mlp.experts.48", - "model.language_model.layers.2.mlp.experts.49", - "model.language_model.layers.2.mlp.experts.50", - "model.language_model.layers.2.mlp.experts.51", - "model.language_model.layers.2.mlp.experts.52", - "model.language_model.layers.2.mlp.experts.53", - "model.language_model.layers.2.mlp.experts.54", - "model.language_model.layers.2.mlp.experts.55", - "model.language_model.layers.2.mlp.experts.56", - "model.language_model.layers.2.mlp.experts.57", - "model.language_model.layers.2.mlp.experts.58", - "model.language_model.layers.2.mlp.experts.59", - "model.language_model.layers.2.mlp.experts.60", - "model.language_model.layers.2.mlp.experts.61", - "model.language_model.layers.2.mlp.experts.62", - "model.language_model.layers.2.mlp.experts.63", - "model.language_model.layers.2.mlp.experts.64", - "model.language_model.layers.2.mlp.experts.65", - "model.language_model.layers.2.mlp.experts.66", - "model.language_model.layers.2.mlp.experts.67", - "model.language_model.layers.2.mlp.experts.68", - "model.language_model.layers.2.mlp.experts.69", - "model.language_model.layers.2.mlp.experts.70", - "model.language_model.layers.2.mlp.experts.71", - "model.language_model.layers.2.mlp.experts.72", - "model.language_model.layers.2.mlp.experts.73", - "model.language_model.layers.2.mlp.experts.74", - "model.language_model.layers.2.mlp.experts.75", - "model.language_model.layers.2.mlp.experts.76", - "model.language_model.layers.2.mlp.experts.77", - "model.language_model.layers.2.mlp.experts.78", - "model.language_model.layers.2.mlp.experts.79", - "model.language_model.layers.2.mlp.experts.80", - "model.language_model.layers.2.mlp.experts.81", - "model.language_model.layers.2.mlp.experts.82", - "model.language_model.layers.2.mlp.experts.83", - "model.language_model.layers.2.mlp.experts.84", - "model.language_model.layers.2.mlp.experts.85", - "model.language_model.layers.2.mlp.experts.86", - "model.language_model.layers.2.mlp.experts.87", - "model.language_model.layers.2.mlp.experts.88", - "model.language_model.layers.2.mlp.experts.89", - "model.language_model.layers.2.mlp.experts.90", - "model.language_model.layers.2.mlp.experts.91", - "model.language_model.layers.2.mlp.experts.92", - "model.language_model.layers.2.mlp.experts.93", - "model.language_model.layers.2.mlp.experts.94", - "model.language_model.layers.2.mlp.experts.95", - "model.language_model.layers.2.mlp.experts.96", - "model.language_model.layers.2.mlp.experts.97", - "model.language_model.layers.2.mlp.experts.98", - "model.language_model.layers.2.mlp.experts.99", - "model.language_model.layers.2.mlp.experts.100", - "model.language_model.layers.2.mlp.experts.101", - "model.language_model.layers.2.mlp.experts.102", - "model.language_model.layers.2.mlp.experts.103", - "model.language_model.layers.2.mlp.experts.104", - "model.language_model.layers.2.mlp.experts.105", - "model.language_model.layers.2.mlp.experts.106", - "model.language_model.layers.2.mlp.experts.107", - "model.language_model.layers.2.mlp.experts.108", - "model.language_model.layers.2.mlp.experts.109", - "model.language_model.layers.2.mlp.experts.110", - "model.language_model.layers.2.mlp.experts.111", - "model.language_model.layers.2.mlp.experts.112", - "model.language_model.layers.2.mlp.experts.113", - "model.language_model.layers.2.mlp.experts.114", - "model.language_model.layers.2.mlp.experts.115", - "model.language_model.layers.2.mlp.experts.116", - "model.language_model.layers.2.mlp.experts.117", - "model.language_model.layers.2.mlp.experts.118", - "model.language_model.layers.2.mlp.experts.119", - "model.language_model.layers.2.mlp.experts.120", - "model.language_model.layers.2.mlp.experts.121", - "model.language_model.layers.2.mlp.experts.122", - "model.language_model.layers.2.mlp.experts.123", - "model.language_model.layers.2.mlp.experts.124", - "model.language_model.layers.2.mlp.experts.125", - "model.language_model.layers.2.mlp.experts.126", - "model.language_model.layers.2.mlp.experts.127", - "model.language_model.layers.2.mlp.experts.128", - "model.language_model.layers.2.mlp.experts.129", - "model.language_model.layers.2.mlp.experts.130", - "model.language_model.layers.2.mlp.experts.131", - "model.language_model.layers.2.mlp.experts.132", - "model.language_model.layers.2.mlp.experts.133", - "model.language_model.layers.2.mlp.experts.134", - "model.language_model.layers.2.mlp.experts.135", - "model.language_model.layers.2.mlp.experts.136", - "model.language_model.layers.2.mlp.experts.137", - "model.language_model.layers.2.mlp.experts.138", - "model.language_model.layers.2.mlp.experts.139", - "model.language_model.layers.2.mlp.experts.140", - "model.language_model.layers.2.mlp.experts.141", - "model.language_model.layers.2.mlp.experts.142", - "model.language_model.layers.2.mlp.experts.143", - "model.language_model.layers.2.mlp.experts.144", - "model.language_model.layers.2.mlp.experts.145", - "model.language_model.layers.2.mlp.experts.146", - "model.language_model.layers.2.mlp.experts.147", - "model.language_model.layers.2.mlp.experts.148", - "model.language_model.layers.2.mlp.experts.149", - "model.language_model.layers.2.mlp.experts.150", - "model.language_model.layers.2.mlp.experts.151", - "model.language_model.layers.2.mlp.experts.152", - "model.language_model.layers.2.mlp.experts.153", - "model.language_model.layers.2.mlp.experts.154", - "model.language_model.layers.2.mlp.experts.155", - "model.language_model.layers.2.mlp.experts.156", - "model.language_model.layers.2.mlp.experts.157", - "model.language_model.layers.2.mlp.experts.158", - "model.language_model.layers.2.mlp.experts.159", - "model.language_model.layers.2.mlp.experts.160", - "model.language_model.layers.2.mlp.experts.161", - "model.language_model.layers.2.mlp.experts.162", - "model.language_model.layers.2.mlp.experts.163", - "model.language_model.layers.2.mlp.experts.164", - "model.language_model.layers.2.mlp.experts.165", - "model.language_model.layers.2.mlp.experts.166", - "model.language_model.layers.2.mlp.experts.167", - "model.language_model.layers.2.mlp.experts.168", - "model.language_model.layers.2.mlp.experts.169", - "model.language_model.layers.2.mlp.experts.170", - "model.language_model.layers.2.mlp.experts.171", - "model.language_model.layers.2.mlp.experts.172", - "model.language_model.layers.2.mlp.experts.173", - "model.language_model.layers.2.mlp.experts.174", - "model.language_model.layers.2.mlp.experts.175", - "model.language_model.layers.2.mlp.experts.176", - "model.language_model.layers.2.mlp.experts.177", - "model.language_model.layers.2.mlp.experts.178", - "model.language_model.layers.2.mlp.experts.179", - "model.language_model.layers.2.mlp.experts.180", - "model.language_model.layers.2.mlp.experts.181", - "model.language_model.layers.2.mlp.experts.182", - "model.language_model.layers.2.mlp.experts.183", - "model.language_model.layers.2.mlp.experts.184", - "model.language_model.layers.2.mlp.experts.185", - "model.language_model.layers.2.mlp.experts.186", - "model.language_model.layers.2.mlp.experts.187", - "model.language_model.layers.2.mlp.experts.188", - "model.language_model.layers.2.mlp.experts.189", - "model.language_model.layers.2.mlp.experts.190", - "model.language_model.layers.2.mlp.experts.191", - "model.language_model.layers.2.mlp.experts.192", - "model.language_model.layers.2.mlp.experts.193", - "model.language_model.layers.2.mlp.experts.194", - "model.language_model.layers.2.mlp.experts.195", - "model.language_model.layers.2.mlp.experts.196", - "model.language_model.layers.2.mlp.experts.197", - "model.language_model.layers.2.mlp.experts.198", - "model.language_model.layers.2.mlp.experts.199", - "model.language_model.layers.2.mlp.experts.200", - "model.language_model.layers.2.mlp.experts.201", - "model.language_model.layers.2.mlp.experts.202", - "model.language_model.layers.2.mlp.experts.203", - "model.language_model.layers.2.mlp.experts.204", - "model.language_model.layers.2.mlp.experts.205", - "model.language_model.layers.2.mlp.experts.206", - "model.language_model.layers.2.mlp.experts.207", - "model.language_model.layers.2.mlp.experts.208", - "model.language_model.layers.2.mlp.experts.209", - "model.language_model.layers.2.mlp.experts.210", - "model.language_model.layers.2.mlp.experts.211", - "model.language_model.layers.2.mlp.experts.212", - "model.language_model.layers.2.mlp.experts.213", - "model.language_model.layers.2.mlp.experts.214", - "model.language_model.layers.2.mlp.experts.215", - "model.language_model.layers.2.mlp.experts.216", - "model.language_model.layers.2.mlp.experts.217", - "model.language_model.layers.2.mlp.experts.218", - "model.language_model.layers.2.mlp.experts.219", - "model.language_model.layers.2.mlp.experts.220", - "model.language_model.layers.2.mlp.experts.221", - "model.language_model.layers.2.mlp.experts.222", - "model.language_model.layers.2.mlp.experts.223", - "model.language_model.layers.2.mlp.experts.224", - "model.language_model.layers.2.mlp.experts.225", - "model.language_model.layers.2.mlp.experts.226", - "model.language_model.layers.2.mlp.experts.227", - "model.language_model.layers.2.mlp.experts.228", - "model.language_model.layers.2.mlp.experts.229", - "model.language_model.layers.2.mlp.experts.230", - "model.language_model.layers.2.mlp.experts.231", - "model.language_model.layers.2.mlp.experts.232", - "model.language_model.layers.2.mlp.experts.233", - "model.language_model.layers.2.mlp.experts.234", - "model.language_model.layers.2.mlp.experts.235", - "model.language_model.layers.2.mlp.experts.236", - "model.language_model.layers.2.mlp.experts.237", - "model.language_model.layers.2.mlp.experts.238", - "model.language_model.layers.2.mlp.experts.239", - "model.language_model.layers.2.mlp.experts.240", - "model.language_model.layers.2.mlp.experts.241", - "model.language_model.layers.2.mlp.experts.242", - "model.language_model.layers.2.mlp.experts.243", - "model.language_model.layers.2.mlp.experts.244", - "model.language_model.layers.2.mlp.experts.245", - "model.language_model.layers.2.mlp.experts.246", - "model.language_model.layers.2.mlp.experts.247", - "model.language_model.layers.2.mlp.experts.248", - "model.language_model.layers.2.mlp.experts.249", - "model.language_model.layers.2.mlp.experts.250", - "model.language_model.layers.2.mlp.experts.251", - "model.language_model.layers.2.mlp.experts.252", - "model.language_model.layers.2.mlp.experts.253", - "model.language_model.layers.2.mlp.experts.254", - "model.language_model.layers.2.mlp.experts.255", "model.language_model.layers.2.mlp.shared_expert_gate", "model.language_model.layers.3.mlp.gate", - "model.language_model.layers.3.mlp.experts.0", - "model.language_model.layers.3.mlp.experts.1", - "model.language_model.layers.3.mlp.experts.2", - "model.language_model.layers.3.mlp.experts.3", - "model.language_model.layers.3.mlp.experts.4", - "model.language_model.layers.3.mlp.experts.5", - "model.language_model.layers.3.mlp.experts.6", - "model.language_model.layers.3.mlp.experts.7", - "model.language_model.layers.3.mlp.experts.8", - "model.language_model.layers.3.mlp.experts.9", - "model.language_model.layers.3.mlp.experts.10", - "model.language_model.layers.3.mlp.experts.11", - "model.language_model.layers.3.mlp.experts.12", - "model.language_model.layers.3.mlp.experts.13", - "model.language_model.layers.3.mlp.experts.14", - "model.language_model.layers.3.mlp.experts.15", - "model.language_model.layers.3.mlp.experts.16", - "model.language_model.layers.3.mlp.experts.17", - "model.language_model.layers.3.mlp.experts.18", - "model.language_model.layers.3.mlp.experts.19", - "model.language_model.layers.3.mlp.experts.20", - "model.language_model.layers.3.mlp.experts.21", - "model.language_model.layers.3.mlp.experts.22", - "model.language_model.layers.3.mlp.experts.23", - "model.language_model.layers.3.mlp.experts.24", - "model.language_model.layers.3.mlp.experts.25", - "model.language_model.layers.3.mlp.experts.26", - "model.language_model.layers.3.mlp.experts.27", - "model.language_model.layers.3.mlp.experts.28", - "model.language_model.layers.3.mlp.experts.29", - "model.language_model.layers.3.mlp.experts.30", - "model.language_model.layers.3.mlp.experts.31", - "model.language_model.layers.3.mlp.experts.32", - "model.language_model.layers.3.mlp.experts.33", - "model.language_model.layers.3.mlp.experts.34", - "model.language_model.layers.3.mlp.experts.35", - "model.language_model.layers.3.mlp.experts.36", - "model.language_model.layers.3.mlp.experts.37", - "model.language_model.layers.3.mlp.experts.38", - "model.language_model.layers.3.mlp.experts.39", - "model.language_model.layers.3.mlp.experts.40", - "model.language_model.layers.3.mlp.experts.41", - "model.language_model.layers.3.mlp.experts.42", - "model.language_model.layers.3.mlp.experts.43", - "model.language_model.layers.3.mlp.experts.44", - "model.language_model.layers.3.mlp.experts.45", - "model.language_model.layers.3.mlp.experts.46", - "model.language_model.layers.3.mlp.experts.47", - "model.language_model.layers.3.mlp.experts.48", - "model.language_model.layers.3.mlp.experts.49", - "model.language_model.layers.3.mlp.experts.50", - "model.language_model.layers.3.mlp.experts.51", - "model.language_model.layers.3.mlp.experts.52", - "model.language_model.layers.3.mlp.experts.53", - "model.language_model.layers.3.mlp.experts.54", - "model.language_model.layers.3.mlp.experts.55", - "model.language_model.layers.3.mlp.experts.56", - "model.language_model.layers.3.mlp.experts.57", - "model.language_model.layers.3.mlp.experts.58", - "model.language_model.layers.3.mlp.experts.59", - "model.language_model.layers.3.mlp.experts.60", - "model.language_model.layers.3.mlp.experts.61", - "model.language_model.layers.3.mlp.experts.62", - "model.language_model.layers.3.mlp.experts.63", - "model.language_model.layers.3.mlp.experts.64", - "model.language_model.layers.3.mlp.experts.65", - "model.language_model.layers.3.mlp.experts.66", - "model.language_model.layers.3.mlp.experts.67", - "model.language_model.layers.3.mlp.experts.68", - "model.language_model.layers.3.mlp.experts.69", - "model.language_model.layers.3.mlp.experts.70", - "model.language_model.layers.3.mlp.experts.71", - "model.language_model.layers.3.mlp.experts.72", - "model.language_model.layers.3.mlp.experts.73", - "model.language_model.layers.3.mlp.experts.74", - "model.language_model.layers.3.mlp.experts.75", - "model.language_model.layers.3.mlp.experts.76", - "model.language_model.layers.3.mlp.experts.77", - "model.language_model.layers.3.mlp.experts.78", - "model.language_model.layers.3.mlp.experts.79", - "model.language_model.layers.3.mlp.experts.80", - "model.language_model.layers.3.mlp.experts.81", - "model.language_model.layers.3.mlp.experts.82", - "model.language_model.layers.3.mlp.experts.83", - "model.language_model.layers.3.mlp.experts.84", - "model.language_model.layers.3.mlp.experts.85", - "model.language_model.layers.3.mlp.experts.86", - "model.language_model.layers.3.mlp.experts.87", - "model.language_model.layers.3.mlp.experts.88", - "model.language_model.layers.3.mlp.experts.89", - "model.language_model.layers.3.mlp.experts.90", - "model.language_model.layers.3.mlp.experts.91", - "model.language_model.layers.3.mlp.experts.92", - "model.language_model.layers.3.mlp.experts.93", - "model.language_model.layers.3.mlp.experts.94", - "model.language_model.layers.3.mlp.experts.95", - "model.language_model.layers.3.mlp.experts.96", - "model.language_model.layers.3.mlp.experts.97", - "model.language_model.layers.3.mlp.experts.98", - "model.language_model.layers.3.mlp.experts.99", - "model.language_model.layers.3.mlp.experts.100", - "model.language_model.layers.3.mlp.experts.101", - "model.language_model.layers.3.mlp.experts.102", - "model.language_model.layers.3.mlp.experts.103", - "model.language_model.layers.3.mlp.experts.104", - "model.language_model.layers.3.mlp.experts.105", - "model.language_model.layers.3.mlp.experts.106", - "model.language_model.layers.3.mlp.experts.107", - "model.language_model.layers.3.mlp.experts.108", - "model.language_model.layers.3.mlp.experts.109", - "model.language_model.layers.3.mlp.experts.110", - "model.language_model.layers.3.mlp.experts.111", - "model.language_model.layers.3.mlp.experts.112", - "model.language_model.layers.3.mlp.experts.113", - "model.language_model.layers.3.mlp.experts.114", - "model.language_model.layers.3.mlp.experts.115", - "model.language_model.layers.3.mlp.experts.116", - "model.language_model.layers.3.mlp.experts.117", - "model.language_model.layers.3.mlp.experts.118", - "model.language_model.layers.3.mlp.experts.119", - "model.language_model.layers.3.mlp.experts.120", - "model.language_model.layers.3.mlp.experts.121", - "model.language_model.layers.3.mlp.experts.122", - "model.language_model.layers.3.mlp.experts.123", - "model.language_model.layers.3.mlp.experts.124", - "model.language_model.layers.3.mlp.experts.125", - "model.language_model.layers.3.mlp.experts.126", - "model.language_model.layers.3.mlp.experts.127", - "model.language_model.layers.3.mlp.experts.128", - "model.language_model.layers.3.mlp.experts.129", - "model.language_model.layers.3.mlp.experts.130", - "model.language_model.layers.3.mlp.experts.131", - "model.language_model.layers.3.mlp.experts.132", - "model.language_model.layers.3.mlp.experts.133", - "model.language_model.layers.3.mlp.experts.134", - "model.language_model.layers.3.mlp.experts.135", - "model.language_model.layers.3.mlp.experts.136", - "model.language_model.layers.3.mlp.experts.137", - "model.language_model.layers.3.mlp.experts.138", - "model.language_model.layers.3.mlp.experts.139", - "model.language_model.layers.3.mlp.experts.140", - "model.language_model.layers.3.mlp.experts.141", - "model.language_model.layers.3.mlp.experts.142", - "model.language_model.layers.3.mlp.experts.143", - "model.language_model.layers.3.mlp.experts.144", - "model.language_model.layers.3.mlp.experts.145", - "model.language_model.layers.3.mlp.experts.146", - "model.language_model.layers.3.mlp.experts.147", - "model.language_model.layers.3.mlp.experts.148", - "model.language_model.layers.3.mlp.experts.149", - "model.language_model.layers.3.mlp.experts.150", - "model.language_model.layers.3.mlp.experts.151", - "model.language_model.layers.3.mlp.experts.152", - "model.language_model.layers.3.mlp.experts.153", - "model.language_model.layers.3.mlp.experts.154", - "model.language_model.layers.3.mlp.experts.155", - "model.language_model.layers.3.mlp.experts.156", - "model.language_model.layers.3.mlp.experts.157", - "model.language_model.layers.3.mlp.experts.158", - "model.language_model.layers.3.mlp.experts.159", - "model.language_model.layers.3.mlp.experts.160", - "model.language_model.layers.3.mlp.experts.161", - "model.language_model.layers.3.mlp.experts.162", - "model.language_model.layers.3.mlp.experts.163", - "model.language_model.layers.3.mlp.experts.164", - "model.language_model.layers.3.mlp.experts.165", - "model.language_model.layers.3.mlp.experts.166", - "model.language_model.layers.3.mlp.experts.167", - "model.language_model.layers.3.mlp.experts.168", - "model.language_model.layers.3.mlp.experts.169", - "model.language_model.layers.3.mlp.experts.170", - "model.language_model.layers.3.mlp.experts.171", - "model.language_model.layers.3.mlp.experts.172", - "model.language_model.layers.3.mlp.experts.173", - "model.language_model.layers.3.mlp.experts.174", - "model.language_model.layers.3.mlp.experts.175", - "model.language_model.layers.3.mlp.experts.176", - "model.language_model.layers.3.mlp.experts.177", - "model.language_model.layers.3.mlp.experts.178", - "model.language_model.layers.3.mlp.experts.179", - "model.language_model.layers.3.mlp.experts.180", - "model.language_model.layers.3.mlp.experts.181", - "model.language_model.layers.3.mlp.experts.182", - "model.language_model.layers.3.mlp.experts.183", - "model.language_model.layers.3.mlp.experts.184", - "model.language_model.layers.3.mlp.experts.185", - "model.language_model.layers.3.mlp.experts.186", - "model.language_model.layers.3.mlp.experts.187", - "model.language_model.layers.3.mlp.experts.188", - "model.language_model.layers.3.mlp.experts.189", - "model.language_model.layers.3.mlp.experts.190", - "model.language_model.layers.3.mlp.experts.191", - "model.language_model.layers.3.mlp.experts.192", - "model.language_model.layers.3.mlp.experts.193", - "model.language_model.layers.3.mlp.experts.194", - "model.language_model.layers.3.mlp.experts.195", - "model.language_model.layers.3.mlp.experts.196", - "model.language_model.layers.3.mlp.experts.197", - "model.language_model.layers.3.mlp.experts.198", - "model.language_model.layers.3.mlp.experts.199", - "model.language_model.layers.3.mlp.experts.200", - "model.language_model.layers.3.mlp.experts.201", - "model.language_model.layers.3.mlp.experts.202", - "model.language_model.layers.3.mlp.experts.203", - "model.language_model.layers.3.mlp.experts.204", - "model.language_model.layers.3.mlp.experts.205", - "model.language_model.layers.3.mlp.experts.206", - "model.language_model.layers.3.mlp.experts.207", - "model.language_model.layers.3.mlp.experts.208", - "model.language_model.layers.3.mlp.experts.209", - "model.language_model.layers.3.mlp.experts.210", - "model.language_model.layers.3.mlp.experts.211", - "model.language_model.layers.3.mlp.experts.212", - "model.language_model.layers.3.mlp.experts.213", - "model.language_model.layers.3.mlp.experts.214", - "model.language_model.layers.3.mlp.experts.215", - "model.language_model.layers.3.mlp.experts.216", - "model.language_model.layers.3.mlp.experts.217", - "model.language_model.layers.3.mlp.experts.218", - "model.language_model.layers.3.mlp.experts.219", - "model.language_model.layers.3.mlp.experts.220", - "model.language_model.layers.3.mlp.experts.221", - "model.language_model.layers.3.mlp.experts.222", - "model.language_model.layers.3.mlp.experts.223", - "model.language_model.layers.3.mlp.experts.224", - "model.language_model.layers.3.mlp.experts.225", - "model.language_model.layers.3.mlp.experts.226", - "model.language_model.layers.3.mlp.experts.227", - "model.language_model.layers.3.mlp.experts.228", - "model.language_model.layers.3.mlp.experts.229", - "model.language_model.layers.3.mlp.experts.230", - "model.language_model.layers.3.mlp.experts.231", - "model.language_model.layers.3.mlp.experts.232", - "model.language_model.layers.3.mlp.experts.233", - "model.language_model.layers.3.mlp.experts.234", - "model.language_model.layers.3.mlp.experts.235", - "model.language_model.layers.3.mlp.experts.236", - "model.language_model.layers.3.mlp.experts.237", - "model.language_model.layers.3.mlp.experts.238", - "model.language_model.layers.3.mlp.experts.239", - "model.language_model.layers.3.mlp.experts.240", - "model.language_model.layers.3.mlp.experts.241", - "model.language_model.layers.3.mlp.experts.242", - "model.language_model.layers.3.mlp.experts.243", - "model.language_model.layers.3.mlp.experts.244", - "model.language_model.layers.3.mlp.experts.245", - "model.language_model.layers.3.mlp.experts.246", - "model.language_model.layers.3.mlp.experts.247", - "model.language_model.layers.3.mlp.experts.248", - "model.language_model.layers.3.mlp.experts.249", - "model.language_model.layers.3.mlp.experts.250", - "model.language_model.layers.3.mlp.experts.251", - "model.language_model.layers.3.mlp.experts.252", - "model.language_model.layers.3.mlp.experts.253", - "model.language_model.layers.3.mlp.experts.254", - "model.language_model.layers.3.mlp.experts.255", "model.language_model.layers.3.mlp.shared_expert_gate", - "model.language_model.layers.4.linear_attn", "model.language_model.layers.4.linear_attn.norm", "model.language_model.layers.4.linear_attn.in_proj_b", "model.language_model.layers.4.linear_attn.in_proj_a", "model.language_model.layers.4.mlp.gate", - "model.language_model.layers.4.mlp.experts.0", - "model.language_model.layers.4.mlp.experts.1", - "model.language_model.layers.4.mlp.experts.2", - "model.language_model.layers.4.mlp.experts.3", - "model.language_model.layers.4.mlp.experts.4", - "model.language_model.layers.4.mlp.experts.5", - "model.language_model.layers.4.mlp.experts.6", - "model.language_model.layers.4.mlp.experts.7", - "model.language_model.layers.4.mlp.experts.8", - "model.language_model.layers.4.mlp.experts.9", - "model.language_model.layers.4.mlp.experts.10", - "model.language_model.layers.4.mlp.experts.11", - "model.language_model.layers.4.mlp.experts.12", - "model.language_model.layers.4.mlp.experts.13", - "model.language_model.layers.4.mlp.experts.14", - "model.language_model.layers.4.mlp.experts.15", - "model.language_model.layers.4.mlp.experts.16", - "model.language_model.layers.4.mlp.experts.17", - "model.language_model.layers.4.mlp.experts.18", - "model.language_model.layers.4.mlp.experts.19", - "model.language_model.layers.4.mlp.experts.20", - "model.language_model.layers.4.mlp.experts.21", - "model.language_model.layers.4.mlp.experts.22", - "model.language_model.layers.4.mlp.experts.23", - "model.language_model.layers.4.mlp.experts.24", - "model.language_model.layers.4.mlp.experts.25", - "model.language_model.layers.4.mlp.experts.26", - "model.language_model.layers.4.mlp.experts.27", - "model.language_model.layers.4.mlp.experts.28", - "model.language_model.layers.4.mlp.experts.29", - "model.language_model.layers.4.mlp.experts.30", - "model.language_model.layers.4.mlp.experts.31", - "model.language_model.layers.4.mlp.experts.32", - "model.language_model.layers.4.mlp.experts.33", - "model.language_model.layers.4.mlp.experts.34", - "model.language_model.layers.4.mlp.experts.35", - "model.language_model.layers.4.mlp.experts.36", - "model.language_model.layers.4.mlp.experts.37", - "model.language_model.layers.4.mlp.experts.38", - "model.language_model.layers.4.mlp.experts.39", - "model.language_model.layers.4.mlp.experts.40", - "model.language_model.layers.4.mlp.experts.41", - "model.language_model.layers.4.mlp.experts.42", - "model.language_model.layers.4.mlp.experts.43", - "model.language_model.layers.4.mlp.experts.44", - "model.language_model.layers.4.mlp.experts.45", - "model.language_model.layers.4.mlp.experts.46", - "model.language_model.layers.4.mlp.experts.47", - "model.language_model.layers.4.mlp.experts.48", - "model.language_model.layers.4.mlp.experts.49", - "model.language_model.layers.4.mlp.experts.50", - "model.language_model.layers.4.mlp.experts.51", - "model.language_model.layers.4.mlp.experts.52", - "model.language_model.layers.4.mlp.experts.53", - "model.language_model.layers.4.mlp.experts.54", - "model.language_model.layers.4.mlp.experts.55", - "model.language_model.layers.4.mlp.experts.56", - "model.language_model.layers.4.mlp.experts.57", - "model.language_model.layers.4.mlp.experts.58", - "model.language_model.layers.4.mlp.experts.59", - "model.language_model.layers.4.mlp.experts.60", - "model.language_model.layers.4.mlp.experts.61", - "model.language_model.layers.4.mlp.experts.62", - "model.language_model.layers.4.mlp.experts.63", - "model.language_model.layers.4.mlp.experts.64", - "model.language_model.layers.4.mlp.experts.65", - "model.language_model.layers.4.mlp.experts.66", - "model.language_model.layers.4.mlp.experts.67", - "model.language_model.layers.4.mlp.experts.68", - "model.language_model.layers.4.mlp.experts.69", - "model.language_model.layers.4.mlp.experts.70", - "model.language_model.layers.4.mlp.experts.71", - "model.language_model.layers.4.mlp.experts.72", - "model.language_model.layers.4.mlp.experts.73", - "model.language_model.layers.4.mlp.experts.74", - "model.language_model.layers.4.mlp.experts.75", - "model.language_model.layers.4.mlp.experts.76", - "model.language_model.layers.4.mlp.experts.77", - "model.language_model.layers.4.mlp.experts.78", - "model.language_model.layers.4.mlp.experts.79", - "model.language_model.layers.4.mlp.experts.80", - "model.language_model.layers.4.mlp.experts.81", - "model.language_model.layers.4.mlp.experts.82", - "model.language_model.layers.4.mlp.experts.83", - "model.language_model.layers.4.mlp.experts.84", - "model.language_model.layers.4.mlp.experts.85", - "model.language_model.layers.4.mlp.experts.86", - "model.language_model.layers.4.mlp.experts.87", - "model.language_model.layers.4.mlp.experts.88", - "model.language_model.layers.4.mlp.experts.89", - "model.language_model.layers.4.mlp.experts.90", - "model.language_model.layers.4.mlp.experts.91", - "model.language_model.layers.4.mlp.experts.92", - "model.language_model.layers.4.mlp.experts.93", - "model.language_model.layers.4.mlp.experts.94", - "model.language_model.layers.4.mlp.experts.95", - "model.language_model.layers.4.mlp.experts.96", - "model.language_model.layers.4.mlp.experts.97", - "model.language_model.layers.4.mlp.experts.98", - "model.language_model.layers.4.mlp.experts.99", - "model.language_model.layers.4.mlp.experts.100", - "model.language_model.layers.4.mlp.experts.101", - "model.language_model.layers.4.mlp.experts.102", - "model.language_model.layers.4.mlp.experts.103", - "model.language_model.layers.4.mlp.experts.104", - "model.language_model.layers.4.mlp.experts.105", - "model.language_model.layers.4.mlp.experts.106", - "model.language_model.layers.4.mlp.experts.107", - "model.language_model.layers.4.mlp.experts.108", - "model.language_model.layers.4.mlp.experts.109", - "model.language_model.layers.4.mlp.experts.110", - "model.language_model.layers.4.mlp.experts.111", - "model.language_model.layers.4.mlp.experts.112", - "model.language_model.layers.4.mlp.experts.113", - "model.language_model.layers.4.mlp.experts.114", - "model.language_model.layers.4.mlp.experts.115", - "model.language_model.layers.4.mlp.experts.116", - "model.language_model.layers.4.mlp.experts.117", - "model.language_model.layers.4.mlp.experts.118", - "model.language_model.layers.4.mlp.experts.119", - "model.language_model.layers.4.mlp.experts.120", - "model.language_model.layers.4.mlp.experts.121", - "model.language_model.layers.4.mlp.experts.122", - "model.language_model.layers.4.mlp.experts.123", - "model.language_model.layers.4.mlp.experts.124", - "model.language_model.layers.4.mlp.experts.125", - "model.language_model.layers.4.mlp.experts.126", - "model.language_model.layers.4.mlp.experts.127", - "model.language_model.layers.4.mlp.experts.128", - "model.language_model.layers.4.mlp.experts.129", - "model.language_model.layers.4.mlp.experts.130", - "model.language_model.layers.4.mlp.experts.131", - "model.language_model.layers.4.mlp.experts.132", - "model.language_model.layers.4.mlp.experts.133", - "model.language_model.layers.4.mlp.experts.134", - "model.language_model.layers.4.mlp.experts.135", - "model.language_model.layers.4.mlp.experts.136", - "model.language_model.layers.4.mlp.experts.137", - "model.language_model.layers.4.mlp.experts.138", - "model.language_model.layers.4.mlp.experts.139", - "model.language_model.layers.4.mlp.experts.140", - "model.language_model.layers.4.mlp.experts.141", - "model.language_model.layers.4.mlp.experts.142", - "model.language_model.layers.4.mlp.experts.143", - "model.language_model.layers.4.mlp.experts.144", - "model.language_model.layers.4.mlp.experts.145", - "model.language_model.layers.4.mlp.experts.146", - "model.language_model.layers.4.mlp.experts.147", - "model.language_model.layers.4.mlp.experts.148", - "model.language_model.layers.4.mlp.experts.149", - "model.language_model.layers.4.mlp.experts.150", - "model.language_model.layers.4.mlp.experts.151", - "model.language_model.layers.4.mlp.experts.152", - "model.language_model.layers.4.mlp.experts.153", - "model.language_model.layers.4.mlp.experts.154", - "model.language_model.layers.4.mlp.experts.155", - "model.language_model.layers.4.mlp.experts.156", - "model.language_model.layers.4.mlp.experts.157", - "model.language_model.layers.4.mlp.experts.158", - "model.language_model.layers.4.mlp.experts.159", - "model.language_model.layers.4.mlp.experts.160", - "model.language_model.layers.4.mlp.experts.161", - "model.language_model.layers.4.mlp.experts.162", - "model.language_model.layers.4.mlp.experts.163", - "model.language_model.layers.4.mlp.experts.164", - "model.language_model.layers.4.mlp.experts.165", - "model.language_model.layers.4.mlp.experts.166", - "model.language_model.layers.4.mlp.experts.167", - "model.language_model.layers.4.mlp.experts.168", - "model.language_model.layers.4.mlp.experts.169", - "model.language_model.layers.4.mlp.experts.170", - "model.language_model.layers.4.mlp.experts.171", - "model.language_model.layers.4.mlp.experts.172", - "model.language_model.layers.4.mlp.experts.173", - "model.language_model.layers.4.mlp.experts.174", - "model.language_model.layers.4.mlp.experts.175", - "model.language_model.layers.4.mlp.experts.176", - "model.language_model.layers.4.mlp.experts.177", - "model.language_model.layers.4.mlp.experts.178", - "model.language_model.layers.4.mlp.experts.179", - "model.language_model.layers.4.mlp.experts.180", - "model.language_model.layers.4.mlp.experts.181", - "model.language_model.layers.4.mlp.experts.182", - "model.language_model.layers.4.mlp.experts.183", - "model.language_model.layers.4.mlp.experts.184", - "model.language_model.layers.4.mlp.experts.185", - "model.language_model.layers.4.mlp.experts.186", - "model.language_model.layers.4.mlp.experts.187", - "model.language_model.layers.4.mlp.experts.188", - "model.language_model.layers.4.mlp.experts.189", - "model.language_model.layers.4.mlp.experts.190", - "model.language_model.layers.4.mlp.experts.191", - "model.language_model.layers.4.mlp.experts.192", - "model.language_model.layers.4.mlp.experts.193", - "model.language_model.layers.4.mlp.experts.194", - "model.language_model.layers.4.mlp.experts.195", - "model.language_model.layers.4.mlp.experts.196", - "model.language_model.layers.4.mlp.experts.197", - "model.language_model.layers.4.mlp.experts.198", - "model.language_model.layers.4.mlp.experts.199", - "model.language_model.layers.4.mlp.experts.200", - "model.language_model.layers.4.mlp.experts.201", - "model.language_model.layers.4.mlp.experts.202", - "model.language_model.layers.4.mlp.experts.203", - "model.language_model.layers.4.mlp.experts.204", - "model.language_model.layers.4.mlp.experts.205", - "model.language_model.layers.4.mlp.experts.206", - "model.language_model.layers.4.mlp.experts.207", - "model.language_model.layers.4.mlp.experts.208", - "model.language_model.layers.4.mlp.experts.209", - "model.language_model.layers.4.mlp.experts.210", - "model.language_model.layers.4.mlp.experts.211", - "model.language_model.layers.4.mlp.experts.212", - "model.language_model.layers.4.mlp.experts.213", - "model.language_model.layers.4.mlp.experts.214", - "model.language_model.layers.4.mlp.experts.215", - "model.language_model.layers.4.mlp.experts.216", - "model.language_model.layers.4.mlp.experts.217", - "model.language_model.layers.4.mlp.experts.218", - "model.language_model.layers.4.mlp.experts.219", - "model.language_model.layers.4.mlp.experts.220", - "model.language_model.layers.4.mlp.experts.221", - "model.language_model.layers.4.mlp.experts.222", - "model.language_model.layers.4.mlp.experts.223", - "model.language_model.layers.4.mlp.experts.224", - "model.language_model.layers.4.mlp.experts.225", - "model.language_model.layers.4.mlp.experts.226", - "model.language_model.layers.4.mlp.experts.227", - "model.language_model.layers.4.mlp.experts.228", - "model.language_model.layers.4.mlp.experts.229", - "model.language_model.layers.4.mlp.experts.230", - "model.language_model.layers.4.mlp.experts.231", - "model.language_model.layers.4.mlp.experts.232", - "model.language_model.layers.4.mlp.experts.233", - "model.language_model.layers.4.mlp.experts.234", - "model.language_model.layers.4.mlp.experts.235", - "model.language_model.layers.4.mlp.experts.236", - "model.language_model.layers.4.mlp.experts.237", - "model.language_model.layers.4.mlp.experts.238", - "model.language_model.layers.4.mlp.experts.239", - "model.language_model.layers.4.mlp.experts.240", - "model.language_model.layers.4.mlp.experts.241", - "model.language_model.layers.4.mlp.experts.242", - "model.language_model.layers.4.mlp.experts.243", - "model.language_model.layers.4.mlp.experts.244", - "model.language_model.layers.4.mlp.experts.245", - "model.language_model.layers.4.mlp.experts.246", - "model.language_model.layers.4.mlp.experts.247", - "model.language_model.layers.4.mlp.experts.248", - "model.language_model.layers.4.mlp.experts.249", - "model.language_model.layers.4.mlp.experts.250", - "model.language_model.layers.4.mlp.experts.251", - "model.language_model.layers.4.mlp.experts.252", - "model.language_model.layers.4.mlp.experts.253", - "model.language_model.layers.4.mlp.experts.254", - "model.language_model.layers.4.mlp.experts.255", "model.language_model.layers.4.mlp.shared_expert_gate", - "model.language_model.layers.5.linear_attn", "model.language_model.layers.5.linear_attn.norm", "model.language_model.layers.5.linear_attn.in_proj_b", "model.language_model.layers.5.linear_attn.in_proj_a", "model.language_model.layers.5.mlp.gate", - "model.language_model.layers.5.mlp.experts.0", - "model.language_model.layers.5.mlp.experts.1", - "model.language_model.layers.5.mlp.experts.2", - "model.language_model.layers.5.mlp.experts.3", - "model.language_model.layers.5.mlp.experts.4", - "model.language_model.layers.5.mlp.experts.5", - "model.language_model.layers.5.mlp.experts.6", - "model.language_model.layers.5.mlp.experts.7", - "model.language_model.layers.5.mlp.experts.8", - "model.language_model.layers.5.mlp.experts.9", - "model.language_model.layers.5.mlp.experts.10", - "model.language_model.layers.5.mlp.experts.11", - "model.language_model.layers.5.mlp.experts.12", - "model.language_model.layers.5.mlp.experts.13", - "model.language_model.layers.5.mlp.experts.14", - "model.language_model.layers.5.mlp.experts.15", - "model.language_model.layers.5.mlp.experts.16", - "model.language_model.layers.5.mlp.experts.17", - "model.language_model.layers.5.mlp.experts.18", - "model.language_model.layers.5.mlp.experts.19", - "model.language_model.layers.5.mlp.experts.20", - "model.language_model.layers.5.mlp.experts.21", - "model.language_model.layers.5.mlp.experts.22", - "model.language_model.layers.5.mlp.experts.23", - "model.language_model.layers.5.mlp.experts.24", - "model.language_model.layers.5.mlp.experts.25", - "model.language_model.layers.5.mlp.experts.26", - "model.language_model.layers.5.mlp.experts.27", - "model.language_model.layers.5.mlp.experts.28", - "model.language_model.layers.5.mlp.experts.29", - "model.language_model.layers.5.mlp.experts.30", - "model.language_model.layers.5.mlp.experts.31", - "model.language_model.layers.5.mlp.experts.32", - "model.language_model.layers.5.mlp.experts.33", - "model.language_model.layers.5.mlp.experts.34", - "model.language_model.layers.5.mlp.experts.35", - "model.language_model.layers.5.mlp.experts.36", - "model.language_model.layers.5.mlp.experts.37", - "model.language_model.layers.5.mlp.experts.38", - "model.language_model.layers.5.mlp.experts.39", - "model.language_model.layers.5.mlp.experts.40", - "model.language_model.layers.5.mlp.experts.41", - "model.language_model.layers.5.mlp.experts.42", - "model.language_model.layers.5.mlp.experts.43", - "model.language_model.layers.5.mlp.experts.44", - "model.language_model.layers.5.mlp.experts.45", - "model.language_model.layers.5.mlp.experts.46", - "model.language_model.layers.5.mlp.experts.47", - "model.language_model.layers.5.mlp.experts.48", - "model.language_model.layers.5.mlp.experts.49", - "model.language_model.layers.5.mlp.experts.50", - "model.language_model.layers.5.mlp.experts.51", - "model.language_model.layers.5.mlp.experts.52", - "model.language_model.layers.5.mlp.experts.53", - "model.language_model.layers.5.mlp.experts.54", - "model.language_model.layers.5.mlp.experts.55", - "model.language_model.layers.5.mlp.experts.56", - "model.language_model.layers.5.mlp.experts.57", - "model.language_model.layers.5.mlp.experts.58", - "model.language_model.layers.5.mlp.experts.59", - "model.language_model.layers.5.mlp.experts.60", - "model.language_model.layers.5.mlp.experts.61", - "model.language_model.layers.5.mlp.experts.62", - "model.language_model.layers.5.mlp.experts.63", - "model.language_model.layers.5.mlp.experts.64", - "model.language_model.layers.5.mlp.experts.65", - "model.language_model.layers.5.mlp.experts.66", - "model.language_model.layers.5.mlp.experts.67", - "model.language_model.layers.5.mlp.experts.68", - "model.language_model.layers.5.mlp.experts.69", - "model.language_model.layers.5.mlp.experts.70", - "model.language_model.layers.5.mlp.experts.71", - "model.language_model.layers.5.mlp.experts.72", - "model.language_model.layers.5.mlp.experts.73", - "model.language_model.layers.5.mlp.experts.74", - "model.language_model.layers.5.mlp.experts.75", - "model.language_model.layers.5.mlp.experts.76", - "model.language_model.layers.5.mlp.experts.77", - "model.language_model.layers.5.mlp.experts.78", - "model.language_model.layers.5.mlp.experts.79", - "model.language_model.layers.5.mlp.experts.80", - "model.language_model.layers.5.mlp.experts.81", - "model.language_model.layers.5.mlp.experts.82", - "model.language_model.layers.5.mlp.experts.83", - "model.language_model.layers.5.mlp.experts.84", - "model.language_model.layers.5.mlp.experts.85", - "model.language_model.layers.5.mlp.experts.86", - "model.language_model.layers.5.mlp.experts.87", - "model.language_model.layers.5.mlp.experts.88", - "model.language_model.layers.5.mlp.experts.89", - "model.language_model.layers.5.mlp.experts.90", - "model.language_model.layers.5.mlp.experts.91", - "model.language_model.layers.5.mlp.experts.92", - "model.language_model.layers.5.mlp.experts.93", - "model.language_model.layers.5.mlp.experts.94", - "model.language_model.layers.5.mlp.experts.95", - "model.language_model.layers.5.mlp.experts.96", - "model.language_model.layers.5.mlp.experts.97", - "model.language_model.layers.5.mlp.experts.98", - "model.language_model.layers.5.mlp.experts.99", - "model.language_model.layers.5.mlp.experts.100", - "model.language_model.layers.5.mlp.experts.101", - "model.language_model.layers.5.mlp.experts.102", - "model.language_model.layers.5.mlp.experts.103", - "model.language_model.layers.5.mlp.experts.104", - "model.language_model.layers.5.mlp.experts.105", - "model.language_model.layers.5.mlp.experts.106", - "model.language_model.layers.5.mlp.experts.107", - "model.language_model.layers.5.mlp.experts.108", - "model.language_model.layers.5.mlp.experts.109", - "model.language_model.layers.5.mlp.experts.110", - "model.language_model.layers.5.mlp.experts.111", - "model.language_model.layers.5.mlp.experts.112", - "model.language_model.layers.5.mlp.experts.113", - "model.language_model.layers.5.mlp.experts.114", - "model.language_model.layers.5.mlp.experts.115", - "model.language_model.layers.5.mlp.experts.116", - "model.language_model.layers.5.mlp.experts.117", - "model.language_model.layers.5.mlp.experts.118", - "model.language_model.layers.5.mlp.experts.119", - "model.language_model.layers.5.mlp.experts.120", - "model.language_model.layers.5.mlp.experts.121", - "model.language_model.layers.5.mlp.experts.122", - "model.language_model.layers.5.mlp.experts.123", - "model.language_model.layers.5.mlp.experts.124", - "model.language_model.layers.5.mlp.experts.125", - "model.language_model.layers.5.mlp.experts.126", - "model.language_model.layers.5.mlp.experts.127", - "model.language_model.layers.5.mlp.experts.128", - "model.language_model.layers.5.mlp.experts.129", - "model.language_model.layers.5.mlp.experts.130", - "model.language_model.layers.5.mlp.experts.131", - "model.language_model.layers.5.mlp.experts.132", - "model.language_model.layers.5.mlp.experts.133", - "model.language_model.layers.5.mlp.experts.134", - "model.language_model.layers.5.mlp.experts.135", - "model.language_model.layers.5.mlp.experts.136", - "model.language_model.layers.5.mlp.experts.137", - "model.language_model.layers.5.mlp.experts.138", - "model.language_model.layers.5.mlp.experts.139", - "model.language_model.layers.5.mlp.experts.140", - "model.language_model.layers.5.mlp.experts.141", - "model.language_model.layers.5.mlp.experts.142", - "model.language_model.layers.5.mlp.experts.143", - "model.language_model.layers.5.mlp.experts.144", - "model.language_model.layers.5.mlp.experts.145", - "model.language_model.layers.5.mlp.experts.146", - "model.language_model.layers.5.mlp.experts.147", - "model.language_model.layers.5.mlp.experts.148", - "model.language_model.layers.5.mlp.experts.149", - "model.language_model.layers.5.mlp.experts.150", - "model.language_model.layers.5.mlp.experts.151", - "model.language_model.layers.5.mlp.experts.152", - "model.language_model.layers.5.mlp.experts.153", - "model.language_model.layers.5.mlp.experts.154", - "model.language_model.layers.5.mlp.experts.155", - "model.language_model.layers.5.mlp.experts.156", - "model.language_model.layers.5.mlp.experts.157", - "model.language_model.layers.5.mlp.experts.158", - "model.language_model.layers.5.mlp.experts.159", - "model.language_model.layers.5.mlp.experts.160", - "model.language_model.layers.5.mlp.experts.161", - "model.language_model.layers.5.mlp.experts.162", - "model.language_model.layers.5.mlp.experts.163", - "model.language_model.layers.5.mlp.experts.164", - "model.language_model.layers.5.mlp.experts.165", - "model.language_model.layers.5.mlp.experts.166", - "model.language_model.layers.5.mlp.experts.167", - "model.language_model.layers.5.mlp.experts.168", - "model.language_model.layers.5.mlp.experts.169", - "model.language_model.layers.5.mlp.experts.170", - "model.language_model.layers.5.mlp.experts.171", - "model.language_model.layers.5.mlp.experts.172", - "model.language_model.layers.5.mlp.experts.173", - "model.language_model.layers.5.mlp.experts.174", - "model.language_model.layers.5.mlp.experts.175", - "model.language_model.layers.5.mlp.experts.176", - "model.language_model.layers.5.mlp.experts.177", - "model.language_model.layers.5.mlp.experts.178", - "model.language_model.layers.5.mlp.experts.179", - "model.language_model.layers.5.mlp.experts.180", - "model.language_model.layers.5.mlp.experts.181", - "model.language_model.layers.5.mlp.experts.182", - "model.language_model.layers.5.mlp.experts.183", - "model.language_model.layers.5.mlp.experts.184", - "model.language_model.layers.5.mlp.experts.185", - "model.language_model.layers.5.mlp.experts.186", - "model.language_model.layers.5.mlp.experts.187", - "model.language_model.layers.5.mlp.experts.188", - "model.language_model.layers.5.mlp.experts.189", - "model.language_model.layers.5.mlp.experts.190", - "model.language_model.layers.5.mlp.experts.191", - "model.language_model.layers.5.mlp.experts.192", - "model.language_model.layers.5.mlp.experts.193", - "model.language_model.layers.5.mlp.experts.194", - "model.language_model.layers.5.mlp.experts.195", - "model.language_model.layers.5.mlp.experts.196", - "model.language_model.layers.5.mlp.experts.197", - "model.language_model.layers.5.mlp.experts.198", - "model.language_model.layers.5.mlp.experts.199", - "model.language_model.layers.5.mlp.experts.200", - "model.language_model.layers.5.mlp.experts.201", - "model.language_model.layers.5.mlp.experts.202", - "model.language_model.layers.5.mlp.experts.203", - "model.language_model.layers.5.mlp.experts.204", - "model.language_model.layers.5.mlp.experts.205", - "model.language_model.layers.5.mlp.experts.206", - "model.language_model.layers.5.mlp.experts.207", - "model.language_model.layers.5.mlp.experts.208", - "model.language_model.layers.5.mlp.experts.209", - "model.language_model.layers.5.mlp.experts.210", - "model.language_model.layers.5.mlp.experts.211", - "model.language_model.layers.5.mlp.experts.212", - "model.language_model.layers.5.mlp.experts.213", - "model.language_model.layers.5.mlp.experts.214", - "model.language_model.layers.5.mlp.experts.215", - "model.language_model.layers.5.mlp.experts.216", - "model.language_model.layers.5.mlp.experts.217", - "model.language_model.layers.5.mlp.experts.218", - "model.language_model.layers.5.mlp.experts.219", - "model.language_model.layers.5.mlp.experts.220", - "model.language_model.layers.5.mlp.experts.221", - "model.language_model.layers.5.mlp.experts.222", - "model.language_model.layers.5.mlp.experts.223", - "model.language_model.layers.5.mlp.experts.224", - "model.language_model.layers.5.mlp.experts.225", - "model.language_model.layers.5.mlp.experts.226", - "model.language_model.layers.5.mlp.experts.227", - "model.language_model.layers.5.mlp.experts.228", - "model.language_model.layers.5.mlp.experts.229", - "model.language_model.layers.5.mlp.experts.230", - "model.language_model.layers.5.mlp.experts.231", - "model.language_model.layers.5.mlp.experts.232", - "model.language_model.layers.5.mlp.experts.233", - "model.language_model.layers.5.mlp.experts.234", - "model.language_model.layers.5.mlp.experts.235", - "model.language_model.layers.5.mlp.experts.236", - "model.language_model.layers.5.mlp.experts.237", - "model.language_model.layers.5.mlp.experts.238", - "model.language_model.layers.5.mlp.experts.239", - "model.language_model.layers.5.mlp.experts.240", - "model.language_model.layers.5.mlp.experts.241", - "model.language_model.layers.5.mlp.experts.242", - "model.language_model.layers.5.mlp.experts.243", - "model.language_model.layers.5.mlp.experts.244", - "model.language_model.layers.5.mlp.experts.245", - "model.language_model.layers.5.mlp.experts.246", - "model.language_model.layers.5.mlp.experts.247", - "model.language_model.layers.5.mlp.experts.248", - "model.language_model.layers.5.mlp.experts.249", - "model.language_model.layers.5.mlp.experts.250", - "model.language_model.layers.5.mlp.experts.251", - "model.language_model.layers.5.mlp.experts.252", - "model.language_model.layers.5.mlp.experts.253", - "model.language_model.layers.5.mlp.experts.254", - "model.language_model.layers.5.mlp.experts.255", "model.language_model.layers.5.mlp.shared_expert_gate", - "model.language_model.layers.6.linear_attn", "model.language_model.layers.6.linear_attn.norm", "model.language_model.layers.6.linear_attn.in_proj_b", "model.language_model.layers.6.linear_attn.in_proj_a", "model.language_model.layers.6.mlp.gate", - "model.language_model.layers.6.mlp.experts.0", - "model.language_model.layers.6.mlp.experts.1", - "model.language_model.layers.6.mlp.experts.2", - "model.language_model.layers.6.mlp.experts.3", - "model.language_model.layers.6.mlp.experts.4", - "model.language_model.layers.6.mlp.experts.5", - "model.language_model.layers.6.mlp.experts.6", - "model.language_model.layers.6.mlp.experts.7", - "model.language_model.layers.6.mlp.experts.8", - "model.language_model.layers.6.mlp.experts.9", - "model.language_model.layers.6.mlp.experts.10", - "model.language_model.layers.6.mlp.experts.11", - "model.language_model.layers.6.mlp.experts.12", - "model.language_model.layers.6.mlp.experts.13", - "model.language_model.layers.6.mlp.experts.14", - "model.language_model.layers.6.mlp.experts.15", - "model.language_model.layers.6.mlp.experts.16", - "model.language_model.layers.6.mlp.experts.17", - "model.language_model.layers.6.mlp.experts.18", - "model.language_model.layers.6.mlp.experts.19", - "model.language_model.layers.6.mlp.experts.20", - "model.language_model.layers.6.mlp.experts.21", - "model.language_model.layers.6.mlp.experts.22", - "model.language_model.layers.6.mlp.experts.23", - "model.language_model.layers.6.mlp.experts.24", - "model.language_model.layers.6.mlp.experts.25", - "model.language_model.layers.6.mlp.experts.26", - "model.language_model.layers.6.mlp.experts.27", - "model.language_model.layers.6.mlp.experts.28", - "model.language_model.layers.6.mlp.experts.29", - "model.language_model.layers.6.mlp.experts.30", - "model.language_model.layers.6.mlp.experts.31", - "model.language_model.layers.6.mlp.experts.32", - "model.language_model.layers.6.mlp.experts.33", - "model.language_model.layers.6.mlp.experts.34", - "model.language_model.layers.6.mlp.experts.35", - "model.language_model.layers.6.mlp.experts.36", - "model.language_model.layers.6.mlp.experts.37", - "model.language_model.layers.6.mlp.experts.38", - "model.language_model.layers.6.mlp.experts.39", - "model.language_model.layers.6.mlp.experts.40", - "model.language_model.layers.6.mlp.experts.41", - "model.language_model.layers.6.mlp.experts.42", - "model.language_model.layers.6.mlp.experts.43", - "model.language_model.layers.6.mlp.experts.44", - "model.language_model.layers.6.mlp.experts.45", - "model.language_model.layers.6.mlp.experts.46", - "model.language_model.layers.6.mlp.experts.47", - "model.language_model.layers.6.mlp.experts.48", - "model.language_model.layers.6.mlp.experts.49", - "model.language_model.layers.6.mlp.experts.50", - "model.language_model.layers.6.mlp.experts.51", - "model.language_model.layers.6.mlp.experts.52", - "model.language_model.layers.6.mlp.experts.53", - "model.language_model.layers.6.mlp.experts.54", - "model.language_model.layers.6.mlp.experts.55", - "model.language_model.layers.6.mlp.experts.56", - "model.language_model.layers.6.mlp.experts.57", - "model.language_model.layers.6.mlp.experts.58", - "model.language_model.layers.6.mlp.experts.59", - "model.language_model.layers.6.mlp.experts.60", - "model.language_model.layers.6.mlp.experts.61", - "model.language_model.layers.6.mlp.experts.62", - "model.language_model.layers.6.mlp.experts.63", - "model.language_model.layers.6.mlp.experts.64", - "model.language_model.layers.6.mlp.experts.65", - "model.language_model.layers.6.mlp.experts.66", - "model.language_model.layers.6.mlp.experts.67", - "model.language_model.layers.6.mlp.experts.68", - "model.language_model.layers.6.mlp.experts.69", - "model.language_model.layers.6.mlp.experts.70", - "model.language_model.layers.6.mlp.experts.71", - "model.language_model.layers.6.mlp.experts.72", - "model.language_model.layers.6.mlp.experts.73", - "model.language_model.layers.6.mlp.experts.74", - "model.language_model.layers.6.mlp.experts.75", - "model.language_model.layers.6.mlp.experts.76", - "model.language_model.layers.6.mlp.experts.77", - "model.language_model.layers.6.mlp.experts.78", - "model.language_model.layers.6.mlp.experts.79", - "model.language_model.layers.6.mlp.experts.80", - "model.language_model.layers.6.mlp.experts.81", - "model.language_model.layers.6.mlp.experts.82", - "model.language_model.layers.6.mlp.experts.83", - "model.language_model.layers.6.mlp.experts.84", - "model.language_model.layers.6.mlp.experts.85", - "model.language_model.layers.6.mlp.experts.86", - "model.language_model.layers.6.mlp.experts.87", - "model.language_model.layers.6.mlp.experts.88", - "model.language_model.layers.6.mlp.experts.89", - "model.language_model.layers.6.mlp.experts.90", - "model.language_model.layers.6.mlp.experts.91", - "model.language_model.layers.6.mlp.experts.92", - "model.language_model.layers.6.mlp.experts.93", - "model.language_model.layers.6.mlp.experts.94", - "model.language_model.layers.6.mlp.experts.95", - "model.language_model.layers.6.mlp.experts.96", - "model.language_model.layers.6.mlp.experts.97", - "model.language_model.layers.6.mlp.experts.98", - "model.language_model.layers.6.mlp.experts.99", - "model.language_model.layers.6.mlp.experts.100", - "model.language_model.layers.6.mlp.experts.101", - "model.language_model.layers.6.mlp.experts.102", - "model.language_model.layers.6.mlp.experts.103", - "model.language_model.layers.6.mlp.experts.104", - "model.language_model.layers.6.mlp.experts.105", - "model.language_model.layers.6.mlp.experts.106", - "model.language_model.layers.6.mlp.experts.107", - "model.language_model.layers.6.mlp.experts.108", - "model.language_model.layers.6.mlp.experts.109", - "model.language_model.layers.6.mlp.experts.110", - "model.language_model.layers.6.mlp.experts.111", - "model.language_model.layers.6.mlp.experts.112", - "model.language_model.layers.6.mlp.experts.113", - "model.language_model.layers.6.mlp.experts.114", - "model.language_model.layers.6.mlp.experts.115", - "model.language_model.layers.6.mlp.experts.116", - "model.language_model.layers.6.mlp.experts.117", - "model.language_model.layers.6.mlp.experts.118", - "model.language_model.layers.6.mlp.experts.119", - "model.language_model.layers.6.mlp.experts.120", - "model.language_model.layers.6.mlp.experts.121", - "model.language_model.layers.6.mlp.experts.122", - "model.language_model.layers.6.mlp.experts.123", - "model.language_model.layers.6.mlp.experts.124", - "model.language_model.layers.6.mlp.experts.125", - "model.language_model.layers.6.mlp.experts.126", - "model.language_model.layers.6.mlp.experts.127", - "model.language_model.layers.6.mlp.experts.128", - "model.language_model.layers.6.mlp.experts.129", - "model.language_model.layers.6.mlp.experts.130", - "model.language_model.layers.6.mlp.experts.131", - "model.language_model.layers.6.mlp.experts.132", - "model.language_model.layers.6.mlp.experts.133", - "model.language_model.layers.6.mlp.experts.134", - "model.language_model.layers.6.mlp.experts.135", - "model.language_model.layers.6.mlp.experts.136", - "model.language_model.layers.6.mlp.experts.137", - "model.language_model.layers.6.mlp.experts.138", - "model.language_model.layers.6.mlp.experts.139", - "model.language_model.layers.6.mlp.experts.140", - "model.language_model.layers.6.mlp.experts.141", - "model.language_model.layers.6.mlp.experts.142", - "model.language_model.layers.6.mlp.experts.143", - "model.language_model.layers.6.mlp.experts.144", - "model.language_model.layers.6.mlp.experts.145", - "model.language_model.layers.6.mlp.experts.146", - "model.language_model.layers.6.mlp.experts.147", - "model.language_model.layers.6.mlp.experts.148", - "model.language_model.layers.6.mlp.experts.149", - "model.language_model.layers.6.mlp.experts.150", - "model.language_model.layers.6.mlp.experts.151", - "model.language_model.layers.6.mlp.experts.152", - "model.language_model.layers.6.mlp.experts.153", - "model.language_model.layers.6.mlp.experts.154", - "model.language_model.layers.6.mlp.experts.155", - "model.language_model.layers.6.mlp.experts.156", - "model.language_model.layers.6.mlp.experts.157", - "model.language_model.layers.6.mlp.experts.158", - "model.language_model.layers.6.mlp.experts.159", - "model.language_model.layers.6.mlp.experts.160", - "model.language_model.layers.6.mlp.experts.161", - "model.language_model.layers.6.mlp.experts.162", - "model.language_model.layers.6.mlp.experts.163", - "model.language_model.layers.6.mlp.experts.164", - "model.language_model.layers.6.mlp.experts.165", - "model.language_model.layers.6.mlp.experts.166", - "model.language_model.layers.6.mlp.experts.167", - "model.language_model.layers.6.mlp.experts.168", - "model.language_model.layers.6.mlp.experts.169", - "model.language_model.layers.6.mlp.experts.170", - "model.language_model.layers.6.mlp.experts.171", - "model.language_model.layers.6.mlp.experts.172", - "model.language_model.layers.6.mlp.experts.173", - "model.language_model.layers.6.mlp.experts.174", - "model.language_model.layers.6.mlp.experts.175", - "model.language_model.layers.6.mlp.experts.176", - "model.language_model.layers.6.mlp.experts.177", - "model.language_model.layers.6.mlp.experts.178", - "model.language_model.layers.6.mlp.experts.179", - "model.language_model.layers.6.mlp.experts.180", - "model.language_model.layers.6.mlp.experts.181", - "model.language_model.layers.6.mlp.experts.182", - "model.language_model.layers.6.mlp.experts.183", - "model.language_model.layers.6.mlp.experts.184", - "model.language_model.layers.6.mlp.experts.185", - "model.language_model.layers.6.mlp.experts.186", - "model.language_model.layers.6.mlp.experts.187", - "model.language_model.layers.6.mlp.experts.188", - "model.language_model.layers.6.mlp.experts.189", - "model.language_model.layers.6.mlp.experts.190", - "model.language_model.layers.6.mlp.experts.191", - "model.language_model.layers.6.mlp.experts.192", - "model.language_model.layers.6.mlp.experts.193", - "model.language_model.layers.6.mlp.experts.194", - "model.language_model.layers.6.mlp.experts.195", - "model.language_model.layers.6.mlp.experts.196", - "model.language_model.layers.6.mlp.experts.197", - "model.language_model.layers.6.mlp.experts.198", - "model.language_model.layers.6.mlp.experts.199", - "model.language_model.layers.6.mlp.experts.200", - "model.language_model.layers.6.mlp.experts.201", - "model.language_model.layers.6.mlp.experts.202", - "model.language_model.layers.6.mlp.experts.203", - "model.language_model.layers.6.mlp.experts.204", - "model.language_model.layers.6.mlp.experts.205", - "model.language_model.layers.6.mlp.experts.206", - "model.language_model.layers.6.mlp.experts.207", - "model.language_model.layers.6.mlp.experts.208", - "model.language_model.layers.6.mlp.experts.209", - "model.language_model.layers.6.mlp.experts.210", - "model.language_model.layers.6.mlp.experts.211", - "model.language_model.layers.6.mlp.experts.212", - "model.language_model.layers.6.mlp.experts.213", - "model.language_model.layers.6.mlp.experts.214", - "model.language_model.layers.6.mlp.experts.215", - "model.language_model.layers.6.mlp.experts.216", - "model.language_model.layers.6.mlp.experts.217", - "model.language_model.layers.6.mlp.experts.218", - "model.language_model.layers.6.mlp.experts.219", - "model.language_model.layers.6.mlp.experts.220", - "model.language_model.layers.6.mlp.experts.221", - "model.language_model.layers.6.mlp.experts.222", - "model.language_model.layers.6.mlp.experts.223", - "model.language_model.layers.6.mlp.experts.224", - "model.language_model.layers.6.mlp.experts.225", - "model.language_model.layers.6.mlp.experts.226", - "model.language_model.layers.6.mlp.experts.227", - "model.language_model.layers.6.mlp.experts.228", - "model.language_model.layers.6.mlp.experts.229", - "model.language_model.layers.6.mlp.experts.230", - "model.language_model.layers.6.mlp.experts.231", - "model.language_model.layers.6.mlp.experts.232", - "model.language_model.layers.6.mlp.experts.233", - "model.language_model.layers.6.mlp.experts.234", - "model.language_model.layers.6.mlp.experts.235", - "model.language_model.layers.6.mlp.experts.236", - "model.language_model.layers.6.mlp.experts.237", - "model.language_model.layers.6.mlp.experts.238", - "model.language_model.layers.6.mlp.experts.239", - "model.language_model.layers.6.mlp.experts.240", - "model.language_model.layers.6.mlp.experts.241", - "model.language_model.layers.6.mlp.experts.242", - "model.language_model.layers.6.mlp.experts.243", - "model.language_model.layers.6.mlp.experts.244", - "model.language_model.layers.6.mlp.experts.245", - "model.language_model.layers.6.mlp.experts.246", - "model.language_model.layers.6.mlp.experts.247", - "model.language_model.layers.6.mlp.experts.248", - "model.language_model.layers.6.mlp.experts.249", - "model.language_model.layers.6.mlp.experts.250", - "model.language_model.layers.6.mlp.experts.251", - "model.language_model.layers.6.mlp.experts.252", - "model.language_model.layers.6.mlp.experts.253", - "model.language_model.layers.6.mlp.experts.254", - "model.language_model.layers.6.mlp.experts.255", "model.language_model.layers.6.mlp.shared_expert_gate", "model.language_model.layers.7.mlp.gate", - "model.language_model.layers.7.mlp.experts.0", - "model.language_model.layers.7.mlp.experts.1", - "model.language_model.layers.7.mlp.experts.2", - "model.language_model.layers.7.mlp.experts.3", - "model.language_model.layers.7.mlp.experts.4", - "model.language_model.layers.7.mlp.experts.5", - "model.language_model.layers.7.mlp.experts.6", - "model.language_model.layers.7.mlp.experts.7", - "model.language_model.layers.7.mlp.experts.8", - "model.language_model.layers.7.mlp.experts.9", - "model.language_model.layers.7.mlp.experts.10", - "model.language_model.layers.7.mlp.experts.11", - "model.language_model.layers.7.mlp.experts.12", - "model.language_model.layers.7.mlp.experts.13", - "model.language_model.layers.7.mlp.experts.14", - "model.language_model.layers.7.mlp.experts.15", - "model.language_model.layers.7.mlp.experts.16", - "model.language_model.layers.7.mlp.experts.17", - "model.language_model.layers.7.mlp.experts.18", - "model.language_model.layers.7.mlp.experts.19", - "model.language_model.layers.7.mlp.experts.20", - "model.language_model.layers.7.mlp.experts.21", - "model.language_model.layers.7.mlp.experts.22", - "model.language_model.layers.7.mlp.experts.23", - "model.language_model.layers.7.mlp.experts.24", - "model.language_model.layers.7.mlp.experts.25", - "model.language_model.layers.7.mlp.experts.26", - "model.language_model.layers.7.mlp.experts.27", - "model.language_model.layers.7.mlp.experts.28", - "model.language_model.layers.7.mlp.experts.29", - "model.language_model.layers.7.mlp.experts.30", - "model.language_model.layers.7.mlp.experts.31", - "model.language_model.layers.7.mlp.experts.32", - "model.language_model.layers.7.mlp.experts.33", - "model.language_model.layers.7.mlp.experts.34", - "model.language_model.layers.7.mlp.experts.35", - "model.language_model.layers.7.mlp.experts.36", - "model.language_model.layers.7.mlp.experts.37", - "model.language_model.layers.7.mlp.experts.38", - "model.language_model.layers.7.mlp.experts.39", - "model.language_model.layers.7.mlp.experts.40", - "model.language_model.layers.7.mlp.experts.41", - "model.language_model.layers.7.mlp.experts.42", - "model.language_model.layers.7.mlp.experts.43", - "model.language_model.layers.7.mlp.experts.44", - "model.language_model.layers.7.mlp.experts.45", - "model.language_model.layers.7.mlp.experts.46", - "model.language_model.layers.7.mlp.experts.47", - "model.language_model.layers.7.mlp.experts.48", - "model.language_model.layers.7.mlp.experts.49", - "model.language_model.layers.7.mlp.experts.50", - "model.language_model.layers.7.mlp.experts.51", - "model.language_model.layers.7.mlp.experts.52", - "model.language_model.layers.7.mlp.experts.53", - "model.language_model.layers.7.mlp.experts.54", - "model.language_model.layers.7.mlp.experts.55", - "model.language_model.layers.7.mlp.experts.56", - "model.language_model.layers.7.mlp.experts.57", - "model.language_model.layers.7.mlp.experts.58", - "model.language_model.layers.7.mlp.experts.59", - "model.language_model.layers.7.mlp.experts.60", - "model.language_model.layers.7.mlp.experts.61", - "model.language_model.layers.7.mlp.experts.62", - "model.language_model.layers.7.mlp.experts.63", - "model.language_model.layers.7.mlp.experts.64", - "model.language_model.layers.7.mlp.experts.65", - "model.language_model.layers.7.mlp.experts.66", - "model.language_model.layers.7.mlp.experts.67", - "model.language_model.layers.7.mlp.experts.68", - "model.language_model.layers.7.mlp.experts.69", - "model.language_model.layers.7.mlp.experts.70", - "model.language_model.layers.7.mlp.experts.71", - "model.language_model.layers.7.mlp.experts.72", - "model.language_model.layers.7.mlp.experts.73", - "model.language_model.layers.7.mlp.experts.74", - "model.language_model.layers.7.mlp.experts.75", - "model.language_model.layers.7.mlp.experts.76", - "model.language_model.layers.7.mlp.experts.77", - "model.language_model.layers.7.mlp.experts.78", - "model.language_model.layers.7.mlp.experts.79", - "model.language_model.layers.7.mlp.experts.80", - "model.language_model.layers.7.mlp.experts.81", - "model.language_model.layers.7.mlp.experts.82", - "model.language_model.layers.7.mlp.experts.83", - "model.language_model.layers.7.mlp.experts.84", - "model.language_model.layers.7.mlp.experts.85", - "model.language_model.layers.7.mlp.experts.86", - "model.language_model.layers.7.mlp.experts.87", - "model.language_model.layers.7.mlp.experts.88", - "model.language_model.layers.7.mlp.experts.89", - "model.language_model.layers.7.mlp.experts.90", - "model.language_model.layers.7.mlp.experts.91", - "model.language_model.layers.7.mlp.experts.92", - "model.language_model.layers.7.mlp.experts.93", - "model.language_model.layers.7.mlp.experts.94", - "model.language_model.layers.7.mlp.experts.95", - "model.language_model.layers.7.mlp.experts.96", - "model.language_model.layers.7.mlp.experts.97", - "model.language_model.layers.7.mlp.experts.98", - "model.language_model.layers.7.mlp.experts.99", - "model.language_model.layers.7.mlp.experts.100", - "model.language_model.layers.7.mlp.experts.101", - "model.language_model.layers.7.mlp.experts.102", - "model.language_model.layers.7.mlp.experts.103", - "model.language_model.layers.7.mlp.experts.104", - "model.language_model.layers.7.mlp.experts.105", - "model.language_model.layers.7.mlp.experts.106", - "model.language_model.layers.7.mlp.experts.107", - "model.language_model.layers.7.mlp.experts.108", - "model.language_model.layers.7.mlp.experts.109", - "model.language_model.layers.7.mlp.experts.110", - "model.language_model.layers.7.mlp.experts.111", - "model.language_model.layers.7.mlp.experts.112", - "model.language_model.layers.7.mlp.experts.113", - "model.language_model.layers.7.mlp.experts.114", - "model.language_model.layers.7.mlp.experts.115", - "model.language_model.layers.7.mlp.experts.116", - "model.language_model.layers.7.mlp.experts.117", - "model.language_model.layers.7.mlp.experts.118", - "model.language_model.layers.7.mlp.experts.119", - "model.language_model.layers.7.mlp.experts.120", - "model.language_model.layers.7.mlp.experts.121", - "model.language_model.layers.7.mlp.experts.122", - "model.language_model.layers.7.mlp.experts.123", - "model.language_model.layers.7.mlp.experts.124", - "model.language_model.layers.7.mlp.experts.125", - "model.language_model.layers.7.mlp.experts.126", - "model.language_model.layers.7.mlp.experts.127", - "model.language_model.layers.7.mlp.experts.128", - "model.language_model.layers.7.mlp.experts.129", - "model.language_model.layers.7.mlp.experts.130", - "model.language_model.layers.7.mlp.experts.131", - "model.language_model.layers.7.mlp.experts.132", - "model.language_model.layers.7.mlp.experts.133", - "model.language_model.layers.7.mlp.experts.134", - "model.language_model.layers.7.mlp.experts.135", - "model.language_model.layers.7.mlp.experts.136", - "model.language_model.layers.7.mlp.experts.137", - "model.language_model.layers.7.mlp.experts.138", - "model.language_model.layers.7.mlp.experts.139", - "model.language_model.layers.7.mlp.experts.140", - "model.language_model.layers.7.mlp.experts.141", - "model.language_model.layers.7.mlp.experts.142", - "model.language_model.layers.7.mlp.experts.143", - "model.language_model.layers.7.mlp.experts.144", - "model.language_model.layers.7.mlp.experts.145", - "model.language_model.layers.7.mlp.experts.146", - "model.language_model.layers.7.mlp.experts.147", - "model.language_model.layers.7.mlp.experts.148", - "model.language_model.layers.7.mlp.experts.149", - "model.language_model.layers.7.mlp.experts.150", - "model.language_model.layers.7.mlp.experts.151", - "model.language_model.layers.7.mlp.experts.152", - "model.language_model.layers.7.mlp.experts.153", - "model.language_model.layers.7.mlp.experts.154", - "model.language_model.layers.7.mlp.experts.155", - "model.language_model.layers.7.mlp.experts.156", - "model.language_model.layers.7.mlp.experts.157", - "model.language_model.layers.7.mlp.experts.158", - "model.language_model.layers.7.mlp.experts.159", - "model.language_model.layers.7.mlp.experts.160", - "model.language_model.layers.7.mlp.experts.161", - "model.language_model.layers.7.mlp.experts.162", - "model.language_model.layers.7.mlp.experts.163", - "model.language_model.layers.7.mlp.experts.164", - "model.language_model.layers.7.mlp.experts.165", - "model.language_model.layers.7.mlp.experts.166", - "model.language_model.layers.7.mlp.experts.167", - "model.language_model.layers.7.mlp.experts.168", - "model.language_model.layers.7.mlp.experts.169", - "model.language_model.layers.7.mlp.experts.170", - "model.language_model.layers.7.mlp.experts.171", - "model.language_model.layers.7.mlp.experts.172", - "model.language_model.layers.7.mlp.experts.173", - "model.language_model.layers.7.mlp.experts.174", - "model.language_model.layers.7.mlp.experts.175", - "model.language_model.layers.7.mlp.experts.176", - "model.language_model.layers.7.mlp.experts.177", - "model.language_model.layers.7.mlp.experts.178", - "model.language_model.layers.7.mlp.experts.179", - "model.language_model.layers.7.mlp.experts.180", - "model.language_model.layers.7.mlp.experts.181", - "model.language_model.layers.7.mlp.experts.182", - "model.language_model.layers.7.mlp.experts.183", - "model.language_model.layers.7.mlp.experts.184", - "model.language_model.layers.7.mlp.experts.185", - "model.language_model.layers.7.mlp.experts.186", - "model.language_model.layers.7.mlp.experts.187", - "model.language_model.layers.7.mlp.experts.188", - "model.language_model.layers.7.mlp.experts.189", - "model.language_model.layers.7.mlp.experts.190", - "model.language_model.layers.7.mlp.experts.191", - "model.language_model.layers.7.mlp.experts.192", - "model.language_model.layers.7.mlp.experts.193", - "model.language_model.layers.7.mlp.experts.194", - "model.language_model.layers.7.mlp.experts.195", - "model.language_model.layers.7.mlp.experts.196", - "model.language_model.layers.7.mlp.experts.197", - "model.language_model.layers.7.mlp.experts.198", - "model.language_model.layers.7.mlp.experts.199", - "model.language_model.layers.7.mlp.experts.200", - "model.language_model.layers.7.mlp.experts.201", - "model.language_model.layers.7.mlp.experts.202", - "model.language_model.layers.7.mlp.experts.203", - "model.language_model.layers.7.mlp.experts.204", - "model.language_model.layers.7.mlp.experts.205", - "model.language_model.layers.7.mlp.experts.206", - "model.language_model.layers.7.mlp.experts.207", - "model.language_model.layers.7.mlp.experts.208", - "model.language_model.layers.7.mlp.experts.209", - "model.language_model.layers.7.mlp.experts.210", - "model.language_model.layers.7.mlp.experts.211", - "model.language_model.layers.7.mlp.experts.212", - "model.language_model.layers.7.mlp.experts.213", - "model.language_model.layers.7.mlp.experts.214", - "model.language_model.layers.7.mlp.experts.215", - "model.language_model.layers.7.mlp.experts.216", - "model.language_model.layers.7.mlp.experts.217", - "model.language_model.layers.7.mlp.experts.218", - "model.language_model.layers.7.mlp.experts.219", - "model.language_model.layers.7.mlp.experts.220", - "model.language_model.layers.7.mlp.experts.221", - "model.language_model.layers.7.mlp.experts.222", - "model.language_model.layers.7.mlp.experts.223", - "model.language_model.layers.7.mlp.experts.224", - "model.language_model.layers.7.mlp.experts.225", - "model.language_model.layers.7.mlp.experts.226", - "model.language_model.layers.7.mlp.experts.227", - "model.language_model.layers.7.mlp.experts.228", - "model.language_model.layers.7.mlp.experts.229", - "model.language_model.layers.7.mlp.experts.230", - "model.language_model.layers.7.mlp.experts.231", - "model.language_model.layers.7.mlp.experts.232", - "model.language_model.layers.7.mlp.experts.233", - "model.language_model.layers.7.mlp.experts.234", - "model.language_model.layers.7.mlp.experts.235", - "model.language_model.layers.7.mlp.experts.236", - "model.language_model.layers.7.mlp.experts.237", - "model.language_model.layers.7.mlp.experts.238", - "model.language_model.layers.7.mlp.experts.239", - "model.language_model.layers.7.mlp.experts.240", - "model.language_model.layers.7.mlp.experts.241", - "model.language_model.layers.7.mlp.experts.242", - "model.language_model.layers.7.mlp.experts.243", - "model.language_model.layers.7.mlp.experts.244", - "model.language_model.layers.7.mlp.experts.245", - "model.language_model.layers.7.mlp.experts.246", - "model.language_model.layers.7.mlp.experts.247", - "model.language_model.layers.7.mlp.experts.248", - "model.language_model.layers.7.mlp.experts.249", - "model.language_model.layers.7.mlp.experts.250", - "model.language_model.layers.7.mlp.experts.251", - "model.language_model.layers.7.mlp.experts.252", - "model.language_model.layers.7.mlp.experts.253", - "model.language_model.layers.7.mlp.experts.254", - "model.language_model.layers.7.mlp.experts.255", "model.language_model.layers.7.mlp.shared_expert_gate", - "model.language_model.layers.8.linear_attn", "model.language_model.layers.8.linear_attn.norm", "model.language_model.layers.8.linear_attn.in_proj_b", "model.language_model.layers.8.linear_attn.in_proj_a", "model.language_model.layers.8.mlp.gate", - "model.language_model.layers.8.mlp.experts.0", - "model.language_model.layers.8.mlp.experts.1", - "model.language_model.layers.8.mlp.experts.2", - "model.language_model.layers.8.mlp.experts.3", - "model.language_model.layers.8.mlp.experts.4", - "model.language_model.layers.8.mlp.experts.5", - "model.language_model.layers.8.mlp.experts.6", - "model.language_model.layers.8.mlp.experts.7", - "model.language_model.layers.8.mlp.experts.8", - "model.language_model.layers.8.mlp.experts.9", - "model.language_model.layers.8.mlp.experts.10", - "model.language_model.layers.8.mlp.experts.11", - "model.language_model.layers.8.mlp.experts.12", - "model.language_model.layers.8.mlp.experts.13", - "model.language_model.layers.8.mlp.experts.14", - "model.language_model.layers.8.mlp.experts.15", - "model.language_model.layers.8.mlp.experts.16", - "model.language_model.layers.8.mlp.experts.17", - "model.language_model.layers.8.mlp.experts.18", - "model.language_model.layers.8.mlp.experts.19", - "model.language_model.layers.8.mlp.experts.20", - "model.language_model.layers.8.mlp.experts.21", - "model.language_model.layers.8.mlp.experts.22", - "model.language_model.layers.8.mlp.experts.23", - "model.language_model.layers.8.mlp.experts.24", - "model.language_model.layers.8.mlp.experts.25", - "model.language_model.layers.8.mlp.experts.26", - "model.language_model.layers.8.mlp.experts.27", - "model.language_model.layers.8.mlp.experts.28", - "model.language_model.layers.8.mlp.experts.29", - "model.language_model.layers.8.mlp.experts.30", - "model.language_model.layers.8.mlp.experts.31", - "model.language_model.layers.8.mlp.experts.32", - "model.language_model.layers.8.mlp.experts.33", - "model.language_model.layers.8.mlp.experts.34", - "model.language_model.layers.8.mlp.experts.35", - "model.language_model.layers.8.mlp.experts.36", - "model.language_model.layers.8.mlp.experts.37", - "model.language_model.layers.8.mlp.experts.38", - "model.language_model.layers.8.mlp.experts.39", - "model.language_model.layers.8.mlp.experts.40", - "model.language_model.layers.8.mlp.experts.41", - "model.language_model.layers.8.mlp.experts.42", - "model.language_model.layers.8.mlp.experts.43", - "model.language_model.layers.8.mlp.experts.44", - "model.language_model.layers.8.mlp.experts.45", - "model.language_model.layers.8.mlp.experts.46", - "model.language_model.layers.8.mlp.experts.47", - "model.language_model.layers.8.mlp.experts.48", - "model.language_model.layers.8.mlp.experts.49", - "model.language_model.layers.8.mlp.experts.50", - "model.language_model.layers.8.mlp.experts.51", - "model.language_model.layers.8.mlp.experts.52", - "model.language_model.layers.8.mlp.experts.53", - "model.language_model.layers.8.mlp.experts.54", - "model.language_model.layers.8.mlp.experts.55", - "model.language_model.layers.8.mlp.experts.56", - "model.language_model.layers.8.mlp.experts.57", - "model.language_model.layers.8.mlp.experts.58", - "model.language_model.layers.8.mlp.experts.59", - "model.language_model.layers.8.mlp.experts.60", - "model.language_model.layers.8.mlp.experts.61", - "model.language_model.layers.8.mlp.experts.62", - "model.language_model.layers.8.mlp.experts.63", - "model.language_model.layers.8.mlp.experts.64", - "model.language_model.layers.8.mlp.experts.65", - "model.language_model.layers.8.mlp.experts.66", - "model.language_model.layers.8.mlp.experts.67", - "model.language_model.layers.8.mlp.experts.68", - "model.language_model.layers.8.mlp.experts.69", - "model.language_model.layers.8.mlp.experts.70", - "model.language_model.layers.8.mlp.experts.71", - "model.language_model.layers.8.mlp.experts.72", - "model.language_model.layers.8.mlp.experts.73", - "model.language_model.layers.8.mlp.experts.74", - "model.language_model.layers.8.mlp.experts.75", - "model.language_model.layers.8.mlp.experts.76", - "model.language_model.layers.8.mlp.experts.77", - "model.language_model.layers.8.mlp.experts.78", - "model.language_model.layers.8.mlp.experts.79", - "model.language_model.layers.8.mlp.experts.80", - "model.language_model.layers.8.mlp.experts.81", - "model.language_model.layers.8.mlp.experts.82", - "model.language_model.layers.8.mlp.experts.83", - "model.language_model.layers.8.mlp.experts.84", - "model.language_model.layers.8.mlp.experts.85", - "model.language_model.layers.8.mlp.experts.86", - "model.language_model.layers.8.mlp.experts.87", - "model.language_model.layers.8.mlp.experts.88", - "model.language_model.layers.8.mlp.experts.89", - "model.language_model.layers.8.mlp.experts.90", - "model.language_model.layers.8.mlp.experts.91", - "model.language_model.layers.8.mlp.experts.92", - "model.language_model.layers.8.mlp.experts.93", - "model.language_model.layers.8.mlp.experts.94", - "model.language_model.layers.8.mlp.experts.95", - "model.language_model.layers.8.mlp.experts.96", - "model.language_model.layers.8.mlp.experts.97", - "model.language_model.layers.8.mlp.experts.98", - "model.language_model.layers.8.mlp.experts.99", - "model.language_model.layers.8.mlp.experts.100", - "model.language_model.layers.8.mlp.experts.101", - "model.language_model.layers.8.mlp.experts.102", - "model.language_model.layers.8.mlp.experts.103", - "model.language_model.layers.8.mlp.experts.104", - "model.language_model.layers.8.mlp.experts.105", - "model.language_model.layers.8.mlp.experts.106", - "model.language_model.layers.8.mlp.experts.107", - "model.language_model.layers.8.mlp.experts.108", - "model.language_model.layers.8.mlp.experts.109", - "model.language_model.layers.8.mlp.experts.110", - "model.language_model.layers.8.mlp.experts.111", - "model.language_model.layers.8.mlp.experts.112", - "model.language_model.layers.8.mlp.experts.113", - "model.language_model.layers.8.mlp.experts.114", - "model.language_model.layers.8.mlp.experts.115", - "model.language_model.layers.8.mlp.experts.116", - "model.language_model.layers.8.mlp.experts.117", - "model.language_model.layers.8.mlp.experts.118", - "model.language_model.layers.8.mlp.experts.119", - "model.language_model.layers.8.mlp.experts.120", - "model.language_model.layers.8.mlp.experts.121", - "model.language_model.layers.8.mlp.experts.122", - "model.language_model.layers.8.mlp.experts.123", - "model.language_model.layers.8.mlp.experts.124", - "model.language_model.layers.8.mlp.experts.125", - "model.language_model.layers.8.mlp.experts.126", - "model.language_model.layers.8.mlp.experts.127", - "model.language_model.layers.8.mlp.experts.128", - "model.language_model.layers.8.mlp.experts.129", - "model.language_model.layers.8.mlp.experts.130", - "model.language_model.layers.8.mlp.experts.131", - "model.language_model.layers.8.mlp.experts.132", - "model.language_model.layers.8.mlp.experts.133", - "model.language_model.layers.8.mlp.experts.134", - "model.language_model.layers.8.mlp.experts.135", - "model.language_model.layers.8.mlp.experts.136", - "model.language_model.layers.8.mlp.experts.137", - "model.language_model.layers.8.mlp.experts.138", - "model.language_model.layers.8.mlp.experts.139", - "model.language_model.layers.8.mlp.experts.140", - "model.language_model.layers.8.mlp.experts.141", - "model.language_model.layers.8.mlp.experts.142", - "model.language_model.layers.8.mlp.experts.143", - "model.language_model.layers.8.mlp.experts.144", - "model.language_model.layers.8.mlp.experts.145", - "model.language_model.layers.8.mlp.experts.146", - "model.language_model.layers.8.mlp.experts.147", - "model.language_model.layers.8.mlp.experts.148", - "model.language_model.layers.8.mlp.experts.149", - "model.language_model.layers.8.mlp.experts.150", - "model.language_model.layers.8.mlp.experts.151", - "model.language_model.layers.8.mlp.experts.152", - "model.language_model.layers.8.mlp.experts.153", - "model.language_model.layers.8.mlp.experts.154", - "model.language_model.layers.8.mlp.experts.155", - "model.language_model.layers.8.mlp.experts.156", - "model.language_model.layers.8.mlp.experts.157", - "model.language_model.layers.8.mlp.experts.158", - "model.language_model.layers.8.mlp.experts.159", - "model.language_model.layers.8.mlp.experts.160", - "model.language_model.layers.8.mlp.experts.161", - "model.language_model.layers.8.mlp.experts.162", - "model.language_model.layers.8.mlp.experts.163", - "model.language_model.layers.8.mlp.experts.164", - "model.language_model.layers.8.mlp.experts.165", - "model.language_model.layers.8.mlp.experts.166", - "model.language_model.layers.8.mlp.experts.167", - "model.language_model.layers.8.mlp.experts.168", - "model.language_model.layers.8.mlp.experts.169", - "model.language_model.layers.8.mlp.experts.170", - "model.language_model.layers.8.mlp.experts.171", - "model.language_model.layers.8.mlp.experts.172", - "model.language_model.layers.8.mlp.experts.173", - "model.language_model.layers.8.mlp.experts.174", - "model.language_model.layers.8.mlp.experts.175", - "model.language_model.layers.8.mlp.experts.176", - "model.language_model.layers.8.mlp.experts.177", - "model.language_model.layers.8.mlp.experts.178", - "model.language_model.layers.8.mlp.experts.179", - "model.language_model.layers.8.mlp.experts.180", - "model.language_model.layers.8.mlp.experts.181", - "model.language_model.layers.8.mlp.experts.182", - "model.language_model.layers.8.mlp.experts.183", - "model.language_model.layers.8.mlp.experts.184", - "model.language_model.layers.8.mlp.experts.185", - "model.language_model.layers.8.mlp.experts.186", - "model.language_model.layers.8.mlp.experts.187", - "model.language_model.layers.8.mlp.experts.188", - "model.language_model.layers.8.mlp.experts.189", - "model.language_model.layers.8.mlp.experts.190", - "model.language_model.layers.8.mlp.experts.191", - "model.language_model.layers.8.mlp.experts.192", - "model.language_model.layers.8.mlp.experts.193", - "model.language_model.layers.8.mlp.experts.194", - "model.language_model.layers.8.mlp.experts.195", - "model.language_model.layers.8.mlp.experts.196", - "model.language_model.layers.8.mlp.experts.197", - "model.language_model.layers.8.mlp.experts.198", - "model.language_model.layers.8.mlp.experts.199", - "model.language_model.layers.8.mlp.experts.200", - "model.language_model.layers.8.mlp.experts.201", - "model.language_model.layers.8.mlp.experts.202", - "model.language_model.layers.8.mlp.experts.203", - "model.language_model.layers.8.mlp.experts.204", - "model.language_model.layers.8.mlp.experts.205", - "model.language_model.layers.8.mlp.experts.206", - "model.language_model.layers.8.mlp.experts.207", - "model.language_model.layers.8.mlp.experts.208", - "model.language_model.layers.8.mlp.experts.209", - "model.language_model.layers.8.mlp.experts.210", - "model.language_model.layers.8.mlp.experts.211", - "model.language_model.layers.8.mlp.experts.212", - "model.language_model.layers.8.mlp.experts.213", - "model.language_model.layers.8.mlp.experts.214", - "model.language_model.layers.8.mlp.experts.215", - "model.language_model.layers.8.mlp.experts.216", - "model.language_model.layers.8.mlp.experts.217", - "model.language_model.layers.8.mlp.experts.218", - "model.language_model.layers.8.mlp.experts.219", - "model.language_model.layers.8.mlp.experts.220", - "model.language_model.layers.8.mlp.experts.221", - "model.language_model.layers.8.mlp.experts.222", - "model.language_model.layers.8.mlp.experts.223", - "model.language_model.layers.8.mlp.experts.224", - "model.language_model.layers.8.mlp.experts.225", - "model.language_model.layers.8.mlp.experts.226", - "model.language_model.layers.8.mlp.experts.227", - "model.language_model.layers.8.mlp.experts.228", - "model.language_model.layers.8.mlp.experts.229", - "model.language_model.layers.8.mlp.experts.230", - "model.language_model.layers.8.mlp.experts.231", - "model.language_model.layers.8.mlp.experts.232", - "model.language_model.layers.8.mlp.experts.233", - "model.language_model.layers.8.mlp.experts.234", - "model.language_model.layers.8.mlp.experts.235", - "model.language_model.layers.8.mlp.experts.236", - "model.language_model.layers.8.mlp.experts.237", - "model.language_model.layers.8.mlp.experts.238", - "model.language_model.layers.8.mlp.experts.239", - "model.language_model.layers.8.mlp.experts.240", - "model.language_model.layers.8.mlp.experts.241", - "model.language_model.layers.8.mlp.experts.242", - "model.language_model.layers.8.mlp.experts.243", - "model.language_model.layers.8.mlp.experts.244", - "model.language_model.layers.8.mlp.experts.245", - "model.language_model.layers.8.mlp.experts.246", - "model.language_model.layers.8.mlp.experts.247", - "model.language_model.layers.8.mlp.experts.248", - "model.language_model.layers.8.mlp.experts.249", - "model.language_model.layers.8.mlp.experts.250", - "model.language_model.layers.8.mlp.experts.251", - "model.language_model.layers.8.mlp.experts.252", - "model.language_model.layers.8.mlp.experts.253", - "model.language_model.layers.8.mlp.experts.254", - "model.language_model.layers.8.mlp.experts.255", "model.language_model.layers.8.mlp.shared_expert_gate", - "model.language_model.layers.9.linear_attn", "model.language_model.layers.9.linear_attn.norm", "model.language_model.layers.9.linear_attn.in_proj_b", "model.language_model.layers.9.linear_attn.in_proj_a", "model.language_model.layers.9.mlp.gate", - "model.language_model.layers.9.mlp.experts.0", - "model.language_model.layers.9.mlp.experts.1", - "model.language_model.layers.9.mlp.experts.2", - "model.language_model.layers.9.mlp.experts.3", - "model.language_model.layers.9.mlp.experts.4", - "model.language_model.layers.9.mlp.experts.5", - "model.language_model.layers.9.mlp.experts.6", - "model.language_model.layers.9.mlp.experts.7", - "model.language_model.layers.9.mlp.experts.8", - "model.language_model.layers.9.mlp.experts.9", - "model.language_model.layers.9.mlp.experts.10", - "model.language_model.layers.9.mlp.experts.11", - "model.language_model.layers.9.mlp.experts.12", - "model.language_model.layers.9.mlp.experts.13", - "model.language_model.layers.9.mlp.experts.14", - "model.language_model.layers.9.mlp.experts.15", - "model.language_model.layers.9.mlp.experts.16", - "model.language_model.layers.9.mlp.experts.17", - "model.language_model.layers.9.mlp.experts.18", - "model.language_model.layers.9.mlp.experts.19", - "model.language_model.layers.9.mlp.experts.20", - "model.language_model.layers.9.mlp.experts.21", - "model.language_model.layers.9.mlp.experts.22", - "model.language_model.layers.9.mlp.experts.23", - "model.language_model.layers.9.mlp.experts.24", - "model.language_model.layers.9.mlp.experts.25", - "model.language_model.layers.9.mlp.experts.26", - "model.language_model.layers.9.mlp.experts.27", - "model.language_model.layers.9.mlp.experts.28", - "model.language_model.layers.9.mlp.experts.29", - "model.language_model.layers.9.mlp.experts.30", - "model.language_model.layers.9.mlp.experts.31", - "model.language_model.layers.9.mlp.experts.32", - "model.language_model.layers.9.mlp.experts.33", - "model.language_model.layers.9.mlp.experts.34", - "model.language_model.layers.9.mlp.experts.35", - "model.language_model.layers.9.mlp.experts.36", - "model.language_model.layers.9.mlp.experts.37", - "model.language_model.layers.9.mlp.experts.38", - "model.language_model.layers.9.mlp.experts.39", - "model.language_model.layers.9.mlp.experts.40", - "model.language_model.layers.9.mlp.experts.41", - "model.language_model.layers.9.mlp.experts.42", - "model.language_model.layers.9.mlp.experts.43", - "model.language_model.layers.9.mlp.experts.44", - "model.language_model.layers.9.mlp.experts.45", - "model.language_model.layers.9.mlp.experts.46", - "model.language_model.layers.9.mlp.experts.47", - "model.language_model.layers.9.mlp.experts.48", - "model.language_model.layers.9.mlp.experts.49", - "model.language_model.layers.9.mlp.experts.50", - "model.language_model.layers.9.mlp.experts.51", - "model.language_model.layers.9.mlp.experts.52", - "model.language_model.layers.9.mlp.experts.53", - "model.language_model.layers.9.mlp.experts.54", - "model.language_model.layers.9.mlp.experts.55", - "model.language_model.layers.9.mlp.experts.56", - "model.language_model.layers.9.mlp.experts.57", - "model.language_model.layers.9.mlp.experts.58", - "model.language_model.layers.9.mlp.experts.59", - "model.language_model.layers.9.mlp.experts.60", - "model.language_model.layers.9.mlp.experts.61", - "model.language_model.layers.9.mlp.experts.62", - "model.language_model.layers.9.mlp.experts.63", - "model.language_model.layers.9.mlp.experts.64", - "model.language_model.layers.9.mlp.experts.65", - "model.language_model.layers.9.mlp.experts.66", - "model.language_model.layers.9.mlp.experts.67", - "model.language_model.layers.9.mlp.experts.68", - "model.language_model.layers.9.mlp.experts.69", - "model.language_model.layers.9.mlp.experts.70", - "model.language_model.layers.9.mlp.experts.71", - "model.language_model.layers.9.mlp.experts.72", - "model.language_model.layers.9.mlp.experts.73", - "model.language_model.layers.9.mlp.experts.74", - "model.language_model.layers.9.mlp.experts.75", - "model.language_model.layers.9.mlp.experts.76", - "model.language_model.layers.9.mlp.experts.77", - "model.language_model.layers.9.mlp.experts.78", - "model.language_model.layers.9.mlp.experts.79", - "model.language_model.layers.9.mlp.experts.80", - "model.language_model.layers.9.mlp.experts.81", - "model.language_model.layers.9.mlp.experts.82", - "model.language_model.layers.9.mlp.experts.83", - "model.language_model.layers.9.mlp.experts.84", - "model.language_model.layers.9.mlp.experts.85", - "model.language_model.layers.9.mlp.experts.86", - "model.language_model.layers.9.mlp.experts.87", - "model.language_model.layers.9.mlp.experts.88", - "model.language_model.layers.9.mlp.experts.89", - "model.language_model.layers.9.mlp.experts.90", - "model.language_model.layers.9.mlp.experts.91", - "model.language_model.layers.9.mlp.experts.92", - "model.language_model.layers.9.mlp.experts.93", - "model.language_model.layers.9.mlp.experts.94", - "model.language_model.layers.9.mlp.experts.95", - "model.language_model.layers.9.mlp.experts.96", - "model.language_model.layers.9.mlp.experts.97", - "model.language_model.layers.9.mlp.experts.98", - "model.language_model.layers.9.mlp.experts.99", - "model.language_model.layers.9.mlp.experts.100", - "model.language_model.layers.9.mlp.experts.101", - "model.language_model.layers.9.mlp.experts.102", - "model.language_model.layers.9.mlp.experts.103", - "model.language_model.layers.9.mlp.experts.104", - "model.language_model.layers.9.mlp.experts.105", - "model.language_model.layers.9.mlp.experts.106", - "model.language_model.layers.9.mlp.experts.107", - "model.language_model.layers.9.mlp.experts.108", - "model.language_model.layers.9.mlp.experts.109", - "model.language_model.layers.9.mlp.experts.110", - "model.language_model.layers.9.mlp.experts.111", - "model.language_model.layers.9.mlp.experts.112", - "model.language_model.layers.9.mlp.experts.113", - "model.language_model.layers.9.mlp.experts.114", - "model.language_model.layers.9.mlp.experts.115", - "model.language_model.layers.9.mlp.experts.116", - "model.language_model.layers.9.mlp.experts.117", - "model.language_model.layers.9.mlp.experts.118", - "model.language_model.layers.9.mlp.experts.119", - "model.language_model.layers.9.mlp.experts.120", - "model.language_model.layers.9.mlp.experts.121", - "model.language_model.layers.9.mlp.experts.122", - "model.language_model.layers.9.mlp.experts.123", - "model.language_model.layers.9.mlp.experts.124", - "model.language_model.layers.9.mlp.experts.125", - "model.language_model.layers.9.mlp.experts.126", - "model.language_model.layers.9.mlp.experts.127", - "model.language_model.layers.9.mlp.experts.128", - "model.language_model.layers.9.mlp.experts.129", - "model.language_model.layers.9.mlp.experts.130", - "model.language_model.layers.9.mlp.experts.131", - "model.language_model.layers.9.mlp.experts.132", - "model.language_model.layers.9.mlp.experts.133", - "model.language_model.layers.9.mlp.experts.134", - "model.language_model.layers.9.mlp.experts.135", - "model.language_model.layers.9.mlp.experts.136", - "model.language_model.layers.9.mlp.experts.137", - "model.language_model.layers.9.mlp.experts.138", - "model.language_model.layers.9.mlp.experts.139", - "model.language_model.layers.9.mlp.experts.140", - "model.language_model.layers.9.mlp.experts.141", - "model.language_model.layers.9.mlp.experts.142", - "model.language_model.layers.9.mlp.experts.143", - "model.language_model.layers.9.mlp.experts.144", - "model.language_model.layers.9.mlp.experts.145", - "model.language_model.layers.9.mlp.experts.146", - "model.language_model.layers.9.mlp.experts.147", - "model.language_model.layers.9.mlp.experts.148", - "model.language_model.layers.9.mlp.experts.149", - "model.language_model.layers.9.mlp.experts.150", - "model.language_model.layers.9.mlp.experts.151", - "model.language_model.layers.9.mlp.experts.152", - "model.language_model.layers.9.mlp.experts.153", - "model.language_model.layers.9.mlp.experts.154", - "model.language_model.layers.9.mlp.experts.155", - "model.language_model.layers.9.mlp.experts.156", - "model.language_model.layers.9.mlp.experts.157", - "model.language_model.layers.9.mlp.experts.158", - "model.language_model.layers.9.mlp.experts.159", - "model.language_model.layers.9.mlp.experts.160", - "model.language_model.layers.9.mlp.experts.161", - "model.language_model.layers.9.mlp.experts.162", - "model.language_model.layers.9.mlp.experts.163", - "model.language_model.layers.9.mlp.experts.164", - "model.language_model.layers.9.mlp.experts.165", - "model.language_model.layers.9.mlp.experts.166", - "model.language_model.layers.9.mlp.experts.167", - "model.language_model.layers.9.mlp.experts.168", - "model.language_model.layers.9.mlp.experts.169", - "model.language_model.layers.9.mlp.experts.170", - "model.language_model.layers.9.mlp.experts.171", - "model.language_model.layers.9.mlp.experts.172", - "model.language_model.layers.9.mlp.experts.173", - "model.language_model.layers.9.mlp.experts.174", - "model.language_model.layers.9.mlp.experts.175", - "model.language_model.layers.9.mlp.experts.176", - "model.language_model.layers.9.mlp.experts.177", - "model.language_model.layers.9.mlp.experts.178", - "model.language_model.layers.9.mlp.experts.179", - "model.language_model.layers.9.mlp.experts.180", - "model.language_model.layers.9.mlp.experts.181", - "model.language_model.layers.9.mlp.experts.182", - "model.language_model.layers.9.mlp.experts.183", - "model.language_model.layers.9.mlp.experts.184", - "model.language_model.layers.9.mlp.experts.185", - "model.language_model.layers.9.mlp.experts.186", - "model.language_model.layers.9.mlp.experts.187", - "model.language_model.layers.9.mlp.experts.188", - "model.language_model.layers.9.mlp.experts.189", - "model.language_model.layers.9.mlp.experts.190", - "model.language_model.layers.9.mlp.experts.191", - "model.language_model.layers.9.mlp.experts.192", - "model.language_model.layers.9.mlp.experts.193", - "model.language_model.layers.9.mlp.experts.194", - "model.language_model.layers.9.mlp.experts.195", - "model.language_model.layers.9.mlp.experts.196", - "model.language_model.layers.9.mlp.experts.197", - "model.language_model.layers.9.mlp.experts.198", - "model.language_model.layers.9.mlp.experts.199", - "model.language_model.layers.9.mlp.experts.200", - "model.language_model.layers.9.mlp.experts.201", - "model.language_model.layers.9.mlp.experts.202", - "model.language_model.layers.9.mlp.experts.203", - "model.language_model.layers.9.mlp.experts.204", - "model.language_model.layers.9.mlp.experts.205", - "model.language_model.layers.9.mlp.experts.206", - "model.language_model.layers.9.mlp.experts.207", - "model.language_model.layers.9.mlp.experts.208", - "model.language_model.layers.9.mlp.experts.209", - "model.language_model.layers.9.mlp.experts.210", - "model.language_model.layers.9.mlp.experts.211", - "model.language_model.layers.9.mlp.experts.212", - "model.language_model.layers.9.mlp.experts.213", - "model.language_model.layers.9.mlp.experts.214", - "model.language_model.layers.9.mlp.experts.215", - "model.language_model.layers.9.mlp.experts.216", - "model.language_model.layers.9.mlp.experts.217", - "model.language_model.layers.9.mlp.experts.218", - "model.language_model.layers.9.mlp.experts.219", - "model.language_model.layers.9.mlp.experts.220", - "model.language_model.layers.9.mlp.experts.221", - "model.language_model.layers.9.mlp.experts.222", - "model.language_model.layers.9.mlp.experts.223", - "model.language_model.layers.9.mlp.experts.224", - "model.language_model.layers.9.mlp.experts.225", - "model.language_model.layers.9.mlp.experts.226", - "model.language_model.layers.9.mlp.experts.227", - "model.language_model.layers.9.mlp.experts.228", - "model.language_model.layers.9.mlp.experts.229", - "model.language_model.layers.9.mlp.experts.230", - "model.language_model.layers.9.mlp.experts.231", - "model.language_model.layers.9.mlp.experts.232", - "model.language_model.layers.9.mlp.experts.233", - "model.language_model.layers.9.mlp.experts.234", - "model.language_model.layers.9.mlp.experts.235", - "model.language_model.layers.9.mlp.experts.236", - "model.language_model.layers.9.mlp.experts.237", - "model.language_model.layers.9.mlp.experts.238", - "model.language_model.layers.9.mlp.experts.239", - "model.language_model.layers.9.mlp.experts.240", - "model.language_model.layers.9.mlp.experts.241", - "model.language_model.layers.9.mlp.experts.242", - "model.language_model.layers.9.mlp.experts.243", - "model.language_model.layers.9.mlp.experts.244", - "model.language_model.layers.9.mlp.experts.245", - "model.language_model.layers.9.mlp.experts.246", - "model.language_model.layers.9.mlp.experts.247", - "model.language_model.layers.9.mlp.experts.248", - "model.language_model.layers.9.mlp.experts.249", - "model.language_model.layers.9.mlp.experts.250", - "model.language_model.layers.9.mlp.experts.251", - "model.language_model.layers.9.mlp.experts.252", - "model.language_model.layers.9.mlp.experts.253", - "model.language_model.layers.9.mlp.experts.254", - "model.language_model.layers.9.mlp.experts.255", "model.language_model.layers.9.mlp.shared_expert_gate", - "model.language_model.layers.10.linear_attn", "model.language_model.layers.10.linear_attn.norm", "model.language_model.layers.10.linear_attn.in_proj_b", "model.language_model.layers.10.linear_attn.in_proj_a", "model.language_model.layers.10.mlp.gate", - "model.language_model.layers.10.mlp.experts.0", - "model.language_model.layers.10.mlp.experts.1", - "model.language_model.layers.10.mlp.experts.2", - "model.language_model.layers.10.mlp.experts.3", - "model.language_model.layers.10.mlp.experts.4", - "model.language_model.layers.10.mlp.experts.5", - "model.language_model.layers.10.mlp.experts.6", - "model.language_model.layers.10.mlp.experts.7", - "model.language_model.layers.10.mlp.experts.8", - "model.language_model.layers.10.mlp.experts.9", - "model.language_model.layers.10.mlp.experts.10", - "model.language_model.layers.10.mlp.experts.11", - "model.language_model.layers.10.mlp.experts.12", - "model.language_model.layers.10.mlp.experts.13", - "model.language_model.layers.10.mlp.experts.14", - "model.language_model.layers.10.mlp.experts.15", - "model.language_model.layers.10.mlp.experts.16", - "model.language_model.layers.10.mlp.experts.17", - "model.language_model.layers.10.mlp.experts.18", - "model.language_model.layers.10.mlp.experts.19", - "model.language_model.layers.10.mlp.experts.20", - "model.language_model.layers.10.mlp.experts.21", - "model.language_model.layers.10.mlp.experts.22", - "model.language_model.layers.10.mlp.experts.23", - "model.language_model.layers.10.mlp.experts.24", - "model.language_model.layers.10.mlp.experts.25", - "model.language_model.layers.10.mlp.experts.26", - "model.language_model.layers.10.mlp.experts.27", - "model.language_model.layers.10.mlp.experts.28", - "model.language_model.layers.10.mlp.experts.29", - "model.language_model.layers.10.mlp.experts.30", - "model.language_model.layers.10.mlp.experts.31", - "model.language_model.layers.10.mlp.experts.32", - "model.language_model.layers.10.mlp.experts.33", - "model.language_model.layers.10.mlp.experts.34", - "model.language_model.layers.10.mlp.experts.35", - "model.language_model.layers.10.mlp.experts.36", - "model.language_model.layers.10.mlp.experts.37", - "model.language_model.layers.10.mlp.experts.38", - "model.language_model.layers.10.mlp.experts.39", - "model.language_model.layers.10.mlp.experts.40", - "model.language_model.layers.10.mlp.experts.41", - "model.language_model.layers.10.mlp.experts.42", - "model.language_model.layers.10.mlp.experts.43", - "model.language_model.layers.10.mlp.experts.44", - "model.language_model.layers.10.mlp.experts.45", - "model.language_model.layers.10.mlp.experts.46", - "model.language_model.layers.10.mlp.experts.47", - "model.language_model.layers.10.mlp.experts.48", - "model.language_model.layers.10.mlp.experts.49", - "model.language_model.layers.10.mlp.experts.50", - "model.language_model.layers.10.mlp.experts.51", - "model.language_model.layers.10.mlp.experts.52", - "model.language_model.layers.10.mlp.experts.53", - "model.language_model.layers.10.mlp.experts.54", - "model.language_model.layers.10.mlp.experts.55", - "model.language_model.layers.10.mlp.experts.56", - "model.language_model.layers.10.mlp.experts.57", - "model.language_model.layers.10.mlp.experts.58", - "model.language_model.layers.10.mlp.experts.59", - "model.language_model.layers.10.mlp.experts.60", - "model.language_model.layers.10.mlp.experts.61", - "model.language_model.layers.10.mlp.experts.62", - "model.language_model.layers.10.mlp.experts.63", - "model.language_model.layers.10.mlp.experts.64", - "model.language_model.layers.10.mlp.experts.65", - "model.language_model.layers.10.mlp.experts.66", - "model.language_model.layers.10.mlp.experts.67", - "model.language_model.layers.10.mlp.experts.68", - "model.language_model.layers.10.mlp.experts.69", - "model.language_model.layers.10.mlp.experts.70", - "model.language_model.layers.10.mlp.experts.71", - "model.language_model.layers.10.mlp.experts.72", - "model.language_model.layers.10.mlp.experts.73", - "model.language_model.layers.10.mlp.experts.74", - "model.language_model.layers.10.mlp.experts.75", - "model.language_model.layers.10.mlp.experts.76", - "model.language_model.layers.10.mlp.experts.77", - "model.language_model.layers.10.mlp.experts.78", - "model.language_model.layers.10.mlp.experts.79", - "model.language_model.layers.10.mlp.experts.80", - "model.language_model.layers.10.mlp.experts.81", - "model.language_model.layers.10.mlp.experts.82", - "model.language_model.layers.10.mlp.experts.83", - "model.language_model.layers.10.mlp.experts.84", - "model.language_model.layers.10.mlp.experts.85", - "model.language_model.layers.10.mlp.experts.86", - "model.language_model.layers.10.mlp.experts.87", - "model.language_model.layers.10.mlp.experts.88", - "model.language_model.layers.10.mlp.experts.89", - "model.language_model.layers.10.mlp.experts.90", - "model.language_model.layers.10.mlp.experts.91", - "model.language_model.layers.10.mlp.experts.92", - "model.language_model.layers.10.mlp.experts.93", - "model.language_model.layers.10.mlp.experts.94", - "model.language_model.layers.10.mlp.experts.95", - "model.language_model.layers.10.mlp.experts.96", - "model.language_model.layers.10.mlp.experts.97", - "model.language_model.layers.10.mlp.experts.98", - "model.language_model.layers.10.mlp.experts.99", - "model.language_model.layers.10.mlp.experts.100", - "model.language_model.layers.10.mlp.experts.101", - "model.language_model.layers.10.mlp.experts.102", - "model.language_model.layers.10.mlp.experts.103", - "model.language_model.layers.10.mlp.experts.104", - "model.language_model.layers.10.mlp.experts.105", - "model.language_model.layers.10.mlp.experts.106", - "model.language_model.layers.10.mlp.experts.107", - "model.language_model.layers.10.mlp.experts.108", - "model.language_model.layers.10.mlp.experts.109", - "model.language_model.layers.10.mlp.experts.110", - "model.language_model.layers.10.mlp.experts.111", - "model.language_model.layers.10.mlp.experts.112", - "model.language_model.layers.10.mlp.experts.113", - "model.language_model.layers.10.mlp.experts.114", - "model.language_model.layers.10.mlp.experts.115", - "model.language_model.layers.10.mlp.experts.116", - "model.language_model.layers.10.mlp.experts.117", - "model.language_model.layers.10.mlp.experts.118", - "model.language_model.layers.10.mlp.experts.119", - "model.language_model.layers.10.mlp.experts.120", - "model.language_model.layers.10.mlp.experts.121", - "model.language_model.layers.10.mlp.experts.122", - "model.language_model.layers.10.mlp.experts.123", - "model.language_model.layers.10.mlp.experts.124", - "model.language_model.layers.10.mlp.experts.125", - "model.language_model.layers.10.mlp.experts.126", - "model.language_model.layers.10.mlp.experts.127", - "model.language_model.layers.10.mlp.experts.128", - "model.language_model.layers.10.mlp.experts.129", - "model.language_model.layers.10.mlp.experts.130", - "model.language_model.layers.10.mlp.experts.131", - "model.language_model.layers.10.mlp.experts.132", - "model.language_model.layers.10.mlp.experts.133", - "model.language_model.layers.10.mlp.experts.134", - "model.language_model.layers.10.mlp.experts.135", - "model.language_model.layers.10.mlp.experts.136", - "model.language_model.layers.10.mlp.experts.137", - "model.language_model.layers.10.mlp.experts.138", - "model.language_model.layers.10.mlp.experts.139", - "model.language_model.layers.10.mlp.experts.140", - "model.language_model.layers.10.mlp.experts.141", - "model.language_model.layers.10.mlp.experts.142", - "model.language_model.layers.10.mlp.experts.143", - "model.language_model.layers.10.mlp.experts.144", - "model.language_model.layers.10.mlp.experts.145", - "model.language_model.layers.10.mlp.experts.146", - "model.language_model.layers.10.mlp.experts.147", - "model.language_model.layers.10.mlp.experts.148", - "model.language_model.layers.10.mlp.experts.149", - "model.language_model.layers.10.mlp.experts.150", - "model.language_model.layers.10.mlp.experts.151", - "model.language_model.layers.10.mlp.experts.152", - "model.language_model.layers.10.mlp.experts.153", - "model.language_model.layers.10.mlp.experts.154", - "model.language_model.layers.10.mlp.experts.155", - "model.language_model.layers.10.mlp.experts.156", - "model.language_model.layers.10.mlp.experts.157", - "model.language_model.layers.10.mlp.experts.158", - "model.language_model.layers.10.mlp.experts.159", - "model.language_model.layers.10.mlp.experts.160", - "model.language_model.layers.10.mlp.experts.161", - "model.language_model.layers.10.mlp.experts.162", - "model.language_model.layers.10.mlp.experts.163", - "model.language_model.layers.10.mlp.experts.164", - "model.language_model.layers.10.mlp.experts.165", - "model.language_model.layers.10.mlp.experts.166", - "model.language_model.layers.10.mlp.experts.167", - "model.language_model.layers.10.mlp.experts.168", - "model.language_model.layers.10.mlp.experts.169", - "model.language_model.layers.10.mlp.experts.170", - "model.language_model.layers.10.mlp.experts.171", - "model.language_model.layers.10.mlp.experts.172", - "model.language_model.layers.10.mlp.experts.173", - "model.language_model.layers.10.mlp.experts.174", - "model.language_model.layers.10.mlp.experts.175", - "model.language_model.layers.10.mlp.experts.176", - "model.language_model.layers.10.mlp.experts.177", - "model.language_model.layers.10.mlp.experts.178", - "model.language_model.layers.10.mlp.experts.179", - "model.language_model.layers.10.mlp.experts.180", - "model.language_model.layers.10.mlp.experts.181", - "model.language_model.layers.10.mlp.experts.182", - "model.language_model.layers.10.mlp.experts.183", - "model.language_model.layers.10.mlp.experts.184", - "model.language_model.layers.10.mlp.experts.185", - "model.language_model.layers.10.mlp.experts.186", - "model.language_model.layers.10.mlp.experts.187", - "model.language_model.layers.10.mlp.experts.188", - "model.language_model.layers.10.mlp.experts.189", - "model.language_model.layers.10.mlp.experts.190", - "model.language_model.layers.10.mlp.experts.191", - "model.language_model.layers.10.mlp.experts.192", - "model.language_model.layers.10.mlp.experts.193", - "model.language_model.layers.10.mlp.experts.194", - "model.language_model.layers.10.mlp.experts.195", - "model.language_model.layers.10.mlp.experts.196", - "model.language_model.layers.10.mlp.experts.197", - "model.language_model.layers.10.mlp.experts.198", - "model.language_model.layers.10.mlp.experts.199", - "model.language_model.layers.10.mlp.experts.200", - "model.language_model.layers.10.mlp.experts.201", - "model.language_model.layers.10.mlp.experts.202", - "model.language_model.layers.10.mlp.experts.203", - "model.language_model.layers.10.mlp.experts.204", - "model.language_model.layers.10.mlp.experts.205", - "model.language_model.layers.10.mlp.experts.206", - "model.language_model.layers.10.mlp.experts.207", - "model.language_model.layers.10.mlp.experts.208", - "model.language_model.layers.10.mlp.experts.209", - "model.language_model.layers.10.mlp.experts.210", - "model.language_model.layers.10.mlp.experts.211", - "model.language_model.layers.10.mlp.experts.212", - "model.language_model.layers.10.mlp.experts.213", - "model.language_model.layers.10.mlp.experts.214", - "model.language_model.layers.10.mlp.experts.215", - "model.language_model.layers.10.mlp.experts.216", - "model.language_model.layers.10.mlp.experts.217", - "model.language_model.layers.10.mlp.experts.218", - "model.language_model.layers.10.mlp.experts.219", - "model.language_model.layers.10.mlp.experts.220", - "model.language_model.layers.10.mlp.experts.221", - "model.language_model.layers.10.mlp.experts.222", - "model.language_model.layers.10.mlp.experts.223", - "model.language_model.layers.10.mlp.experts.224", - "model.language_model.layers.10.mlp.experts.225", - "model.language_model.layers.10.mlp.experts.226", - "model.language_model.layers.10.mlp.experts.227", - "model.language_model.layers.10.mlp.experts.228", - "model.language_model.layers.10.mlp.experts.229", - "model.language_model.layers.10.mlp.experts.230", - "model.language_model.layers.10.mlp.experts.231", - "model.language_model.layers.10.mlp.experts.232", - "model.language_model.layers.10.mlp.experts.233", - "model.language_model.layers.10.mlp.experts.234", - "model.language_model.layers.10.mlp.experts.235", - "model.language_model.layers.10.mlp.experts.236", - "model.language_model.layers.10.mlp.experts.237", - "model.language_model.layers.10.mlp.experts.238", - "model.language_model.layers.10.mlp.experts.239", - "model.language_model.layers.10.mlp.experts.240", - "model.language_model.layers.10.mlp.experts.241", - "model.language_model.layers.10.mlp.experts.242", - "model.language_model.layers.10.mlp.experts.243", - "model.language_model.layers.10.mlp.experts.244", - "model.language_model.layers.10.mlp.experts.245", - "model.language_model.layers.10.mlp.experts.246", - "model.language_model.layers.10.mlp.experts.247", - "model.language_model.layers.10.mlp.experts.248", - "model.language_model.layers.10.mlp.experts.249", - "model.language_model.layers.10.mlp.experts.250", - "model.language_model.layers.10.mlp.experts.251", - "model.language_model.layers.10.mlp.experts.252", - "model.language_model.layers.10.mlp.experts.253", - "model.language_model.layers.10.mlp.experts.254", - "model.language_model.layers.10.mlp.experts.255", "model.language_model.layers.10.mlp.shared_expert_gate", "model.language_model.layers.11.mlp.gate", - "model.language_model.layers.11.mlp.experts.0", - "model.language_model.layers.11.mlp.experts.1", - "model.language_model.layers.11.mlp.experts.2", - "model.language_model.layers.11.mlp.experts.3", - "model.language_model.layers.11.mlp.experts.4", - "model.language_model.layers.11.mlp.experts.5", - "model.language_model.layers.11.mlp.experts.6", - "model.language_model.layers.11.mlp.experts.7", - "model.language_model.layers.11.mlp.experts.8", - "model.language_model.layers.11.mlp.experts.9", - "model.language_model.layers.11.mlp.experts.10", - "model.language_model.layers.11.mlp.experts.11", - "model.language_model.layers.11.mlp.experts.12", - "model.language_model.layers.11.mlp.experts.13", - "model.language_model.layers.11.mlp.experts.14", - "model.language_model.layers.11.mlp.experts.15", - "model.language_model.layers.11.mlp.experts.16", - "model.language_model.layers.11.mlp.experts.17", - "model.language_model.layers.11.mlp.experts.18", - "model.language_model.layers.11.mlp.experts.19", - "model.language_model.layers.11.mlp.experts.20", - "model.language_model.layers.11.mlp.experts.21", - "model.language_model.layers.11.mlp.experts.22", - "model.language_model.layers.11.mlp.experts.23", - "model.language_model.layers.11.mlp.experts.24", - "model.language_model.layers.11.mlp.experts.25", - "model.language_model.layers.11.mlp.experts.26", - "model.language_model.layers.11.mlp.experts.27", - "model.language_model.layers.11.mlp.experts.28", - "model.language_model.layers.11.mlp.experts.29", - "model.language_model.layers.11.mlp.experts.30", - "model.language_model.layers.11.mlp.experts.31", - "model.language_model.layers.11.mlp.experts.32", - "model.language_model.layers.11.mlp.experts.33", - "model.language_model.layers.11.mlp.experts.34", - "model.language_model.layers.11.mlp.experts.35", - "model.language_model.layers.11.mlp.experts.36", - "model.language_model.layers.11.mlp.experts.37", - "model.language_model.layers.11.mlp.experts.38", - "model.language_model.layers.11.mlp.experts.39", - "model.language_model.layers.11.mlp.experts.40", - "model.language_model.layers.11.mlp.experts.41", - "model.language_model.layers.11.mlp.experts.42", - "model.language_model.layers.11.mlp.experts.43", - "model.language_model.layers.11.mlp.experts.44", - "model.language_model.layers.11.mlp.experts.45", - "model.language_model.layers.11.mlp.experts.46", - "model.language_model.layers.11.mlp.experts.47", - "model.language_model.layers.11.mlp.experts.48", - "model.language_model.layers.11.mlp.experts.49", - "model.language_model.layers.11.mlp.experts.50", - "model.language_model.layers.11.mlp.experts.51", - "model.language_model.layers.11.mlp.experts.52", - "model.language_model.layers.11.mlp.experts.53", - "model.language_model.layers.11.mlp.experts.54", - "model.language_model.layers.11.mlp.experts.55", - "model.language_model.layers.11.mlp.experts.56", - "model.language_model.layers.11.mlp.experts.57", - "model.language_model.layers.11.mlp.experts.58", - "model.language_model.layers.11.mlp.experts.59", - "model.language_model.layers.11.mlp.experts.60", - "model.language_model.layers.11.mlp.experts.61", - "model.language_model.layers.11.mlp.experts.62", - "model.language_model.layers.11.mlp.experts.63", - "model.language_model.layers.11.mlp.experts.64", - "model.language_model.layers.11.mlp.experts.65", - "model.language_model.layers.11.mlp.experts.66", - "model.language_model.layers.11.mlp.experts.67", - "model.language_model.layers.11.mlp.experts.68", - "model.language_model.layers.11.mlp.experts.69", - "model.language_model.layers.11.mlp.experts.70", - "model.language_model.layers.11.mlp.experts.71", - "model.language_model.layers.11.mlp.experts.72", - "model.language_model.layers.11.mlp.experts.73", - "model.language_model.layers.11.mlp.experts.74", - "model.language_model.layers.11.mlp.experts.75", - "model.language_model.layers.11.mlp.experts.76", - "model.language_model.layers.11.mlp.experts.77", - "model.language_model.layers.11.mlp.experts.78", - "model.language_model.layers.11.mlp.experts.79", - "model.language_model.layers.11.mlp.experts.80", - "model.language_model.layers.11.mlp.experts.81", - "model.language_model.layers.11.mlp.experts.82", - "model.language_model.layers.11.mlp.experts.83", - "model.language_model.layers.11.mlp.experts.84", - "model.language_model.layers.11.mlp.experts.85", - "model.language_model.layers.11.mlp.experts.86", - "model.language_model.layers.11.mlp.experts.87", - "model.language_model.layers.11.mlp.experts.88", - "model.language_model.layers.11.mlp.experts.89", - "model.language_model.layers.11.mlp.experts.90", - "model.language_model.layers.11.mlp.experts.91", - "model.language_model.layers.11.mlp.experts.92", - "model.language_model.layers.11.mlp.experts.93", - "model.language_model.layers.11.mlp.experts.94", - "model.language_model.layers.11.mlp.experts.95", - "model.language_model.layers.11.mlp.experts.96", - "model.language_model.layers.11.mlp.experts.97", - "model.language_model.layers.11.mlp.experts.98", - "model.language_model.layers.11.mlp.experts.99", - "model.language_model.layers.11.mlp.experts.100", - "model.language_model.layers.11.mlp.experts.101", - "model.language_model.layers.11.mlp.experts.102", - "model.language_model.layers.11.mlp.experts.103", - "model.language_model.layers.11.mlp.experts.104", - "model.language_model.layers.11.mlp.experts.105", - "model.language_model.layers.11.mlp.experts.106", - "model.language_model.layers.11.mlp.experts.107", - "model.language_model.layers.11.mlp.experts.108", - "model.language_model.layers.11.mlp.experts.109", - "model.language_model.layers.11.mlp.experts.110", - "model.language_model.layers.11.mlp.experts.111", - "model.language_model.layers.11.mlp.experts.112", - "model.language_model.layers.11.mlp.experts.113", - "model.language_model.layers.11.mlp.experts.114", - "model.language_model.layers.11.mlp.experts.115", - "model.language_model.layers.11.mlp.experts.116", - "model.language_model.layers.11.mlp.experts.117", - "model.language_model.layers.11.mlp.experts.118", - "model.language_model.layers.11.mlp.experts.119", - "model.language_model.layers.11.mlp.experts.120", - "model.language_model.layers.11.mlp.experts.121", - "model.language_model.layers.11.mlp.experts.122", - "model.language_model.layers.11.mlp.experts.123", - "model.language_model.layers.11.mlp.experts.124", - "model.language_model.layers.11.mlp.experts.125", - "model.language_model.layers.11.mlp.experts.126", - "model.language_model.layers.11.mlp.experts.127", - "model.language_model.layers.11.mlp.experts.128", - "model.language_model.layers.11.mlp.experts.129", - "model.language_model.layers.11.mlp.experts.130", - "model.language_model.layers.11.mlp.experts.131", - "model.language_model.layers.11.mlp.experts.132", - "model.language_model.layers.11.mlp.experts.133", - "model.language_model.layers.11.mlp.experts.134", - "model.language_model.layers.11.mlp.experts.135", - "model.language_model.layers.11.mlp.experts.136", - "model.language_model.layers.11.mlp.experts.137", - "model.language_model.layers.11.mlp.experts.138", - "model.language_model.layers.11.mlp.experts.139", - "model.language_model.layers.11.mlp.experts.140", - "model.language_model.layers.11.mlp.experts.141", - "model.language_model.layers.11.mlp.experts.142", - "model.language_model.layers.11.mlp.experts.143", - "model.language_model.layers.11.mlp.experts.144", - "model.language_model.layers.11.mlp.experts.145", - "model.language_model.layers.11.mlp.experts.146", - "model.language_model.layers.11.mlp.experts.147", - "model.language_model.layers.11.mlp.experts.148", - "model.language_model.layers.11.mlp.experts.149", - "model.language_model.layers.11.mlp.experts.150", - "model.language_model.layers.11.mlp.experts.151", - "model.language_model.layers.11.mlp.experts.152", - "model.language_model.layers.11.mlp.experts.153", - "model.language_model.layers.11.mlp.experts.154", - "model.language_model.layers.11.mlp.experts.155", - "model.language_model.layers.11.mlp.experts.156", - "model.language_model.layers.11.mlp.experts.157", - "model.language_model.layers.11.mlp.experts.158", - "model.language_model.layers.11.mlp.experts.159", - "model.language_model.layers.11.mlp.experts.160", - "model.language_model.layers.11.mlp.experts.161", - "model.language_model.layers.11.mlp.experts.162", - "model.language_model.layers.11.mlp.experts.163", - "model.language_model.layers.11.mlp.experts.164", - "model.language_model.layers.11.mlp.experts.165", - "model.language_model.layers.11.mlp.experts.166", - "model.language_model.layers.11.mlp.experts.167", - "model.language_model.layers.11.mlp.experts.168", - "model.language_model.layers.11.mlp.experts.169", - "model.language_model.layers.11.mlp.experts.170", - "model.language_model.layers.11.mlp.experts.171", - "model.language_model.layers.11.mlp.experts.172", - "model.language_model.layers.11.mlp.experts.173", - "model.language_model.layers.11.mlp.experts.174", - "model.language_model.layers.11.mlp.experts.175", - "model.language_model.layers.11.mlp.experts.176", - "model.language_model.layers.11.mlp.experts.177", - "model.language_model.layers.11.mlp.experts.178", - "model.language_model.layers.11.mlp.experts.179", - "model.language_model.layers.11.mlp.experts.180", - "model.language_model.layers.11.mlp.experts.181", - "model.language_model.layers.11.mlp.experts.182", - "model.language_model.layers.11.mlp.experts.183", - "model.language_model.layers.11.mlp.experts.184", - "model.language_model.layers.11.mlp.experts.185", - "model.language_model.layers.11.mlp.experts.186", - "model.language_model.layers.11.mlp.experts.187", - "model.language_model.layers.11.mlp.experts.188", - "model.language_model.layers.11.mlp.experts.189", - "model.language_model.layers.11.mlp.experts.190", - "model.language_model.layers.11.mlp.experts.191", - "model.language_model.layers.11.mlp.experts.192", - "model.language_model.layers.11.mlp.experts.193", - "model.language_model.layers.11.mlp.experts.194", - "model.language_model.layers.11.mlp.experts.195", - "model.language_model.layers.11.mlp.experts.196", - "model.language_model.layers.11.mlp.experts.197", - "model.language_model.layers.11.mlp.experts.198", - "model.language_model.layers.11.mlp.experts.199", - "model.language_model.layers.11.mlp.experts.200", - "model.language_model.layers.11.mlp.experts.201", - "model.language_model.layers.11.mlp.experts.202", - "model.language_model.layers.11.mlp.experts.203", - "model.language_model.layers.11.mlp.experts.204", - "model.language_model.layers.11.mlp.experts.205", - "model.language_model.layers.11.mlp.experts.206", - "model.language_model.layers.11.mlp.experts.207", - "model.language_model.layers.11.mlp.experts.208", - "model.language_model.layers.11.mlp.experts.209", - "model.language_model.layers.11.mlp.experts.210", - "model.language_model.layers.11.mlp.experts.211", - "model.language_model.layers.11.mlp.experts.212", - "model.language_model.layers.11.mlp.experts.213", - "model.language_model.layers.11.mlp.experts.214", - "model.language_model.layers.11.mlp.experts.215", - "model.language_model.layers.11.mlp.experts.216", - "model.language_model.layers.11.mlp.experts.217", - "model.language_model.layers.11.mlp.experts.218", - "model.language_model.layers.11.mlp.experts.219", - "model.language_model.layers.11.mlp.experts.220", - "model.language_model.layers.11.mlp.experts.221", - "model.language_model.layers.11.mlp.experts.222", - "model.language_model.layers.11.mlp.experts.223", - "model.language_model.layers.11.mlp.experts.224", - "model.language_model.layers.11.mlp.experts.225", - "model.language_model.layers.11.mlp.experts.226", - "model.language_model.layers.11.mlp.experts.227", - "model.language_model.layers.11.mlp.experts.228", - "model.language_model.layers.11.mlp.experts.229", - "model.language_model.layers.11.mlp.experts.230", - "model.language_model.layers.11.mlp.experts.231", - "model.language_model.layers.11.mlp.experts.232", - "model.language_model.layers.11.mlp.experts.233", - "model.language_model.layers.11.mlp.experts.234", - "model.language_model.layers.11.mlp.experts.235", - "model.language_model.layers.11.mlp.experts.236", - "model.language_model.layers.11.mlp.experts.237", - "model.language_model.layers.11.mlp.experts.238", - "model.language_model.layers.11.mlp.experts.239", - "model.language_model.layers.11.mlp.experts.240", - "model.language_model.layers.11.mlp.experts.241", - "model.language_model.layers.11.mlp.experts.242", - "model.language_model.layers.11.mlp.experts.243", - "model.language_model.layers.11.mlp.experts.244", - "model.language_model.layers.11.mlp.experts.245", - "model.language_model.layers.11.mlp.experts.246", - "model.language_model.layers.11.mlp.experts.247", - "model.language_model.layers.11.mlp.experts.248", - "model.language_model.layers.11.mlp.experts.249", - "model.language_model.layers.11.mlp.experts.250", - "model.language_model.layers.11.mlp.experts.251", - "model.language_model.layers.11.mlp.experts.252", - "model.language_model.layers.11.mlp.experts.253", - "model.language_model.layers.11.mlp.experts.254", - "model.language_model.layers.11.mlp.experts.255", "model.language_model.layers.11.mlp.shared_expert_gate", - "model.language_model.layers.12.linear_attn", "model.language_model.layers.12.linear_attn.norm", "model.language_model.layers.12.linear_attn.in_proj_b", "model.language_model.layers.12.linear_attn.in_proj_a", "model.language_model.layers.12.mlp.gate", - "model.language_model.layers.12.mlp.experts.0", - "model.language_model.layers.12.mlp.experts.1", - "model.language_model.layers.12.mlp.experts.2", - "model.language_model.layers.12.mlp.experts.3", - "model.language_model.layers.12.mlp.experts.4", - "model.language_model.layers.12.mlp.experts.5", - "model.language_model.layers.12.mlp.experts.6", - "model.language_model.layers.12.mlp.experts.7", - "model.language_model.layers.12.mlp.experts.8", - "model.language_model.layers.12.mlp.experts.9", - "model.language_model.layers.12.mlp.experts.10", - "model.language_model.layers.12.mlp.experts.11", - "model.language_model.layers.12.mlp.experts.12", - "model.language_model.layers.12.mlp.experts.13", - "model.language_model.layers.12.mlp.experts.14", - "model.language_model.layers.12.mlp.experts.15", - "model.language_model.layers.12.mlp.experts.16", - "model.language_model.layers.12.mlp.experts.17", - "model.language_model.layers.12.mlp.experts.18", - "model.language_model.layers.12.mlp.experts.19", - "model.language_model.layers.12.mlp.experts.20", - "model.language_model.layers.12.mlp.experts.21", - "model.language_model.layers.12.mlp.experts.22", - "model.language_model.layers.12.mlp.experts.23", - "model.language_model.layers.12.mlp.experts.24", - "model.language_model.layers.12.mlp.experts.25", - "model.language_model.layers.12.mlp.experts.26", - "model.language_model.layers.12.mlp.experts.27", - "model.language_model.layers.12.mlp.experts.28", - "model.language_model.layers.12.mlp.experts.29", - "model.language_model.layers.12.mlp.experts.30", - "model.language_model.layers.12.mlp.experts.31", - "model.language_model.layers.12.mlp.experts.32", - "model.language_model.layers.12.mlp.experts.33", - "model.language_model.layers.12.mlp.experts.34", - "model.language_model.layers.12.mlp.experts.35", - "model.language_model.layers.12.mlp.experts.36", - "model.language_model.layers.12.mlp.experts.37", - "model.language_model.layers.12.mlp.experts.38", - "model.language_model.layers.12.mlp.experts.39", - "model.language_model.layers.12.mlp.experts.40", - "model.language_model.layers.12.mlp.experts.41", - "model.language_model.layers.12.mlp.experts.42", - "model.language_model.layers.12.mlp.experts.43", - "model.language_model.layers.12.mlp.experts.44", - "model.language_model.layers.12.mlp.experts.45", - "model.language_model.layers.12.mlp.experts.46", - "model.language_model.layers.12.mlp.experts.47", - "model.language_model.layers.12.mlp.experts.48", - "model.language_model.layers.12.mlp.experts.49", - "model.language_model.layers.12.mlp.experts.50", - "model.language_model.layers.12.mlp.experts.51", - "model.language_model.layers.12.mlp.experts.52", - "model.language_model.layers.12.mlp.experts.53", - "model.language_model.layers.12.mlp.experts.54", - "model.language_model.layers.12.mlp.experts.55", - "model.language_model.layers.12.mlp.experts.56", - "model.language_model.layers.12.mlp.experts.57", - "model.language_model.layers.12.mlp.experts.58", - "model.language_model.layers.12.mlp.experts.59", - "model.language_model.layers.12.mlp.experts.60", - "model.language_model.layers.12.mlp.experts.61", - "model.language_model.layers.12.mlp.experts.62", - "model.language_model.layers.12.mlp.experts.63", - "model.language_model.layers.12.mlp.experts.64", - "model.language_model.layers.12.mlp.experts.65", - "model.language_model.layers.12.mlp.experts.66", - "model.language_model.layers.12.mlp.experts.67", - "model.language_model.layers.12.mlp.experts.68", - "model.language_model.layers.12.mlp.experts.69", - "model.language_model.layers.12.mlp.experts.70", - "model.language_model.layers.12.mlp.experts.71", - "model.language_model.layers.12.mlp.experts.72", - "model.language_model.layers.12.mlp.experts.73", - "model.language_model.layers.12.mlp.experts.74", - "model.language_model.layers.12.mlp.experts.75", - "model.language_model.layers.12.mlp.experts.76", - "model.language_model.layers.12.mlp.experts.77", - "model.language_model.layers.12.mlp.experts.78", - "model.language_model.layers.12.mlp.experts.79", - "model.language_model.layers.12.mlp.experts.80", - "model.language_model.layers.12.mlp.experts.81", - "model.language_model.layers.12.mlp.experts.82", - "model.language_model.layers.12.mlp.experts.83", - "model.language_model.layers.12.mlp.experts.84", - "model.language_model.layers.12.mlp.experts.85", - "model.language_model.layers.12.mlp.experts.86", - "model.language_model.layers.12.mlp.experts.87", - "model.language_model.layers.12.mlp.experts.88", - "model.language_model.layers.12.mlp.experts.89", - "model.language_model.layers.12.mlp.experts.90", - "model.language_model.layers.12.mlp.experts.91", - "model.language_model.layers.12.mlp.experts.92", - "model.language_model.layers.12.mlp.experts.93", - "model.language_model.layers.12.mlp.experts.94", - "model.language_model.layers.12.mlp.experts.95", - "model.language_model.layers.12.mlp.experts.96", - "model.language_model.layers.12.mlp.experts.97", - "model.language_model.layers.12.mlp.experts.98", - "model.language_model.layers.12.mlp.experts.99", - "model.language_model.layers.12.mlp.experts.100", - "model.language_model.layers.12.mlp.experts.101", - "model.language_model.layers.12.mlp.experts.102", - "model.language_model.layers.12.mlp.experts.103", - "model.language_model.layers.12.mlp.experts.104", - "model.language_model.layers.12.mlp.experts.105", - "model.language_model.layers.12.mlp.experts.106", - "model.language_model.layers.12.mlp.experts.107", - "model.language_model.layers.12.mlp.experts.108", - "model.language_model.layers.12.mlp.experts.109", - "model.language_model.layers.12.mlp.experts.110", - "model.language_model.layers.12.mlp.experts.111", - "model.language_model.layers.12.mlp.experts.112", - "model.language_model.layers.12.mlp.experts.113", - "model.language_model.layers.12.mlp.experts.114", - "model.language_model.layers.12.mlp.experts.115", - "model.language_model.layers.12.mlp.experts.116", - "model.language_model.layers.12.mlp.experts.117", - "model.language_model.layers.12.mlp.experts.118", - "model.language_model.layers.12.mlp.experts.119", - "model.language_model.layers.12.mlp.experts.120", - "model.language_model.layers.12.mlp.experts.121", - "model.language_model.layers.12.mlp.experts.122", - "model.language_model.layers.12.mlp.experts.123", - "model.language_model.layers.12.mlp.experts.124", - "model.language_model.layers.12.mlp.experts.125", - "model.language_model.layers.12.mlp.experts.126", - "model.language_model.layers.12.mlp.experts.127", - "model.language_model.layers.12.mlp.experts.128", - "model.language_model.layers.12.mlp.experts.129", - "model.language_model.layers.12.mlp.experts.130", - "model.language_model.layers.12.mlp.experts.131", - "model.language_model.layers.12.mlp.experts.132", - "model.language_model.layers.12.mlp.experts.133", - "model.language_model.layers.12.mlp.experts.134", - "model.language_model.layers.12.mlp.experts.135", - "model.language_model.layers.12.mlp.experts.136", - "model.language_model.layers.12.mlp.experts.137", - "model.language_model.layers.12.mlp.experts.138", - "model.language_model.layers.12.mlp.experts.139", - "model.language_model.layers.12.mlp.experts.140", - "model.language_model.layers.12.mlp.experts.141", - "model.language_model.layers.12.mlp.experts.142", - "model.language_model.layers.12.mlp.experts.143", - "model.language_model.layers.12.mlp.experts.144", - "model.language_model.layers.12.mlp.experts.145", - "model.language_model.layers.12.mlp.experts.146", - "model.language_model.layers.12.mlp.experts.147", - "model.language_model.layers.12.mlp.experts.148", - "model.language_model.layers.12.mlp.experts.149", - "model.language_model.layers.12.mlp.experts.150", - "model.language_model.layers.12.mlp.experts.151", - "model.language_model.layers.12.mlp.experts.152", - "model.language_model.layers.12.mlp.experts.153", - "model.language_model.layers.12.mlp.experts.154", - "model.language_model.layers.12.mlp.experts.155", - "model.language_model.layers.12.mlp.experts.156", - "model.language_model.layers.12.mlp.experts.157", - "model.language_model.layers.12.mlp.experts.158", - "model.language_model.layers.12.mlp.experts.159", - "model.language_model.layers.12.mlp.experts.160", - "model.language_model.layers.12.mlp.experts.161", - "model.language_model.layers.12.mlp.experts.162", - "model.language_model.layers.12.mlp.experts.163", - "model.language_model.layers.12.mlp.experts.164", - "model.language_model.layers.12.mlp.experts.165", - "model.language_model.layers.12.mlp.experts.166", - "model.language_model.layers.12.mlp.experts.167", - "model.language_model.layers.12.mlp.experts.168", - "model.language_model.layers.12.mlp.experts.169", - "model.language_model.layers.12.mlp.experts.170", - "model.language_model.layers.12.mlp.experts.171", - "model.language_model.layers.12.mlp.experts.172", - "model.language_model.layers.12.mlp.experts.173", - "model.language_model.layers.12.mlp.experts.174", - "model.language_model.layers.12.mlp.experts.175", - "model.language_model.layers.12.mlp.experts.176", - "model.language_model.layers.12.mlp.experts.177", - "model.language_model.layers.12.mlp.experts.178", - "model.language_model.layers.12.mlp.experts.179", - "model.language_model.layers.12.mlp.experts.180", - "model.language_model.layers.12.mlp.experts.181", - "model.language_model.layers.12.mlp.experts.182", - "model.language_model.layers.12.mlp.experts.183", - "model.language_model.layers.12.mlp.experts.184", - "model.language_model.layers.12.mlp.experts.185", - "model.language_model.layers.12.mlp.experts.186", - "model.language_model.layers.12.mlp.experts.187", - "model.language_model.layers.12.mlp.experts.188", - "model.language_model.layers.12.mlp.experts.189", - "model.language_model.layers.12.mlp.experts.190", - "model.language_model.layers.12.mlp.experts.191", - "model.language_model.layers.12.mlp.experts.192", - "model.language_model.layers.12.mlp.experts.193", - "model.language_model.layers.12.mlp.experts.194", - "model.language_model.layers.12.mlp.experts.195", - "model.language_model.layers.12.mlp.experts.196", - "model.language_model.layers.12.mlp.experts.197", - "model.language_model.layers.12.mlp.experts.198", - "model.language_model.layers.12.mlp.experts.199", - "model.language_model.layers.12.mlp.experts.200", - "model.language_model.layers.12.mlp.experts.201", - "model.language_model.layers.12.mlp.experts.202", - "model.language_model.layers.12.mlp.experts.203", - "model.language_model.layers.12.mlp.experts.204", - "model.language_model.layers.12.mlp.experts.205", - "model.language_model.layers.12.mlp.experts.206", - "model.language_model.layers.12.mlp.experts.207", - "model.language_model.layers.12.mlp.experts.208", - "model.language_model.layers.12.mlp.experts.209", - "model.language_model.layers.12.mlp.experts.210", - "model.language_model.layers.12.mlp.experts.211", - "model.language_model.layers.12.mlp.experts.212", - "model.language_model.layers.12.mlp.experts.213", - "model.language_model.layers.12.mlp.experts.214", - "model.language_model.layers.12.mlp.experts.215", - "model.language_model.layers.12.mlp.experts.216", - "model.language_model.layers.12.mlp.experts.217", - "model.language_model.layers.12.mlp.experts.218", - "model.language_model.layers.12.mlp.experts.219", - "model.language_model.layers.12.mlp.experts.220", - "model.language_model.layers.12.mlp.experts.221", - "model.language_model.layers.12.mlp.experts.222", - "model.language_model.layers.12.mlp.experts.223", - "model.language_model.layers.12.mlp.experts.224", - "model.language_model.layers.12.mlp.experts.225", - "model.language_model.layers.12.mlp.experts.226", - "model.language_model.layers.12.mlp.experts.227", - "model.language_model.layers.12.mlp.experts.228", - "model.language_model.layers.12.mlp.experts.229", - "model.language_model.layers.12.mlp.experts.230", - "model.language_model.layers.12.mlp.experts.231", - "model.language_model.layers.12.mlp.experts.232", - "model.language_model.layers.12.mlp.experts.233", - "model.language_model.layers.12.mlp.experts.234", - "model.language_model.layers.12.mlp.experts.235", - "model.language_model.layers.12.mlp.experts.236", - "model.language_model.layers.12.mlp.experts.237", - "model.language_model.layers.12.mlp.experts.238", - "model.language_model.layers.12.mlp.experts.239", - "model.language_model.layers.12.mlp.experts.240", - "model.language_model.layers.12.mlp.experts.241", - "model.language_model.layers.12.mlp.experts.242", - "model.language_model.layers.12.mlp.experts.243", - "model.language_model.layers.12.mlp.experts.244", - "model.language_model.layers.12.mlp.experts.245", - "model.language_model.layers.12.mlp.experts.246", - "model.language_model.layers.12.mlp.experts.247", - "model.language_model.layers.12.mlp.experts.248", - "model.language_model.layers.12.mlp.experts.249", - "model.language_model.layers.12.mlp.experts.250", - "model.language_model.layers.12.mlp.experts.251", - "model.language_model.layers.12.mlp.experts.252", - "model.language_model.layers.12.mlp.experts.253", - "model.language_model.layers.12.mlp.experts.254", - "model.language_model.layers.12.mlp.experts.255", "model.language_model.layers.12.mlp.shared_expert_gate", - "model.language_model.layers.13.linear_attn", "model.language_model.layers.13.linear_attn.norm", "model.language_model.layers.13.linear_attn.in_proj_b", "model.language_model.layers.13.linear_attn.in_proj_a", "model.language_model.layers.13.mlp.gate", - "model.language_model.layers.13.mlp.experts.0", - "model.language_model.layers.13.mlp.experts.1", - "model.language_model.layers.13.mlp.experts.2", - "model.language_model.layers.13.mlp.experts.3", - "model.language_model.layers.13.mlp.experts.4", - "model.language_model.layers.13.mlp.experts.5", - "model.language_model.layers.13.mlp.experts.6", - "model.language_model.layers.13.mlp.experts.7", - "model.language_model.layers.13.mlp.experts.8", - "model.language_model.layers.13.mlp.experts.9", - "model.language_model.layers.13.mlp.experts.10", - "model.language_model.layers.13.mlp.experts.11", - "model.language_model.layers.13.mlp.experts.12", - "model.language_model.layers.13.mlp.experts.13", - "model.language_model.layers.13.mlp.experts.14", - "model.language_model.layers.13.mlp.experts.15", - "model.language_model.layers.13.mlp.experts.16", - "model.language_model.layers.13.mlp.experts.17", - "model.language_model.layers.13.mlp.experts.18", - "model.language_model.layers.13.mlp.experts.19", - "model.language_model.layers.13.mlp.experts.20", - "model.language_model.layers.13.mlp.experts.21", - "model.language_model.layers.13.mlp.experts.22", - "model.language_model.layers.13.mlp.experts.23", - "model.language_model.layers.13.mlp.experts.24", - "model.language_model.layers.13.mlp.experts.25", - "model.language_model.layers.13.mlp.experts.26", - "model.language_model.layers.13.mlp.experts.27", - "model.language_model.layers.13.mlp.experts.28", - "model.language_model.layers.13.mlp.experts.29", - "model.language_model.layers.13.mlp.experts.30", - "model.language_model.layers.13.mlp.experts.31", - "model.language_model.layers.13.mlp.experts.32", - "model.language_model.layers.13.mlp.experts.33", - "model.language_model.layers.13.mlp.experts.34", - "model.language_model.layers.13.mlp.experts.35", - "model.language_model.layers.13.mlp.experts.36", - "model.language_model.layers.13.mlp.experts.37", - "model.language_model.layers.13.mlp.experts.38", - "model.language_model.layers.13.mlp.experts.39", - "model.language_model.layers.13.mlp.experts.40", - "model.language_model.layers.13.mlp.experts.41", - "model.language_model.layers.13.mlp.experts.42", - "model.language_model.layers.13.mlp.experts.43", - "model.language_model.layers.13.mlp.experts.44", - "model.language_model.layers.13.mlp.experts.45", - "model.language_model.layers.13.mlp.experts.46", - "model.language_model.layers.13.mlp.experts.47", - "model.language_model.layers.13.mlp.experts.48", - "model.language_model.layers.13.mlp.experts.49", - "model.language_model.layers.13.mlp.experts.50", - "model.language_model.layers.13.mlp.experts.51", - "model.language_model.layers.13.mlp.experts.52", - "model.language_model.layers.13.mlp.experts.53", - "model.language_model.layers.13.mlp.experts.54", - "model.language_model.layers.13.mlp.experts.55", - "model.language_model.layers.13.mlp.experts.56", - "model.language_model.layers.13.mlp.experts.57", - "model.language_model.layers.13.mlp.experts.58", - "model.language_model.layers.13.mlp.experts.59", - "model.language_model.layers.13.mlp.experts.60", - "model.language_model.layers.13.mlp.experts.61", - "model.language_model.layers.13.mlp.experts.62", - "model.language_model.layers.13.mlp.experts.63", - "model.language_model.layers.13.mlp.experts.64", - "model.language_model.layers.13.mlp.experts.65", - "model.language_model.layers.13.mlp.experts.66", - "model.language_model.layers.13.mlp.experts.67", - "model.language_model.layers.13.mlp.experts.68", - "model.language_model.layers.13.mlp.experts.69", - "model.language_model.layers.13.mlp.experts.70", - "model.language_model.layers.13.mlp.experts.71", - "model.language_model.layers.13.mlp.experts.72", - "model.language_model.layers.13.mlp.experts.73", - "model.language_model.layers.13.mlp.experts.74", - "model.language_model.layers.13.mlp.experts.75", - "model.language_model.layers.13.mlp.experts.76", - "model.language_model.layers.13.mlp.experts.77", - "model.language_model.layers.13.mlp.experts.78", - "model.language_model.layers.13.mlp.experts.79", - "model.language_model.layers.13.mlp.experts.80", - "model.language_model.layers.13.mlp.experts.81", - "model.language_model.layers.13.mlp.experts.82", - "model.language_model.layers.13.mlp.experts.83", - "model.language_model.layers.13.mlp.experts.84", - "model.language_model.layers.13.mlp.experts.85", - "model.language_model.layers.13.mlp.experts.86", - "model.language_model.layers.13.mlp.experts.87", - "model.language_model.layers.13.mlp.experts.88", - "model.language_model.layers.13.mlp.experts.89", - "model.language_model.layers.13.mlp.experts.90", - "model.language_model.layers.13.mlp.experts.91", - "model.language_model.layers.13.mlp.experts.92", - "model.language_model.layers.13.mlp.experts.93", - "model.language_model.layers.13.mlp.experts.94", - "model.language_model.layers.13.mlp.experts.95", - "model.language_model.layers.13.mlp.experts.96", - "model.language_model.layers.13.mlp.experts.97", - "model.language_model.layers.13.mlp.experts.98", - "model.language_model.layers.13.mlp.experts.99", - "model.language_model.layers.13.mlp.experts.100", - "model.language_model.layers.13.mlp.experts.101", - "model.language_model.layers.13.mlp.experts.102", - "model.language_model.layers.13.mlp.experts.103", - "model.language_model.layers.13.mlp.experts.104", - "model.language_model.layers.13.mlp.experts.105", - "model.language_model.layers.13.mlp.experts.106", - "model.language_model.layers.13.mlp.experts.107", - "model.language_model.layers.13.mlp.experts.108", - "model.language_model.layers.13.mlp.experts.109", - "model.language_model.layers.13.mlp.experts.110", - "model.language_model.layers.13.mlp.experts.111", - "model.language_model.layers.13.mlp.experts.112", - "model.language_model.layers.13.mlp.experts.113", - "model.language_model.layers.13.mlp.experts.114", - "model.language_model.layers.13.mlp.experts.115", - "model.language_model.layers.13.mlp.experts.116", - "model.language_model.layers.13.mlp.experts.117", - "model.language_model.layers.13.mlp.experts.118", - "model.language_model.layers.13.mlp.experts.119", - "model.language_model.layers.13.mlp.experts.120", - "model.language_model.layers.13.mlp.experts.121", - "model.language_model.layers.13.mlp.experts.122", - "model.language_model.layers.13.mlp.experts.123", - "model.language_model.layers.13.mlp.experts.124", - "model.language_model.layers.13.mlp.experts.125", - "model.language_model.layers.13.mlp.experts.126", - "model.language_model.layers.13.mlp.experts.127", - "model.language_model.layers.13.mlp.experts.128", - "model.language_model.layers.13.mlp.experts.129", - "model.language_model.layers.13.mlp.experts.130", - "model.language_model.layers.13.mlp.experts.131", - "model.language_model.layers.13.mlp.experts.132", - "model.language_model.layers.13.mlp.experts.133", - "model.language_model.layers.13.mlp.experts.134", - "model.language_model.layers.13.mlp.experts.135", - "model.language_model.layers.13.mlp.experts.136", - "model.language_model.layers.13.mlp.experts.137", - "model.language_model.layers.13.mlp.experts.138", - "model.language_model.layers.13.mlp.experts.139", - "model.language_model.layers.13.mlp.experts.140", - "model.language_model.layers.13.mlp.experts.141", - "model.language_model.layers.13.mlp.experts.142", - "model.language_model.layers.13.mlp.experts.143", - "model.language_model.layers.13.mlp.experts.144", - "model.language_model.layers.13.mlp.experts.145", - "model.language_model.layers.13.mlp.experts.146", - "model.language_model.layers.13.mlp.experts.147", - "model.language_model.layers.13.mlp.experts.148", - "model.language_model.layers.13.mlp.experts.149", - "model.language_model.layers.13.mlp.experts.150", - "model.language_model.layers.13.mlp.experts.151", - "model.language_model.layers.13.mlp.experts.152", - "model.language_model.layers.13.mlp.experts.153", - "model.language_model.layers.13.mlp.experts.154", - "model.language_model.layers.13.mlp.experts.155", - "model.language_model.layers.13.mlp.experts.156", - "model.language_model.layers.13.mlp.experts.157", - "model.language_model.layers.13.mlp.experts.158", - "model.language_model.layers.13.mlp.experts.159", - "model.language_model.layers.13.mlp.experts.160", - "model.language_model.layers.13.mlp.experts.161", - "model.language_model.layers.13.mlp.experts.162", - "model.language_model.layers.13.mlp.experts.163", - "model.language_model.layers.13.mlp.experts.164", - "model.language_model.layers.13.mlp.experts.165", - "model.language_model.layers.13.mlp.experts.166", - "model.language_model.layers.13.mlp.experts.167", - "model.language_model.layers.13.mlp.experts.168", - "model.language_model.layers.13.mlp.experts.169", - "model.language_model.layers.13.mlp.experts.170", - "model.language_model.layers.13.mlp.experts.171", - "model.language_model.layers.13.mlp.experts.172", - "model.language_model.layers.13.mlp.experts.173", - "model.language_model.layers.13.mlp.experts.174", - "model.language_model.layers.13.mlp.experts.175", - "model.language_model.layers.13.mlp.experts.176", - "model.language_model.layers.13.mlp.experts.177", - "model.language_model.layers.13.mlp.experts.178", - "model.language_model.layers.13.mlp.experts.179", - "model.language_model.layers.13.mlp.experts.180", - "model.language_model.layers.13.mlp.experts.181", - "model.language_model.layers.13.mlp.experts.182", - "model.language_model.layers.13.mlp.experts.183", - "model.language_model.layers.13.mlp.experts.184", - "model.language_model.layers.13.mlp.experts.185", - "model.language_model.layers.13.mlp.experts.186", - "model.language_model.layers.13.mlp.experts.187", - "model.language_model.layers.13.mlp.experts.188", - "model.language_model.layers.13.mlp.experts.189", - "model.language_model.layers.13.mlp.experts.190", - "model.language_model.layers.13.mlp.experts.191", - "model.language_model.layers.13.mlp.experts.192", - "model.language_model.layers.13.mlp.experts.193", - "model.language_model.layers.13.mlp.experts.194", - "model.language_model.layers.13.mlp.experts.195", - "model.language_model.layers.13.mlp.experts.196", - "model.language_model.layers.13.mlp.experts.197", - "model.language_model.layers.13.mlp.experts.198", - "model.language_model.layers.13.mlp.experts.199", - "model.language_model.layers.13.mlp.experts.200", - "model.language_model.layers.13.mlp.experts.201", - "model.language_model.layers.13.mlp.experts.202", - "model.language_model.layers.13.mlp.experts.203", - "model.language_model.layers.13.mlp.experts.204", - "model.language_model.layers.13.mlp.experts.205", - "model.language_model.layers.13.mlp.experts.206", - "model.language_model.layers.13.mlp.experts.207", - "model.language_model.layers.13.mlp.experts.208", - "model.language_model.layers.13.mlp.experts.209", - "model.language_model.layers.13.mlp.experts.210", - "model.language_model.layers.13.mlp.experts.211", - "model.language_model.layers.13.mlp.experts.212", - "model.language_model.layers.13.mlp.experts.213", - "model.language_model.layers.13.mlp.experts.214", - "model.language_model.layers.13.mlp.experts.215", - "model.language_model.layers.13.mlp.experts.216", - "model.language_model.layers.13.mlp.experts.217", - "model.language_model.layers.13.mlp.experts.218", - "model.language_model.layers.13.mlp.experts.219", - "model.language_model.layers.13.mlp.experts.220", - "model.language_model.layers.13.mlp.experts.221", - "model.language_model.layers.13.mlp.experts.222", - "model.language_model.layers.13.mlp.experts.223", - "model.language_model.layers.13.mlp.experts.224", - "model.language_model.layers.13.mlp.experts.225", - "model.language_model.layers.13.mlp.experts.226", - "model.language_model.layers.13.mlp.experts.227", - "model.language_model.layers.13.mlp.experts.228", - "model.language_model.layers.13.mlp.experts.229", - "model.language_model.layers.13.mlp.experts.230", - "model.language_model.layers.13.mlp.experts.231", - "model.language_model.layers.13.mlp.experts.232", - "model.language_model.layers.13.mlp.experts.233", - "model.language_model.layers.13.mlp.experts.234", - "model.language_model.layers.13.mlp.experts.235", - "model.language_model.layers.13.mlp.experts.236", - "model.language_model.layers.13.mlp.experts.237", - "model.language_model.layers.13.mlp.experts.238", - "model.language_model.layers.13.mlp.experts.239", - "model.language_model.layers.13.mlp.experts.240", - "model.language_model.layers.13.mlp.experts.241", - "model.language_model.layers.13.mlp.experts.242", - "model.language_model.layers.13.mlp.experts.243", - "model.language_model.layers.13.mlp.experts.244", - "model.language_model.layers.13.mlp.experts.245", - "model.language_model.layers.13.mlp.experts.246", - "model.language_model.layers.13.mlp.experts.247", - "model.language_model.layers.13.mlp.experts.248", - "model.language_model.layers.13.mlp.experts.249", - "model.language_model.layers.13.mlp.experts.250", - "model.language_model.layers.13.mlp.experts.251", - "model.language_model.layers.13.mlp.experts.252", - "model.language_model.layers.13.mlp.experts.253", - "model.language_model.layers.13.mlp.experts.254", - "model.language_model.layers.13.mlp.experts.255", "model.language_model.layers.13.mlp.shared_expert_gate", - "model.language_model.layers.14.linear_attn", "model.language_model.layers.14.linear_attn.norm", "model.language_model.layers.14.linear_attn.in_proj_b", "model.language_model.layers.14.linear_attn.in_proj_a", "model.language_model.layers.14.mlp.gate", - "model.language_model.layers.14.mlp.experts.0", - "model.language_model.layers.14.mlp.experts.1", - "model.language_model.layers.14.mlp.experts.2", - "model.language_model.layers.14.mlp.experts.3", - "model.language_model.layers.14.mlp.experts.4", - "model.language_model.layers.14.mlp.experts.5", - "model.language_model.layers.14.mlp.experts.6", - "model.language_model.layers.14.mlp.experts.7", - "model.language_model.layers.14.mlp.experts.8", - "model.language_model.layers.14.mlp.experts.9", - "model.language_model.layers.14.mlp.experts.10", - "model.language_model.layers.14.mlp.experts.11", - "model.language_model.layers.14.mlp.experts.12", - "model.language_model.layers.14.mlp.experts.13", - "model.language_model.layers.14.mlp.experts.14", - "model.language_model.layers.14.mlp.experts.15", - "model.language_model.layers.14.mlp.experts.16", - "model.language_model.layers.14.mlp.experts.17", - "model.language_model.layers.14.mlp.experts.18", - "model.language_model.layers.14.mlp.experts.19", - "model.language_model.layers.14.mlp.experts.20", - "model.language_model.layers.14.mlp.experts.21", - "model.language_model.layers.14.mlp.experts.22", - "model.language_model.layers.14.mlp.experts.23", - "model.language_model.layers.14.mlp.experts.24", - "model.language_model.layers.14.mlp.experts.25", - "model.language_model.layers.14.mlp.experts.26", - "model.language_model.layers.14.mlp.experts.27", - "model.language_model.layers.14.mlp.experts.28", - "model.language_model.layers.14.mlp.experts.29", - "model.language_model.layers.14.mlp.experts.30", - "model.language_model.layers.14.mlp.experts.31", - "model.language_model.layers.14.mlp.experts.32", - "model.language_model.layers.14.mlp.experts.33", - "model.language_model.layers.14.mlp.experts.34", - "model.language_model.layers.14.mlp.experts.35", - "model.language_model.layers.14.mlp.experts.36", - "model.language_model.layers.14.mlp.experts.37", - "model.language_model.layers.14.mlp.experts.38", - "model.language_model.layers.14.mlp.experts.39", - "model.language_model.layers.14.mlp.experts.40", - "model.language_model.layers.14.mlp.experts.41", - "model.language_model.layers.14.mlp.experts.42", - "model.language_model.layers.14.mlp.experts.43", - "model.language_model.layers.14.mlp.experts.44", - "model.language_model.layers.14.mlp.experts.45", - "model.language_model.layers.14.mlp.experts.46", - "model.language_model.layers.14.mlp.experts.47", - "model.language_model.layers.14.mlp.experts.48", - "model.language_model.layers.14.mlp.experts.49", - "model.language_model.layers.14.mlp.experts.50", - "model.language_model.layers.14.mlp.experts.51", - "model.language_model.layers.14.mlp.experts.52", - "model.language_model.layers.14.mlp.experts.53", - "model.language_model.layers.14.mlp.experts.54", - "model.language_model.layers.14.mlp.experts.55", - "model.language_model.layers.14.mlp.experts.56", - "model.language_model.layers.14.mlp.experts.57", - "model.language_model.layers.14.mlp.experts.58", - "model.language_model.layers.14.mlp.experts.59", - "model.language_model.layers.14.mlp.experts.60", - "model.language_model.layers.14.mlp.experts.61", - "model.language_model.layers.14.mlp.experts.62", - "model.language_model.layers.14.mlp.experts.63", - "model.language_model.layers.14.mlp.experts.64", - "model.language_model.layers.14.mlp.experts.65", - "model.language_model.layers.14.mlp.experts.66", - "model.language_model.layers.14.mlp.experts.67", - "model.language_model.layers.14.mlp.experts.68", - "model.language_model.layers.14.mlp.experts.69", - "model.language_model.layers.14.mlp.experts.70", - "model.language_model.layers.14.mlp.experts.71", - "model.language_model.layers.14.mlp.experts.72", - "model.language_model.layers.14.mlp.experts.73", - "model.language_model.layers.14.mlp.experts.74", - "model.language_model.layers.14.mlp.experts.75", - "model.language_model.layers.14.mlp.experts.76", - "model.language_model.layers.14.mlp.experts.77", - "model.language_model.layers.14.mlp.experts.78", - "model.language_model.layers.14.mlp.experts.79", - "model.language_model.layers.14.mlp.experts.80", - "model.language_model.layers.14.mlp.experts.81", - "model.language_model.layers.14.mlp.experts.82", - "model.language_model.layers.14.mlp.experts.83", - "model.language_model.layers.14.mlp.experts.84", - "model.language_model.layers.14.mlp.experts.85", - "model.language_model.layers.14.mlp.experts.86", - "model.language_model.layers.14.mlp.experts.87", - "model.language_model.layers.14.mlp.experts.88", - "model.language_model.layers.14.mlp.experts.89", - "model.language_model.layers.14.mlp.experts.90", - "model.language_model.layers.14.mlp.experts.91", - "model.language_model.layers.14.mlp.experts.92", - "model.language_model.layers.14.mlp.experts.93", - "model.language_model.layers.14.mlp.experts.94", - "model.language_model.layers.14.mlp.experts.95", - "model.language_model.layers.14.mlp.experts.96", - "model.language_model.layers.14.mlp.experts.97", - "model.language_model.layers.14.mlp.experts.98", - "model.language_model.layers.14.mlp.experts.99", - "model.language_model.layers.14.mlp.experts.100", - "model.language_model.layers.14.mlp.experts.101", - "model.language_model.layers.14.mlp.experts.102", - "model.language_model.layers.14.mlp.experts.103", - "model.language_model.layers.14.mlp.experts.104", - "model.language_model.layers.14.mlp.experts.105", - "model.language_model.layers.14.mlp.experts.106", - "model.language_model.layers.14.mlp.experts.107", - "model.language_model.layers.14.mlp.experts.108", - "model.language_model.layers.14.mlp.experts.109", - "model.language_model.layers.14.mlp.experts.110", - "model.language_model.layers.14.mlp.experts.111", - "model.language_model.layers.14.mlp.experts.112", - "model.language_model.layers.14.mlp.experts.113", - "model.language_model.layers.14.mlp.experts.114", - "model.language_model.layers.14.mlp.experts.115", - "model.language_model.layers.14.mlp.experts.116", - "model.language_model.layers.14.mlp.experts.117", - "model.language_model.layers.14.mlp.experts.118", - "model.language_model.layers.14.mlp.experts.119", - "model.language_model.layers.14.mlp.experts.120", - "model.language_model.layers.14.mlp.experts.121", - "model.language_model.layers.14.mlp.experts.122", - "model.language_model.layers.14.mlp.experts.123", - "model.language_model.layers.14.mlp.experts.124", - "model.language_model.layers.14.mlp.experts.125", - "model.language_model.layers.14.mlp.experts.126", - "model.language_model.layers.14.mlp.experts.127", - "model.language_model.layers.14.mlp.experts.128", - "model.language_model.layers.14.mlp.experts.129", - "model.language_model.layers.14.mlp.experts.130", - "model.language_model.layers.14.mlp.experts.131", - "model.language_model.layers.14.mlp.experts.132", - "model.language_model.layers.14.mlp.experts.133", - "model.language_model.layers.14.mlp.experts.134", - "model.language_model.layers.14.mlp.experts.135", - "model.language_model.layers.14.mlp.experts.136", - "model.language_model.layers.14.mlp.experts.137", - "model.language_model.layers.14.mlp.experts.138", - "model.language_model.layers.14.mlp.experts.139", - "model.language_model.layers.14.mlp.experts.140", - "model.language_model.layers.14.mlp.experts.141", - "model.language_model.layers.14.mlp.experts.142", - "model.language_model.layers.14.mlp.experts.143", - "model.language_model.layers.14.mlp.experts.144", - "model.language_model.layers.14.mlp.experts.145", - "model.language_model.layers.14.mlp.experts.146", - "model.language_model.layers.14.mlp.experts.147", - "model.language_model.layers.14.mlp.experts.148", - "model.language_model.layers.14.mlp.experts.149", - "model.language_model.layers.14.mlp.experts.150", - "model.language_model.layers.14.mlp.experts.151", - "model.language_model.layers.14.mlp.experts.152", - "model.language_model.layers.14.mlp.experts.153", - "model.language_model.layers.14.mlp.experts.154", - "model.language_model.layers.14.mlp.experts.155", - "model.language_model.layers.14.mlp.experts.156", - "model.language_model.layers.14.mlp.experts.157", - "model.language_model.layers.14.mlp.experts.158", - "model.language_model.layers.14.mlp.experts.159", - "model.language_model.layers.14.mlp.experts.160", - "model.language_model.layers.14.mlp.experts.161", - "model.language_model.layers.14.mlp.experts.162", - "model.language_model.layers.14.mlp.experts.163", - "model.language_model.layers.14.mlp.experts.164", - "model.language_model.layers.14.mlp.experts.165", - "model.language_model.layers.14.mlp.experts.166", - "model.language_model.layers.14.mlp.experts.167", - "model.language_model.layers.14.mlp.experts.168", - "model.language_model.layers.14.mlp.experts.169", - "model.language_model.layers.14.mlp.experts.170", - "model.language_model.layers.14.mlp.experts.171", - "model.language_model.layers.14.mlp.experts.172", - "model.language_model.layers.14.mlp.experts.173", - "model.language_model.layers.14.mlp.experts.174", - "model.language_model.layers.14.mlp.experts.175", - "model.language_model.layers.14.mlp.experts.176", - "model.language_model.layers.14.mlp.experts.177", - "model.language_model.layers.14.mlp.experts.178", - "model.language_model.layers.14.mlp.experts.179", - "model.language_model.layers.14.mlp.experts.180", - "model.language_model.layers.14.mlp.experts.181", - "model.language_model.layers.14.mlp.experts.182", - "model.language_model.layers.14.mlp.experts.183", - "model.language_model.layers.14.mlp.experts.184", - "model.language_model.layers.14.mlp.experts.185", - "model.language_model.layers.14.mlp.experts.186", - "model.language_model.layers.14.mlp.experts.187", - "model.language_model.layers.14.mlp.experts.188", - "model.language_model.layers.14.mlp.experts.189", - "model.language_model.layers.14.mlp.experts.190", - "model.language_model.layers.14.mlp.experts.191", - "model.language_model.layers.14.mlp.experts.192", - "model.language_model.layers.14.mlp.experts.193", - "model.language_model.layers.14.mlp.experts.194", - "model.language_model.layers.14.mlp.experts.195", - "model.language_model.layers.14.mlp.experts.196", - "model.language_model.layers.14.mlp.experts.197", - "model.language_model.layers.14.mlp.experts.198", - "model.language_model.layers.14.mlp.experts.199", - "model.language_model.layers.14.mlp.experts.200", - "model.language_model.layers.14.mlp.experts.201", - "model.language_model.layers.14.mlp.experts.202", - "model.language_model.layers.14.mlp.experts.203", - "model.language_model.layers.14.mlp.experts.204", - "model.language_model.layers.14.mlp.experts.205", - "model.language_model.layers.14.mlp.experts.206", - "model.language_model.layers.14.mlp.experts.207", - "model.language_model.layers.14.mlp.experts.208", - "model.language_model.layers.14.mlp.experts.209", - "model.language_model.layers.14.mlp.experts.210", - "model.language_model.layers.14.mlp.experts.211", - "model.language_model.layers.14.mlp.experts.212", - "model.language_model.layers.14.mlp.experts.213", - "model.language_model.layers.14.mlp.experts.214", - "model.language_model.layers.14.mlp.experts.215", - "model.language_model.layers.14.mlp.experts.216", - "model.language_model.layers.14.mlp.experts.217", - "model.language_model.layers.14.mlp.experts.218", - "model.language_model.layers.14.mlp.experts.219", - "model.language_model.layers.14.mlp.experts.220", - "model.language_model.layers.14.mlp.experts.221", - "model.language_model.layers.14.mlp.experts.222", - "model.language_model.layers.14.mlp.experts.223", - "model.language_model.layers.14.mlp.experts.224", - "model.language_model.layers.14.mlp.experts.225", - "model.language_model.layers.14.mlp.experts.226", - "model.language_model.layers.14.mlp.experts.227", - "model.language_model.layers.14.mlp.experts.228", - "model.language_model.layers.14.mlp.experts.229", - "model.language_model.layers.14.mlp.experts.230", - "model.language_model.layers.14.mlp.experts.231", - "model.language_model.layers.14.mlp.experts.232", - "model.language_model.layers.14.mlp.experts.233", - "model.language_model.layers.14.mlp.experts.234", - "model.language_model.layers.14.mlp.experts.235", - "model.language_model.layers.14.mlp.experts.236", - "model.language_model.layers.14.mlp.experts.237", - "model.language_model.layers.14.mlp.experts.238", - "model.language_model.layers.14.mlp.experts.239", - "model.language_model.layers.14.mlp.experts.240", - "model.language_model.layers.14.mlp.experts.241", - "model.language_model.layers.14.mlp.experts.242", - "model.language_model.layers.14.mlp.experts.243", - "model.language_model.layers.14.mlp.experts.244", - "model.language_model.layers.14.mlp.experts.245", - "model.language_model.layers.14.mlp.experts.246", - "model.language_model.layers.14.mlp.experts.247", - "model.language_model.layers.14.mlp.experts.248", - "model.language_model.layers.14.mlp.experts.249", - "model.language_model.layers.14.mlp.experts.250", - "model.language_model.layers.14.mlp.experts.251", - "model.language_model.layers.14.mlp.experts.252", - "model.language_model.layers.14.mlp.experts.253", - "model.language_model.layers.14.mlp.experts.254", - "model.language_model.layers.14.mlp.experts.255", "model.language_model.layers.14.mlp.shared_expert_gate", "model.language_model.layers.15.mlp.gate", - "model.language_model.layers.15.mlp.experts.0", - "model.language_model.layers.15.mlp.experts.1", - "model.language_model.layers.15.mlp.experts.2", - "model.language_model.layers.15.mlp.experts.3", - "model.language_model.layers.15.mlp.experts.4", - "model.language_model.layers.15.mlp.experts.5", - "model.language_model.layers.15.mlp.experts.6", - "model.language_model.layers.15.mlp.experts.7", - "model.language_model.layers.15.mlp.experts.8", - "model.language_model.layers.15.mlp.experts.9", - "model.language_model.layers.15.mlp.experts.10", - "model.language_model.layers.15.mlp.experts.11", - "model.language_model.layers.15.mlp.experts.12", - "model.language_model.layers.15.mlp.experts.13", - "model.language_model.layers.15.mlp.experts.14", - "model.language_model.layers.15.mlp.experts.15", - "model.language_model.layers.15.mlp.experts.16", - "model.language_model.layers.15.mlp.experts.17", - "model.language_model.layers.15.mlp.experts.18", - "model.language_model.layers.15.mlp.experts.19", - "model.language_model.layers.15.mlp.experts.20", - "model.language_model.layers.15.mlp.experts.21", - "model.language_model.layers.15.mlp.experts.22", - "model.language_model.layers.15.mlp.experts.23", - "model.language_model.layers.15.mlp.experts.24", - "model.language_model.layers.15.mlp.experts.25", - "model.language_model.layers.15.mlp.experts.26", - "model.language_model.layers.15.mlp.experts.27", - "model.language_model.layers.15.mlp.experts.28", - "model.language_model.layers.15.mlp.experts.29", - "model.language_model.layers.15.mlp.experts.30", - "model.language_model.layers.15.mlp.experts.31", - "model.language_model.layers.15.mlp.experts.32", - "model.language_model.layers.15.mlp.experts.33", - "model.language_model.layers.15.mlp.experts.34", - "model.language_model.layers.15.mlp.experts.35", - "model.language_model.layers.15.mlp.experts.36", - "model.language_model.layers.15.mlp.experts.37", - "model.language_model.layers.15.mlp.experts.38", - "model.language_model.layers.15.mlp.experts.39", - "model.language_model.layers.15.mlp.experts.40", - "model.language_model.layers.15.mlp.experts.41", - "model.language_model.layers.15.mlp.experts.42", - "model.language_model.layers.15.mlp.experts.43", - "model.language_model.layers.15.mlp.experts.44", - "model.language_model.layers.15.mlp.experts.45", - "model.language_model.layers.15.mlp.experts.46", - "model.language_model.layers.15.mlp.experts.47", - "model.language_model.layers.15.mlp.experts.48", - "model.language_model.layers.15.mlp.experts.49", - "model.language_model.layers.15.mlp.experts.50", - "model.language_model.layers.15.mlp.experts.51", - "model.language_model.layers.15.mlp.experts.52", - "model.language_model.layers.15.mlp.experts.53", - "model.language_model.layers.15.mlp.experts.54", - "model.language_model.layers.15.mlp.experts.55", - "model.language_model.layers.15.mlp.experts.56", - "model.language_model.layers.15.mlp.experts.57", - "model.language_model.layers.15.mlp.experts.58", - "model.language_model.layers.15.mlp.experts.59", - "model.language_model.layers.15.mlp.experts.60", - "model.language_model.layers.15.mlp.experts.61", - "model.language_model.layers.15.mlp.experts.62", - "model.language_model.layers.15.mlp.experts.63", - "model.language_model.layers.15.mlp.experts.64", - "model.language_model.layers.15.mlp.experts.65", - "model.language_model.layers.15.mlp.experts.66", - "model.language_model.layers.15.mlp.experts.67", - "model.language_model.layers.15.mlp.experts.68", - "model.language_model.layers.15.mlp.experts.69", - "model.language_model.layers.15.mlp.experts.70", - "model.language_model.layers.15.mlp.experts.71", - "model.language_model.layers.15.mlp.experts.72", - "model.language_model.layers.15.mlp.experts.73", - "model.language_model.layers.15.mlp.experts.74", - "model.language_model.layers.15.mlp.experts.75", - "model.language_model.layers.15.mlp.experts.76", - "model.language_model.layers.15.mlp.experts.77", - "model.language_model.layers.15.mlp.experts.78", - "model.language_model.layers.15.mlp.experts.79", - "model.language_model.layers.15.mlp.experts.80", - "model.language_model.layers.15.mlp.experts.81", - "model.language_model.layers.15.mlp.experts.82", - "model.language_model.layers.15.mlp.experts.83", - "model.language_model.layers.15.mlp.experts.84", - "model.language_model.layers.15.mlp.experts.85", - "model.language_model.layers.15.mlp.experts.86", - "model.language_model.layers.15.mlp.experts.87", - "model.language_model.layers.15.mlp.experts.88", - "model.language_model.layers.15.mlp.experts.89", - "model.language_model.layers.15.mlp.experts.90", - "model.language_model.layers.15.mlp.experts.91", - "model.language_model.layers.15.mlp.experts.92", - "model.language_model.layers.15.mlp.experts.93", - "model.language_model.layers.15.mlp.experts.94", - "model.language_model.layers.15.mlp.experts.95", - "model.language_model.layers.15.mlp.experts.96", - "model.language_model.layers.15.mlp.experts.97", - "model.language_model.layers.15.mlp.experts.98", - "model.language_model.layers.15.mlp.experts.99", - "model.language_model.layers.15.mlp.experts.100", - "model.language_model.layers.15.mlp.experts.101", - "model.language_model.layers.15.mlp.experts.102", - "model.language_model.layers.15.mlp.experts.103", - "model.language_model.layers.15.mlp.experts.104", - "model.language_model.layers.15.mlp.experts.105", - "model.language_model.layers.15.mlp.experts.106", - "model.language_model.layers.15.mlp.experts.107", - "model.language_model.layers.15.mlp.experts.108", - "model.language_model.layers.15.mlp.experts.109", - "model.language_model.layers.15.mlp.experts.110", - "model.language_model.layers.15.mlp.experts.111", - "model.language_model.layers.15.mlp.experts.112", - "model.language_model.layers.15.mlp.experts.113", - "model.language_model.layers.15.mlp.experts.114", - "model.language_model.layers.15.mlp.experts.115", - "model.language_model.layers.15.mlp.experts.116", - "model.language_model.layers.15.mlp.experts.117", - "model.language_model.layers.15.mlp.experts.118", - "model.language_model.layers.15.mlp.experts.119", - "model.language_model.layers.15.mlp.experts.120", - "model.language_model.layers.15.mlp.experts.121", - "model.language_model.layers.15.mlp.experts.122", - "model.language_model.layers.15.mlp.experts.123", - "model.language_model.layers.15.mlp.experts.124", - "model.language_model.layers.15.mlp.experts.125", - "model.language_model.layers.15.mlp.experts.126", - "model.language_model.layers.15.mlp.experts.127", - "model.language_model.layers.15.mlp.experts.128", - "model.language_model.layers.15.mlp.experts.129", - "model.language_model.layers.15.mlp.experts.130", - "model.language_model.layers.15.mlp.experts.131", - "model.language_model.layers.15.mlp.experts.132", - "model.language_model.layers.15.mlp.experts.133", - "model.language_model.layers.15.mlp.experts.134", - "model.language_model.layers.15.mlp.experts.135", - "model.language_model.layers.15.mlp.experts.136", - "model.language_model.layers.15.mlp.experts.137", - "model.language_model.layers.15.mlp.experts.138", - "model.language_model.layers.15.mlp.experts.139", - "model.language_model.layers.15.mlp.experts.140", - "model.language_model.layers.15.mlp.experts.141", - "model.language_model.layers.15.mlp.experts.142", - "model.language_model.layers.15.mlp.experts.143", - "model.language_model.layers.15.mlp.experts.144", - "model.language_model.layers.15.mlp.experts.145", - "model.language_model.layers.15.mlp.experts.146", - "model.language_model.layers.15.mlp.experts.147", - "model.language_model.layers.15.mlp.experts.148", - "model.language_model.layers.15.mlp.experts.149", - "model.language_model.layers.15.mlp.experts.150", - "model.language_model.layers.15.mlp.experts.151", - "model.language_model.layers.15.mlp.experts.152", - "model.language_model.layers.15.mlp.experts.153", - "model.language_model.layers.15.mlp.experts.154", - "model.language_model.layers.15.mlp.experts.155", - "model.language_model.layers.15.mlp.experts.156", - "model.language_model.layers.15.mlp.experts.157", - "model.language_model.layers.15.mlp.experts.158", - "model.language_model.layers.15.mlp.experts.159", - "model.language_model.layers.15.mlp.experts.160", - "model.language_model.layers.15.mlp.experts.161", - "model.language_model.layers.15.mlp.experts.162", - "model.language_model.layers.15.mlp.experts.163", - "model.language_model.layers.15.mlp.experts.164", - "model.language_model.layers.15.mlp.experts.165", - "model.language_model.layers.15.mlp.experts.166", - "model.language_model.layers.15.mlp.experts.167", - "model.language_model.layers.15.mlp.experts.168", - "model.language_model.layers.15.mlp.experts.169", - "model.language_model.layers.15.mlp.experts.170", - "model.language_model.layers.15.mlp.experts.171", - "model.language_model.layers.15.mlp.experts.172", - "model.language_model.layers.15.mlp.experts.173", - "model.language_model.layers.15.mlp.experts.174", - "model.language_model.layers.15.mlp.experts.175", - "model.language_model.layers.15.mlp.experts.176", - "model.language_model.layers.15.mlp.experts.177", - "model.language_model.layers.15.mlp.experts.178", - "model.language_model.layers.15.mlp.experts.179", - "model.language_model.layers.15.mlp.experts.180", - "model.language_model.layers.15.mlp.experts.181", - "model.language_model.layers.15.mlp.experts.182", - "model.language_model.layers.15.mlp.experts.183", - "model.language_model.layers.15.mlp.experts.184", - "model.language_model.layers.15.mlp.experts.185", - "model.language_model.layers.15.mlp.experts.186", - "model.language_model.layers.15.mlp.experts.187", - "model.language_model.layers.15.mlp.experts.188", - "model.language_model.layers.15.mlp.experts.189", - "model.language_model.layers.15.mlp.experts.190", - "model.language_model.layers.15.mlp.experts.191", - "model.language_model.layers.15.mlp.experts.192", - "model.language_model.layers.15.mlp.experts.193", - "model.language_model.layers.15.mlp.experts.194", - "model.language_model.layers.15.mlp.experts.195", - "model.language_model.layers.15.mlp.experts.196", - "model.language_model.layers.15.mlp.experts.197", - "model.language_model.layers.15.mlp.experts.198", - "model.language_model.layers.15.mlp.experts.199", - "model.language_model.layers.15.mlp.experts.200", - "model.language_model.layers.15.mlp.experts.201", - "model.language_model.layers.15.mlp.experts.202", - "model.language_model.layers.15.mlp.experts.203", - "model.language_model.layers.15.mlp.experts.204", - "model.language_model.layers.15.mlp.experts.205", - "model.language_model.layers.15.mlp.experts.206", - "model.language_model.layers.15.mlp.experts.207", - "model.language_model.layers.15.mlp.experts.208", - "model.language_model.layers.15.mlp.experts.209", - "model.language_model.layers.15.mlp.experts.210", - "model.language_model.layers.15.mlp.experts.211", - "model.language_model.layers.15.mlp.experts.212", - "model.language_model.layers.15.mlp.experts.213", - "model.language_model.layers.15.mlp.experts.214", - "model.language_model.layers.15.mlp.experts.215", - "model.language_model.layers.15.mlp.experts.216", - "model.language_model.layers.15.mlp.experts.217", - "model.language_model.layers.15.mlp.experts.218", - "model.language_model.layers.15.mlp.experts.219", - "model.language_model.layers.15.mlp.experts.220", - "model.language_model.layers.15.mlp.experts.221", - "model.language_model.layers.15.mlp.experts.222", - "model.language_model.layers.15.mlp.experts.223", - "model.language_model.layers.15.mlp.experts.224", - "model.language_model.layers.15.mlp.experts.225", - "model.language_model.layers.15.mlp.experts.226", - "model.language_model.layers.15.mlp.experts.227", - "model.language_model.layers.15.mlp.experts.228", - "model.language_model.layers.15.mlp.experts.229", - "model.language_model.layers.15.mlp.experts.230", - "model.language_model.layers.15.mlp.experts.231", - "model.language_model.layers.15.mlp.experts.232", - "model.language_model.layers.15.mlp.experts.233", - "model.language_model.layers.15.mlp.experts.234", - "model.language_model.layers.15.mlp.experts.235", - "model.language_model.layers.15.mlp.experts.236", - "model.language_model.layers.15.mlp.experts.237", - "model.language_model.layers.15.mlp.experts.238", - "model.language_model.layers.15.mlp.experts.239", - "model.language_model.layers.15.mlp.experts.240", - "model.language_model.layers.15.mlp.experts.241", - "model.language_model.layers.15.mlp.experts.242", - "model.language_model.layers.15.mlp.experts.243", - "model.language_model.layers.15.mlp.experts.244", - "model.language_model.layers.15.mlp.experts.245", - "model.language_model.layers.15.mlp.experts.246", - "model.language_model.layers.15.mlp.experts.247", - "model.language_model.layers.15.mlp.experts.248", - "model.language_model.layers.15.mlp.experts.249", - "model.language_model.layers.15.mlp.experts.250", - "model.language_model.layers.15.mlp.experts.251", - "model.language_model.layers.15.mlp.experts.252", - "model.language_model.layers.15.mlp.experts.253", - "model.language_model.layers.15.mlp.experts.254", - "model.language_model.layers.15.mlp.experts.255", "model.language_model.layers.15.mlp.shared_expert_gate", - "model.language_model.layers.16.linear_attn", "model.language_model.layers.16.linear_attn.norm", "model.language_model.layers.16.linear_attn.in_proj_b", "model.language_model.layers.16.linear_attn.in_proj_a", "model.language_model.layers.16.mlp.gate", - "model.language_model.layers.16.mlp.experts.0", - "model.language_model.layers.16.mlp.experts.1", - "model.language_model.layers.16.mlp.experts.2", - "model.language_model.layers.16.mlp.experts.3", - "model.language_model.layers.16.mlp.experts.4", - "model.language_model.layers.16.mlp.experts.5", - "model.language_model.layers.16.mlp.experts.6", - "model.language_model.layers.16.mlp.experts.7", - "model.language_model.layers.16.mlp.experts.8", - "model.language_model.layers.16.mlp.experts.9", - "model.language_model.layers.16.mlp.experts.10", - "model.language_model.layers.16.mlp.experts.11", - "model.language_model.layers.16.mlp.experts.12", - "model.language_model.layers.16.mlp.experts.13", - "model.language_model.layers.16.mlp.experts.14", - "model.language_model.layers.16.mlp.experts.15", - "model.language_model.layers.16.mlp.experts.16", - "model.language_model.layers.16.mlp.experts.17", - "model.language_model.layers.16.mlp.experts.18", - "model.language_model.layers.16.mlp.experts.19", - "model.language_model.layers.16.mlp.experts.20", - "model.language_model.layers.16.mlp.experts.21", - "model.language_model.layers.16.mlp.experts.22", - "model.language_model.layers.16.mlp.experts.23", - "model.language_model.layers.16.mlp.experts.24", - "model.language_model.layers.16.mlp.experts.25", - "model.language_model.layers.16.mlp.experts.26", - "model.language_model.layers.16.mlp.experts.27", - "model.language_model.layers.16.mlp.experts.28", - "model.language_model.layers.16.mlp.experts.29", - "model.language_model.layers.16.mlp.experts.30", - "model.language_model.layers.16.mlp.experts.31", - "model.language_model.layers.16.mlp.experts.32", - "model.language_model.layers.16.mlp.experts.33", - "model.language_model.layers.16.mlp.experts.34", - "model.language_model.layers.16.mlp.experts.35", - "model.language_model.layers.16.mlp.experts.36", - "model.language_model.layers.16.mlp.experts.37", - "model.language_model.layers.16.mlp.experts.38", - "model.language_model.layers.16.mlp.experts.39", - "model.language_model.layers.16.mlp.experts.40", - "model.language_model.layers.16.mlp.experts.41", - "model.language_model.layers.16.mlp.experts.42", - "model.language_model.layers.16.mlp.experts.43", - "model.language_model.layers.16.mlp.experts.44", - "model.language_model.layers.16.mlp.experts.45", - "model.language_model.layers.16.mlp.experts.46", - "model.language_model.layers.16.mlp.experts.47", - "model.language_model.layers.16.mlp.experts.48", - "model.language_model.layers.16.mlp.experts.49", - "model.language_model.layers.16.mlp.experts.50", - "model.language_model.layers.16.mlp.experts.51", - "model.language_model.layers.16.mlp.experts.52", - "model.language_model.layers.16.mlp.experts.53", - "model.language_model.layers.16.mlp.experts.54", - "model.language_model.layers.16.mlp.experts.55", - "model.language_model.layers.16.mlp.experts.56", - "model.language_model.layers.16.mlp.experts.57", - "model.language_model.layers.16.mlp.experts.58", - "model.language_model.layers.16.mlp.experts.59", - "model.language_model.layers.16.mlp.experts.60", - "model.language_model.layers.16.mlp.experts.61", - "model.language_model.layers.16.mlp.experts.62", - "model.language_model.layers.16.mlp.experts.63", - "model.language_model.layers.16.mlp.experts.64", - "model.language_model.layers.16.mlp.experts.65", - "model.language_model.layers.16.mlp.experts.66", - "model.language_model.layers.16.mlp.experts.67", - "model.language_model.layers.16.mlp.experts.68", - "model.language_model.layers.16.mlp.experts.69", - "model.language_model.layers.16.mlp.experts.70", - "model.language_model.layers.16.mlp.experts.71", - "model.language_model.layers.16.mlp.experts.72", - "model.language_model.layers.16.mlp.experts.73", - "model.language_model.layers.16.mlp.experts.74", - "model.language_model.layers.16.mlp.experts.75", - "model.language_model.layers.16.mlp.experts.76", - "model.language_model.layers.16.mlp.experts.77", - "model.language_model.layers.16.mlp.experts.78", - "model.language_model.layers.16.mlp.experts.79", - "model.language_model.layers.16.mlp.experts.80", - "model.language_model.layers.16.mlp.experts.81", - "model.language_model.layers.16.mlp.experts.82", - "model.language_model.layers.16.mlp.experts.83", - "model.language_model.layers.16.mlp.experts.84", - "model.language_model.layers.16.mlp.experts.85", - "model.language_model.layers.16.mlp.experts.86", - "model.language_model.layers.16.mlp.experts.87", - "model.language_model.layers.16.mlp.experts.88", - "model.language_model.layers.16.mlp.experts.89", - "model.language_model.layers.16.mlp.experts.90", - "model.language_model.layers.16.mlp.experts.91", - "model.language_model.layers.16.mlp.experts.92", - "model.language_model.layers.16.mlp.experts.93", - "model.language_model.layers.16.mlp.experts.94", - "model.language_model.layers.16.mlp.experts.95", - "model.language_model.layers.16.mlp.experts.96", - "model.language_model.layers.16.mlp.experts.97", - "model.language_model.layers.16.mlp.experts.98", - "model.language_model.layers.16.mlp.experts.99", - "model.language_model.layers.16.mlp.experts.100", - "model.language_model.layers.16.mlp.experts.101", - "model.language_model.layers.16.mlp.experts.102", - "model.language_model.layers.16.mlp.experts.103", - "model.language_model.layers.16.mlp.experts.104", - "model.language_model.layers.16.mlp.experts.105", - "model.language_model.layers.16.mlp.experts.106", - "model.language_model.layers.16.mlp.experts.107", - "model.language_model.layers.16.mlp.experts.108", - "model.language_model.layers.16.mlp.experts.109", - "model.language_model.layers.16.mlp.experts.110", - "model.language_model.layers.16.mlp.experts.111", - "model.language_model.layers.16.mlp.experts.112", - "model.language_model.layers.16.mlp.experts.113", - "model.language_model.layers.16.mlp.experts.114", - "model.language_model.layers.16.mlp.experts.115", - "model.language_model.layers.16.mlp.experts.116", - "model.language_model.layers.16.mlp.experts.117", - "model.language_model.layers.16.mlp.experts.118", - "model.language_model.layers.16.mlp.experts.119", - "model.language_model.layers.16.mlp.experts.120", - "model.language_model.layers.16.mlp.experts.121", - "model.language_model.layers.16.mlp.experts.122", - "model.language_model.layers.16.mlp.experts.123", - "model.language_model.layers.16.mlp.experts.124", - "model.language_model.layers.16.mlp.experts.125", - "model.language_model.layers.16.mlp.experts.126", - "model.language_model.layers.16.mlp.experts.127", - "model.language_model.layers.16.mlp.experts.128", - "model.language_model.layers.16.mlp.experts.129", - "model.language_model.layers.16.mlp.experts.130", - "model.language_model.layers.16.mlp.experts.131", - "model.language_model.layers.16.mlp.experts.132", - "model.language_model.layers.16.mlp.experts.133", - "model.language_model.layers.16.mlp.experts.134", - "model.language_model.layers.16.mlp.experts.135", - "model.language_model.layers.16.mlp.experts.136", - "model.language_model.layers.16.mlp.experts.137", - "model.language_model.layers.16.mlp.experts.138", - "model.language_model.layers.16.mlp.experts.139", - "model.language_model.layers.16.mlp.experts.140", - "model.language_model.layers.16.mlp.experts.141", - "model.language_model.layers.16.mlp.experts.142", - "model.language_model.layers.16.mlp.experts.143", - "model.language_model.layers.16.mlp.experts.144", - "model.language_model.layers.16.mlp.experts.145", - "model.language_model.layers.16.mlp.experts.146", - "model.language_model.layers.16.mlp.experts.147", - "model.language_model.layers.16.mlp.experts.148", - "model.language_model.layers.16.mlp.experts.149", - "model.language_model.layers.16.mlp.experts.150", - "model.language_model.layers.16.mlp.experts.151", - "model.language_model.layers.16.mlp.experts.152", - "model.language_model.layers.16.mlp.experts.153", - "model.language_model.layers.16.mlp.experts.154", - "model.language_model.layers.16.mlp.experts.155", - "model.language_model.layers.16.mlp.experts.156", - "model.language_model.layers.16.mlp.experts.157", - "model.language_model.layers.16.mlp.experts.158", - "model.language_model.layers.16.mlp.experts.159", - "model.language_model.layers.16.mlp.experts.160", - "model.language_model.layers.16.mlp.experts.161", - "model.language_model.layers.16.mlp.experts.162", - "model.language_model.layers.16.mlp.experts.163", - "model.language_model.layers.16.mlp.experts.164", - "model.language_model.layers.16.mlp.experts.165", - "model.language_model.layers.16.mlp.experts.166", - "model.language_model.layers.16.mlp.experts.167", - "model.language_model.layers.16.mlp.experts.168", - "model.language_model.layers.16.mlp.experts.169", - "model.language_model.layers.16.mlp.experts.170", - "model.language_model.layers.16.mlp.experts.171", - "model.language_model.layers.16.mlp.experts.172", - "model.language_model.layers.16.mlp.experts.173", - "model.language_model.layers.16.mlp.experts.174", - "model.language_model.layers.16.mlp.experts.175", - "model.language_model.layers.16.mlp.experts.176", - "model.language_model.layers.16.mlp.experts.177", - "model.language_model.layers.16.mlp.experts.178", - "model.language_model.layers.16.mlp.experts.179", - "model.language_model.layers.16.mlp.experts.180", - "model.language_model.layers.16.mlp.experts.181", - "model.language_model.layers.16.mlp.experts.182", - "model.language_model.layers.16.mlp.experts.183", - "model.language_model.layers.16.mlp.experts.184", - "model.language_model.layers.16.mlp.experts.185", - "model.language_model.layers.16.mlp.experts.186", - "model.language_model.layers.16.mlp.experts.187", - "model.language_model.layers.16.mlp.experts.188", - "model.language_model.layers.16.mlp.experts.189", - "model.language_model.layers.16.mlp.experts.190", - "model.language_model.layers.16.mlp.experts.191", - "model.language_model.layers.16.mlp.experts.192", - "model.language_model.layers.16.mlp.experts.193", - "model.language_model.layers.16.mlp.experts.194", - "model.language_model.layers.16.mlp.experts.195", - "model.language_model.layers.16.mlp.experts.196", - "model.language_model.layers.16.mlp.experts.197", - "model.language_model.layers.16.mlp.experts.198", - "model.language_model.layers.16.mlp.experts.199", - "model.language_model.layers.16.mlp.experts.200", - "model.language_model.layers.16.mlp.experts.201", - "model.language_model.layers.16.mlp.experts.202", - "model.language_model.layers.16.mlp.experts.203", - "model.language_model.layers.16.mlp.experts.204", - "model.language_model.layers.16.mlp.experts.205", - "model.language_model.layers.16.mlp.experts.206", - "model.language_model.layers.16.mlp.experts.207", - "model.language_model.layers.16.mlp.experts.208", - "model.language_model.layers.16.mlp.experts.209", - "model.language_model.layers.16.mlp.experts.210", - "model.language_model.layers.16.mlp.experts.211", - "model.language_model.layers.16.mlp.experts.212", - "model.language_model.layers.16.mlp.experts.213", - "model.language_model.layers.16.mlp.experts.214", - "model.language_model.layers.16.mlp.experts.215", - "model.language_model.layers.16.mlp.experts.216", - "model.language_model.layers.16.mlp.experts.217", - "model.language_model.layers.16.mlp.experts.218", - "model.language_model.layers.16.mlp.experts.219", - "model.language_model.layers.16.mlp.experts.220", - "model.language_model.layers.16.mlp.experts.221", - "model.language_model.layers.16.mlp.experts.222", - "model.language_model.layers.16.mlp.experts.223", - "model.language_model.layers.16.mlp.experts.224", - "model.language_model.layers.16.mlp.experts.225", - "model.language_model.layers.16.mlp.experts.226", - "model.language_model.layers.16.mlp.experts.227", - "model.language_model.layers.16.mlp.experts.228", - "model.language_model.layers.16.mlp.experts.229", - "model.language_model.layers.16.mlp.experts.230", - "model.language_model.layers.16.mlp.experts.231", - "model.language_model.layers.16.mlp.experts.232", - "model.language_model.layers.16.mlp.experts.233", - "model.language_model.layers.16.mlp.experts.234", - "model.language_model.layers.16.mlp.experts.235", - "model.language_model.layers.16.mlp.experts.236", - "model.language_model.layers.16.mlp.experts.237", - "model.language_model.layers.16.mlp.experts.238", - "model.language_model.layers.16.mlp.experts.239", - "model.language_model.layers.16.mlp.experts.240", - "model.language_model.layers.16.mlp.experts.241", - "model.language_model.layers.16.mlp.experts.242", - "model.language_model.layers.16.mlp.experts.243", - "model.language_model.layers.16.mlp.experts.244", - "model.language_model.layers.16.mlp.experts.245", - "model.language_model.layers.16.mlp.experts.246", - "model.language_model.layers.16.mlp.experts.247", - "model.language_model.layers.16.mlp.experts.248", - "model.language_model.layers.16.mlp.experts.249", - "model.language_model.layers.16.mlp.experts.250", - "model.language_model.layers.16.mlp.experts.251", - "model.language_model.layers.16.mlp.experts.252", - "model.language_model.layers.16.mlp.experts.253", - "model.language_model.layers.16.mlp.experts.254", - "model.language_model.layers.16.mlp.experts.255", "model.language_model.layers.16.mlp.shared_expert_gate", - "model.language_model.layers.17.linear_attn", "model.language_model.layers.17.linear_attn.norm", "model.language_model.layers.17.linear_attn.in_proj_b", "model.language_model.layers.17.linear_attn.in_proj_a", "model.language_model.layers.17.mlp.gate", - "model.language_model.layers.17.mlp.experts.0", - "model.language_model.layers.17.mlp.experts.1", - "model.language_model.layers.17.mlp.experts.2", - "model.language_model.layers.17.mlp.experts.3", - "model.language_model.layers.17.mlp.experts.4", - "model.language_model.layers.17.mlp.experts.5", - "model.language_model.layers.17.mlp.experts.6", - "model.language_model.layers.17.mlp.experts.7", - "model.language_model.layers.17.mlp.experts.8", - "model.language_model.layers.17.mlp.experts.9", - "model.language_model.layers.17.mlp.experts.10", - "model.language_model.layers.17.mlp.experts.11", - "model.language_model.layers.17.mlp.experts.12", - "model.language_model.layers.17.mlp.experts.13", - "model.language_model.layers.17.mlp.experts.14", - "model.language_model.layers.17.mlp.experts.15", - "model.language_model.layers.17.mlp.experts.16", - "model.language_model.layers.17.mlp.experts.17", - "model.language_model.layers.17.mlp.experts.18", - "model.language_model.layers.17.mlp.experts.19", - "model.language_model.layers.17.mlp.experts.20", - "model.language_model.layers.17.mlp.experts.21", - "model.language_model.layers.17.mlp.experts.22", - "model.language_model.layers.17.mlp.experts.23", - "model.language_model.layers.17.mlp.experts.24", - "model.language_model.layers.17.mlp.experts.25", - "model.language_model.layers.17.mlp.experts.26", - "model.language_model.layers.17.mlp.experts.27", - "model.language_model.layers.17.mlp.experts.28", - "model.language_model.layers.17.mlp.experts.29", - "model.language_model.layers.17.mlp.experts.30", - "model.language_model.layers.17.mlp.experts.31", - "model.language_model.layers.17.mlp.experts.32", - "model.language_model.layers.17.mlp.experts.33", - "model.language_model.layers.17.mlp.experts.34", - "model.language_model.layers.17.mlp.experts.35", - "model.language_model.layers.17.mlp.experts.36", - "model.language_model.layers.17.mlp.experts.37", - "model.language_model.layers.17.mlp.experts.38", - "model.language_model.layers.17.mlp.experts.39", - "model.language_model.layers.17.mlp.experts.40", - "model.language_model.layers.17.mlp.experts.41", - "model.language_model.layers.17.mlp.experts.42", - "model.language_model.layers.17.mlp.experts.43", - "model.language_model.layers.17.mlp.experts.44", - "model.language_model.layers.17.mlp.experts.45", - "model.language_model.layers.17.mlp.experts.46", - "model.language_model.layers.17.mlp.experts.47", - "model.language_model.layers.17.mlp.experts.48", - "model.language_model.layers.17.mlp.experts.49", - "model.language_model.layers.17.mlp.experts.50", - "model.language_model.layers.17.mlp.experts.51", - "model.language_model.layers.17.mlp.experts.52", - "model.language_model.layers.17.mlp.experts.53", - "model.language_model.layers.17.mlp.experts.54", - "model.language_model.layers.17.mlp.experts.55", - "model.language_model.layers.17.mlp.experts.56", - "model.language_model.layers.17.mlp.experts.57", - "model.language_model.layers.17.mlp.experts.58", - "model.language_model.layers.17.mlp.experts.59", - "model.language_model.layers.17.mlp.experts.60", - "model.language_model.layers.17.mlp.experts.61", - "model.language_model.layers.17.mlp.experts.62", - "model.language_model.layers.17.mlp.experts.63", - "model.language_model.layers.17.mlp.experts.64", - "model.language_model.layers.17.mlp.experts.65", - "model.language_model.layers.17.mlp.experts.66", - "model.language_model.layers.17.mlp.experts.67", - "model.language_model.layers.17.mlp.experts.68", - "model.language_model.layers.17.mlp.experts.69", - "model.language_model.layers.17.mlp.experts.70", - "model.language_model.layers.17.mlp.experts.71", - "model.language_model.layers.17.mlp.experts.72", - "model.language_model.layers.17.mlp.experts.73", - "model.language_model.layers.17.mlp.experts.74", - "model.language_model.layers.17.mlp.experts.75", - "model.language_model.layers.17.mlp.experts.76", - "model.language_model.layers.17.mlp.experts.77", - "model.language_model.layers.17.mlp.experts.78", - "model.language_model.layers.17.mlp.experts.79", - "model.language_model.layers.17.mlp.experts.80", - "model.language_model.layers.17.mlp.experts.81", - "model.language_model.layers.17.mlp.experts.82", - "model.language_model.layers.17.mlp.experts.83", - "model.language_model.layers.17.mlp.experts.84", - "model.language_model.layers.17.mlp.experts.85", - "model.language_model.layers.17.mlp.experts.86", - "model.language_model.layers.17.mlp.experts.87", - "model.language_model.layers.17.mlp.experts.88", - "model.language_model.layers.17.mlp.experts.89", - "model.language_model.layers.17.mlp.experts.90", - "model.language_model.layers.17.mlp.experts.91", - "model.language_model.layers.17.mlp.experts.92", - "model.language_model.layers.17.mlp.experts.93", - "model.language_model.layers.17.mlp.experts.94", - "model.language_model.layers.17.mlp.experts.95", - "model.language_model.layers.17.mlp.experts.96", - "model.language_model.layers.17.mlp.experts.97", - "model.language_model.layers.17.mlp.experts.98", - "model.language_model.layers.17.mlp.experts.99", - "model.language_model.layers.17.mlp.experts.100", - "model.language_model.layers.17.mlp.experts.101", - "model.language_model.layers.17.mlp.experts.102", - "model.language_model.layers.17.mlp.experts.103", - "model.language_model.layers.17.mlp.experts.104", - "model.language_model.layers.17.mlp.experts.105", - "model.language_model.layers.17.mlp.experts.106", - "model.language_model.layers.17.mlp.experts.107", - "model.language_model.layers.17.mlp.experts.108", - "model.language_model.layers.17.mlp.experts.109", - "model.language_model.layers.17.mlp.experts.110", - "model.language_model.layers.17.mlp.experts.111", - "model.language_model.layers.17.mlp.experts.112", - "model.language_model.layers.17.mlp.experts.113", - "model.language_model.layers.17.mlp.experts.114", - "model.language_model.layers.17.mlp.experts.115", - "model.language_model.layers.17.mlp.experts.116", - "model.language_model.layers.17.mlp.experts.117", - "model.language_model.layers.17.mlp.experts.118", - "model.language_model.layers.17.mlp.experts.119", - "model.language_model.layers.17.mlp.experts.120", - "model.language_model.layers.17.mlp.experts.121", - "model.language_model.layers.17.mlp.experts.122", - "model.language_model.layers.17.mlp.experts.123", - "model.language_model.layers.17.mlp.experts.124", - "model.language_model.layers.17.mlp.experts.125", - "model.language_model.layers.17.mlp.experts.126", - "model.language_model.layers.17.mlp.experts.127", - "model.language_model.layers.17.mlp.experts.128", - "model.language_model.layers.17.mlp.experts.129", - "model.language_model.layers.17.mlp.experts.130", - "model.language_model.layers.17.mlp.experts.131", - "model.language_model.layers.17.mlp.experts.132", - "model.language_model.layers.17.mlp.experts.133", - "model.language_model.layers.17.mlp.experts.134", - "model.language_model.layers.17.mlp.experts.135", - "model.language_model.layers.17.mlp.experts.136", - "model.language_model.layers.17.mlp.experts.137", - "model.language_model.layers.17.mlp.experts.138", - "model.language_model.layers.17.mlp.experts.139", - "model.language_model.layers.17.mlp.experts.140", - "model.language_model.layers.17.mlp.experts.141", - "model.language_model.layers.17.mlp.experts.142", - "model.language_model.layers.17.mlp.experts.143", - "model.language_model.layers.17.mlp.experts.144", - "model.language_model.layers.17.mlp.experts.145", - "model.language_model.layers.17.mlp.experts.146", - "model.language_model.layers.17.mlp.experts.147", - "model.language_model.layers.17.mlp.experts.148", - "model.language_model.layers.17.mlp.experts.149", - "model.language_model.layers.17.mlp.experts.150", - "model.language_model.layers.17.mlp.experts.151", - "model.language_model.layers.17.mlp.experts.152", - "model.language_model.layers.17.mlp.experts.153", - "model.language_model.layers.17.mlp.experts.154", - "model.language_model.layers.17.mlp.experts.155", - "model.language_model.layers.17.mlp.experts.156", - "model.language_model.layers.17.mlp.experts.157", - "model.language_model.layers.17.mlp.experts.158", - "model.language_model.layers.17.mlp.experts.159", - "model.language_model.layers.17.mlp.experts.160", - "model.language_model.layers.17.mlp.experts.161", - "model.language_model.layers.17.mlp.experts.162", - "model.language_model.layers.17.mlp.experts.163", - "model.language_model.layers.17.mlp.experts.164", - "model.language_model.layers.17.mlp.experts.165", - "model.language_model.layers.17.mlp.experts.166", - "model.language_model.layers.17.mlp.experts.167", - "model.language_model.layers.17.mlp.experts.168", - "model.language_model.layers.17.mlp.experts.169", - "model.language_model.layers.17.mlp.experts.170", - "model.language_model.layers.17.mlp.experts.171", - "model.language_model.layers.17.mlp.experts.172", - "model.language_model.layers.17.mlp.experts.173", - "model.language_model.layers.17.mlp.experts.174", - "model.language_model.layers.17.mlp.experts.175", - "model.language_model.layers.17.mlp.experts.176", - "model.language_model.layers.17.mlp.experts.177", - "model.language_model.layers.17.mlp.experts.178", - "model.language_model.layers.17.mlp.experts.179", - "model.language_model.layers.17.mlp.experts.180", - "model.language_model.layers.17.mlp.experts.181", - "model.language_model.layers.17.mlp.experts.182", - "model.language_model.layers.17.mlp.experts.183", - "model.language_model.layers.17.mlp.experts.184", - "model.language_model.layers.17.mlp.experts.185", - "model.language_model.layers.17.mlp.experts.186", - "model.language_model.layers.17.mlp.experts.187", - "model.language_model.layers.17.mlp.experts.188", - "model.language_model.layers.17.mlp.experts.189", - "model.language_model.layers.17.mlp.experts.190", - "model.language_model.layers.17.mlp.experts.191", - "model.language_model.layers.17.mlp.experts.192", - "model.language_model.layers.17.mlp.experts.193", - "model.language_model.layers.17.mlp.experts.194", - "model.language_model.layers.17.mlp.experts.195", - "model.language_model.layers.17.mlp.experts.196", - "model.language_model.layers.17.mlp.experts.197", - "model.language_model.layers.17.mlp.experts.198", - "model.language_model.layers.17.mlp.experts.199", - "model.language_model.layers.17.mlp.experts.200", - "model.language_model.layers.17.mlp.experts.201", - "model.language_model.layers.17.mlp.experts.202", - "model.language_model.layers.17.mlp.experts.203", - "model.language_model.layers.17.mlp.experts.204", - "model.language_model.layers.17.mlp.experts.205", - "model.language_model.layers.17.mlp.experts.206", - "model.language_model.layers.17.mlp.experts.207", - "model.language_model.layers.17.mlp.experts.208", - "model.language_model.layers.17.mlp.experts.209", - "model.language_model.layers.17.mlp.experts.210", - "model.language_model.layers.17.mlp.experts.211", - "model.language_model.layers.17.mlp.experts.212", - "model.language_model.layers.17.mlp.experts.213", - "model.language_model.layers.17.mlp.experts.214", - "model.language_model.layers.17.mlp.experts.215", - "model.language_model.layers.17.mlp.experts.216", - "model.language_model.layers.17.mlp.experts.217", - "model.language_model.layers.17.mlp.experts.218", - "model.language_model.layers.17.mlp.experts.219", - "model.language_model.layers.17.mlp.experts.220", - "model.language_model.layers.17.mlp.experts.221", - "model.language_model.layers.17.mlp.experts.222", - "model.language_model.layers.17.mlp.experts.223", - "model.language_model.layers.17.mlp.experts.224", - "model.language_model.layers.17.mlp.experts.225", - "model.language_model.layers.17.mlp.experts.226", - "model.language_model.layers.17.mlp.experts.227", - "model.language_model.layers.17.mlp.experts.228", - "model.language_model.layers.17.mlp.experts.229", - "model.language_model.layers.17.mlp.experts.230", - "model.language_model.layers.17.mlp.experts.231", - "model.language_model.layers.17.mlp.experts.232", - "model.language_model.layers.17.mlp.experts.233", - "model.language_model.layers.17.mlp.experts.234", - "model.language_model.layers.17.mlp.experts.235", - "model.language_model.layers.17.mlp.experts.236", - "model.language_model.layers.17.mlp.experts.237", - "model.language_model.layers.17.mlp.experts.238", - "model.language_model.layers.17.mlp.experts.239", - "model.language_model.layers.17.mlp.experts.240", - "model.language_model.layers.17.mlp.experts.241", - "model.language_model.layers.17.mlp.experts.242", - "model.language_model.layers.17.mlp.experts.243", - "model.language_model.layers.17.mlp.experts.244", - "model.language_model.layers.17.mlp.experts.245", - "model.language_model.layers.17.mlp.experts.246", - "model.language_model.layers.17.mlp.experts.247", - "model.language_model.layers.17.mlp.experts.248", - "model.language_model.layers.17.mlp.experts.249", - "model.language_model.layers.17.mlp.experts.250", - "model.language_model.layers.17.mlp.experts.251", - "model.language_model.layers.17.mlp.experts.252", - "model.language_model.layers.17.mlp.experts.253", - "model.language_model.layers.17.mlp.experts.254", - "model.language_model.layers.17.mlp.experts.255", "model.language_model.layers.17.mlp.shared_expert_gate", - "model.language_model.layers.18.linear_attn", "model.language_model.layers.18.linear_attn.norm", "model.language_model.layers.18.linear_attn.in_proj_b", "model.language_model.layers.18.linear_attn.in_proj_a", "model.language_model.layers.18.mlp.gate", - "model.language_model.layers.18.mlp.experts.0", - "model.language_model.layers.18.mlp.experts.1", - "model.language_model.layers.18.mlp.experts.2", - "model.language_model.layers.18.mlp.experts.3", - "model.language_model.layers.18.mlp.experts.4", - "model.language_model.layers.18.mlp.experts.5", - "model.language_model.layers.18.mlp.experts.6", - "model.language_model.layers.18.mlp.experts.7", - "model.language_model.layers.18.mlp.experts.8", - "model.language_model.layers.18.mlp.experts.9", - "model.language_model.layers.18.mlp.experts.10", - "model.language_model.layers.18.mlp.experts.11", - "model.language_model.layers.18.mlp.experts.12", - "model.language_model.layers.18.mlp.experts.13", - "model.language_model.layers.18.mlp.experts.14", - "model.language_model.layers.18.mlp.experts.15", - "model.language_model.layers.18.mlp.experts.16", - "model.language_model.layers.18.mlp.experts.17", - "model.language_model.layers.18.mlp.experts.18", - "model.language_model.layers.18.mlp.experts.19", - "model.language_model.layers.18.mlp.experts.20", - "model.language_model.layers.18.mlp.experts.21", - "model.language_model.layers.18.mlp.experts.22", - "model.language_model.layers.18.mlp.experts.23", - "model.language_model.layers.18.mlp.experts.24", - "model.language_model.layers.18.mlp.experts.25", - "model.language_model.layers.18.mlp.experts.26", - "model.language_model.layers.18.mlp.experts.27", - "model.language_model.layers.18.mlp.experts.28", - "model.language_model.layers.18.mlp.experts.29", - "model.language_model.layers.18.mlp.experts.30", - "model.language_model.layers.18.mlp.experts.31", - "model.language_model.layers.18.mlp.experts.32", - "model.language_model.layers.18.mlp.experts.33", - "model.language_model.layers.18.mlp.experts.34", - "model.language_model.layers.18.mlp.experts.35", - "model.language_model.layers.18.mlp.experts.36", - "model.language_model.layers.18.mlp.experts.37", - "model.language_model.layers.18.mlp.experts.38", - "model.language_model.layers.18.mlp.experts.39", - "model.language_model.layers.18.mlp.experts.40", - "model.language_model.layers.18.mlp.experts.41", - "model.language_model.layers.18.mlp.experts.42", - "model.language_model.layers.18.mlp.experts.43", - "model.language_model.layers.18.mlp.experts.44", - "model.language_model.layers.18.mlp.experts.45", - "model.language_model.layers.18.mlp.experts.46", - "model.language_model.layers.18.mlp.experts.47", - "model.language_model.layers.18.mlp.experts.48", - "model.language_model.layers.18.mlp.experts.49", - "model.language_model.layers.18.mlp.experts.50", - "model.language_model.layers.18.mlp.experts.51", - "model.language_model.layers.18.mlp.experts.52", - "model.language_model.layers.18.mlp.experts.53", - "model.language_model.layers.18.mlp.experts.54", - "model.language_model.layers.18.mlp.experts.55", - "model.language_model.layers.18.mlp.experts.56", - "model.language_model.layers.18.mlp.experts.57", - "model.language_model.layers.18.mlp.experts.58", - "model.language_model.layers.18.mlp.experts.59", - "model.language_model.layers.18.mlp.experts.60", - "model.language_model.layers.18.mlp.experts.61", - "model.language_model.layers.18.mlp.experts.62", - "model.language_model.layers.18.mlp.experts.63", - "model.language_model.layers.18.mlp.experts.64", - "model.language_model.layers.18.mlp.experts.65", - "model.language_model.layers.18.mlp.experts.66", - "model.language_model.layers.18.mlp.experts.67", - "model.language_model.layers.18.mlp.experts.68", - "model.language_model.layers.18.mlp.experts.69", - "model.language_model.layers.18.mlp.experts.70", - "model.language_model.layers.18.mlp.experts.71", - "model.language_model.layers.18.mlp.experts.72", - "model.language_model.layers.18.mlp.experts.73", - "model.language_model.layers.18.mlp.experts.74", - "model.language_model.layers.18.mlp.experts.75", - "model.language_model.layers.18.mlp.experts.76", - "model.language_model.layers.18.mlp.experts.77", - "model.language_model.layers.18.mlp.experts.78", - "model.language_model.layers.18.mlp.experts.79", - "model.language_model.layers.18.mlp.experts.80", - "model.language_model.layers.18.mlp.experts.81", - "model.language_model.layers.18.mlp.experts.82", - "model.language_model.layers.18.mlp.experts.83", - "model.language_model.layers.18.mlp.experts.84", - "model.language_model.layers.18.mlp.experts.85", - "model.language_model.layers.18.mlp.experts.86", - "model.language_model.layers.18.mlp.experts.87", - "model.language_model.layers.18.mlp.experts.88", - "model.language_model.layers.18.mlp.experts.89", - "model.language_model.layers.18.mlp.experts.90", - "model.language_model.layers.18.mlp.experts.91", - "model.language_model.layers.18.mlp.experts.92", - "model.language_model.layers.18.mlp.experts.93", - "model.language_model.layers.18.mlp.experts.94", - "model.language_model.layers.18.mlp.experts.95", - "model.language_model.layers.18.mlp.experts.96", - "model.language_model.layers.18.mlp.experts.97", - "model.language_model.layers.18.mlp.experts.98", - "model.language_model.layers.18.mlp.experts.99", - "model.language_model.layers.18.mlp.experts.100", - "model.language_model.layers.18.mlp.experts.101", - "model.language_model.layers.18.mlp.experts.102", - "model.language_model.layers.18.mlp.experts.103", - "model.language_model.layers.18.mlp.experts.104", - "model.language_model.layers.18.mlp.experts.105", - "model.language_model.layers.18.mlp.experts.106", - "model.language_model.layers.18.mlp.experts.107", - "model.language_model.layers.18.mlp.experts.108", - "model.language_model.layers.18.mlp.experts.109", - "model.language_model.layers.18.mlp.experts.110", - "model.language_model.layers.18.mlp.experts.111", - "model.language_model.layers.18.mlp.experts.112", - "model.language_model.layers.18.mlp.experts.113", - "model.language_model.layers.18.mlp.experts.114", - "model.language_model.layers.18.mlp.experts.115", - "model.language_model.layers.18.mlp.experts.116", - "model.language_model.layers.18.mlp.experts.117", - "model.language_model.layers.18.mlp.experts.118", - "model.language_model.layers.18.mlp.experts.119", - "model.language_model.layers.18.mlp.experts.120", - "model.language_model.layers.18.mlp.experts.121", - "model.language_model.layers.18.mlp.experts.122", - "model.language_model.layers.18.mlp.experts.123", - "model.language_model.layers.18.mlp.experts.124", - "model.language_model.layers.18.mlp.experts.125", - "model.language_model.layers.18.mlp.experts.126", - "model.language_model.layers.18.mlp.experts.127", - "model.language_model.layers.18.mlp.experts.128", - "model.language_model.layers.18.mlp.experts.129", - "model.language_model.layers.18.mlp.experts.130", - "model.language_model.layers.18.mlp.experts.131", - "model.language_model.layers.18.mlp.experts.132", - "model.language_model.layers.18.mlp.experts.133", - "model.language_model.layers.18.mlp.experts.134", - "model.language_model.layers.18.mlp.experts.135", - "model.language_model.layers.18.mlp.experts.136", - "model.language_model.layers.18.mlp.experts.137", - "model.language_model.layers.18.mlp.experts.138", - "model.language_model.layers.18.mlp.experts.139", - "model.language_model.layers.18.mlp.experts.140", - "model.language_model.layers.18.mlp.experts.141", - "model.language_model.layers.18.mlp.experts.142", - "model.language_model.layers.18.mlp.experts.143", - "model.language_model.layers.18.mlp.experts.144", - "model.language_model.layers.18.mlp.experts.145", - "model.language_model.layers.18.mlp.experts.146", - "model.language_model.layers.18.mlp.experts.147", - "model.language_model.layers.18.mlp.experts.148", - "model.language_model.layers.18.mlp.experts.149", - "model.language_model.layers.18.mlp.experts.150", - "model.language_model.layers.18.mlp.experts.151", - "model.language_model.layers.18.mlp.experts.152", - "model.language_model.layers.18.mlp.experts.153", - "model.language_model.layers.18.mlp.experts.154", - "model.language_model.layers.18.mlp.experts.155", - "model.language_model.layers.18.mlp.experts.156", - "model.language_model.layers.18.mlp.experts.157", - "model.language_model.layers.18.mlp.experts.158", - "model.language_model.layers.18.mlp.experts.159", - "model.language_model.layers.18.mlp.experts.160", - "model.language_model.layers.18.mlp.experts.161", - "model.language_model.layers.18.mlp.experts.162", - "model.language_model.layers.18.mlp.experts.163", - "model.language_model.layers.18.mlp.experts.164", - "model.language_model.layers.18.mlp.experts.165", - "model.language_model.layers.18.mlp.experts.166", - "model.language_model.layers.18.mlp.experts.167", - "model.language_model.layers.18.mlp.experts.168", - "model.language_model.layers.18.mlp.experts.169", - "model.language_model.layers.18.mlp.experts.170", - "model.language_model.layers.18.mlp.experts.171", - "model.language_model.layers.18.mlp.experts.172", - "model.language_model.layers.18.mlp.experts.173", - "model.language_model.layers.18.mlp.experts.174", - "model.language_model.layers.18.mlp.experts.175", - "model.language_model.layers.18.mlp.experts.176", - "model.language_model.layers.18.mlp.experts.177", - "model.language_model.layers.18.mlp.experts.178", - "model.language_model.layers.18.mlp.experts.179", - "model.language_model.layers.18.mlp.experts.180", - "model.language_model.layers.18.mlp.experts.181", - "model.language_model.layers.18.mlp.experts.182", - "model.language_model.layers.18.mlp.experts.183", - "model.language_model.layers.18.mlp.experts.184", - "model.language_model.layers.18.mlp.experts.185", - "model.language_model.layers.18.mlp.experts.186", - "model.language_model.layers.18.mlp.experts.187", - "model.language_model.layers.18.mlp.experts.188", - "model.language_model.layers.18.mlp.experts.189", - "model.language_model.layers.18.mlp.experts.190", - "model.language_model.layers.18.mlp.experts.191", - "model.language_model.layers.18.mlp.experts.192", - "model.language_model.layers.18.mlp.experts.193", - "model.language_model.layers.18.mlp.experts.194", - "model.language_model.layers.18.mlp.experts.195", - "model.language_model.layers.18.mlp.experts.196", - "model.language_model.layers.18.mlp.experts.197", - "model.language_model.layers.18.mlp.experts.198", - "model.language_model.layers.18.mlp.experts.199", - "model.language_model.layers.18.mlp.experts.200", - "model.language_model.layers.18.mlp.experts.201", - "model.language_model.layers.18.mlp.experts.202", - "model.language_model.layers.18.mlp.experts.203", - "model.language_model.layers.18.mlp.experts.204", - "model.language_model.layers.18.mlp.experts.205", - "model.language_model.layers.18.mlp.experts.206", - "model.language_model.layers.18.mlp.experts.207", - "model.language_model.layers.18.mlp.experts.208", - "model.language_model.layers.18.mlp.experts.209", - "model.language_model.layers.18.mlp.experts.210", - "model.language_model.layers.18.mlp.experts.211", - "model.language_model.layers.18.mlp.experts.212", - "model.language_model.layers.18.mlp.experts.213", - "model.language_model.layers.18.mlp.experts.214", - "model.language_model.layers.18.mlp.experts.215", - "model.language_model.layers.18.mlp.experts.216", - "model.language_model.layers.18.mlp.experts.217", - "model.language_model.layers.18.mlp.experts.218", - "model.language_model.layers.18.mlp.experts.219", - "model.language_model.layers.18.mlp.experts.220", - "model.language_model.layers.18.mlp.experts.221", - "model.language_model.layers.18.mlp.experts.222", - "model.language_model.layers.18.mlp.experts.223", - "model.language_model.layers.18.mlp.experts.224", - "model.language_model.layers.18.mlp.experts.225", - "model.language_model.layers.18.mlp.experts.226", - "model.language_model.layers.18.mlp.experts.227", - "model.language_model.layers.18.mlp.experts.228", - "model.language_model.layers.18.mlp.experts.229", - "model.language_model.layers.18.mlp.experts.230", - "model.language_model.layers.18.mlp.experts.231", - "model.language_model.layers.18.mlp.experts.232", - "model.language_model.layers.18.mlp.experts.233", - "model.language_model.layers.18.mlp.experts.234", - "model.language_model.layers.18.mlp.experts.235", - "model.language_model.layers.18.mlp.experts.236", - "model.language_model.layers.18.mlp.experts.237", - "model.language_model.layers.18.mlp.experts.238", - "model.language_model.layers.18.mlp.experts.239", - "model.language_model.layers.18.mlp.experts.240", - "model.language_model.layers.18.mlp.experts.241", - "model.language_model.layers.18.mlp.experts.242", - "model.language_model.layers.18.mlp.experts.243", - "model.language_model.layers.18.mlp.experts.244", - "model.language_model.layers.18.mlp.experts.245", - "model.language_model.layers.18.mlp.experts.246", - "model.language_model.layers.18.mlp.experts.247", - "model.language_model.layers.18.mlp.experts.248", - "model.language_model.layers.18.mlp.experts.249", - "model.language_model.layers.18.mlp.experts.250", - "model.language_model.layers.18.mlp.experts.251", - "model.language_model.layers.18.mlp.experts.252", - "model.language_model.layers.18.mlp.experts.253", - "model.language_model.layers.18.mlp.experts.254", - "model.language_model.layers.18.mlp.experts.255", "model.language_model.layers.18.mlp.shared_expert_gate", "model.language_model.layers.19.mlp.gate", - "model.language_model.layers.19.mlp.experts.0", - "model.language_model.layers.19.mlp.experts.1", - "model.language_model.layers.19.mlp.experts.2", - "model.language_model.layers.19.mlp.experts.3", - "model.language_model.layers.19.mlp.experts.4", - "model.language_model.layers.19.mlp.experts.5", - "model.language_model.layers.19.mlp.experts.6", - "model.language_model.layers.19.mlp.experts.7", - "model.language_model.layers.19.mlp.experts.8", - "model.language_model.layers.19.mlp.experts.9", - "model.language_model.layers.19.mlp.experts.10", - "model.language_model.layers.19.mlp.experts.11", - "model.language_model.layers.19.mlp.experts.12", - "model.language_model.layers.19.mlp.experts.13", - "model.language_model.layers.19.mlp.experts.14", - "model.language_model.layers.19.mlp.experts.15", - "model.language_model.layers.19.mlp.experts.16", - "model.language_model.layers.19.mlp.experts.17", - "model.language_model.layers.19.mlp.experts.18", - "model.language_model.layers.19.mlp.experts.19", - "model.language_model.layers.19.mlp.experts.20", - "model.language_model.layers.19.mlp.experts.21", - "model.language_model.layers.19.mlp.experts.22", - "model.language_model.layers.19.mlp.experts.23", - "model.language_model.layers.19.mlp.experts.24", - "model.language_model.layers.19.mlp.experts.25", - "model.language_model.layers.19.mlp.experts.26", - "model.language_model.layers.19.mlp.experts.27", - "model.language_model.layers.19.mlp.experts.28", - "model.language_model.layers.19.mlp.experts.29", - "model.language_model.layers.19.mlp.experts.30", - "model.language_model.layers.19.mlp.experts.31", - "model.language_model.layers.19.mlp.experts.32", - "model.language_model.layers.19.mlp.experts.33", - "model.language_model.layers.19.mlp.experts.34", - "model.language_model.layers.19.mlp.experts.35", - "model.language_model.layers.19.mlp.experts.36", - "model.language_model.layers.19.mlp.experts.37", - "model.language_model.layers.19.mlp.experts.38", - "model.language_model.layers.19.mlp.experts.39", - "model.language_model.layers.19.mlp.experts.40", - "model.language_model.layers.19.mlp.experts.41", - "model.language_model.layers.19.mlp.experts.42", - "model.language_model.layers.19.mlp.experts.43", - "model.language_model.layers.19.mlp.experts.44", - "model.language_model.layers.19.mlp.experts.45", - "model.language_model.layers.19.mlp.experts.46", - "model.language_model.layers.19.mlp.experts.47", - "model.language_model.layers.19.mlp.experts.48", - "model.language_model.layers.19.mlp.experts.49", - "model.language_model.layers.19.mlp.experts.50", - "model.language_model.layers.19.mlp.experts.51", - "model.language_model.layers.19.mlp.experts.52", - "model.language_model.layers.19.mlp.experts.53", - "model.language_model.layers.19.mlp.experts.54", - "model.language_model.layers.19.mlp.experts.55", - "model.language_model.layers.19.mlp.experts.56", - "model.language_model.layers.19.mlp.experts.57", - "model.language_model.layers.19.mlp.experts.58", - "model.language_model.layers.19.mlp.experts.59", - "model.language_model.layers.19.mlp.experts.60", - "model.language_model.layers.19.mlp.experts.61", - "model.language_model.layers.19.mlp.experts.62", - "model.language_model.layers.19.mlp.experts.63", - "model.language_model.layers.19.mlp.experts.64", - "model.language_model.layers.19.mlp.experts.65", - "model.language_model.layers.19.mlp.experts.66", - "model.language_model.layers.19.mlp.experts.67", - "model.language_model.layers.19.mlp.experts.68", - "model.language_model.layers.19.mlp.experts.69", - "model.language_model.layers.19.mlp.experts.70", - "model.language_model.layers.19.mlp.experts.71", - "model.language_model.layers.19.mlp.experts.72", - "model.language_model.layers.19.mlp.experts.73", - "model.language_model.layers.19.mlp.experts.74", - "model.language_model.layers.19.mlp.experts.75", - "model.language_model.layers.19.mlp.experts.76", - "model.language_model.layers.19.mlp.experts.77", - "model.language_model.layers.19.mlp.experts.78", - "model.language_model.layers.19.mlp.experts.79", - "model.language_model.layers.19.mlp.experts.80", - "model.language_model.layers.19.mlp.experts.81", - "model.language_model.layers.19.mlp.experts.82", - "model.language_model.layers.19.mlp.experts.83", - "model.language_model.layers.19.mlp.experts.84", - "model.language_model.layers.19.mlp.experts.85", - "model.language_model.layers.19.mlp.experts.86", - "model.language_model.layers.19.mlp.experts.87", - "model.language_model.layers.19.mlp.experts.88", - "model.language_model.layers.19.mlp.experts.89", - "model.language_model.layers.19.mlp.experts.90", - "model.language_model.layers.19.mlp.experts.91", - "model.language_model.layers.19.mlp.experts.92", - "model.language_model.layers.19.mlp.experts.93", - "model.language_model.layers.19.mlp.experts.94", - "model.language_model.layers.19.mlp.experts.95", - "model.language_model.layers.19.mlp.experts.96", - "model.language_model.layers.19.mlp.experts.97", - "model.language_model.layers.19.mlp.experts.98", - "model.language_model.layers.19.mlp.experts.99", - "model.language_model.layers.19.mlp.experts.100", - "model.language_model.layers.19.mlp.experts.101", - "model.language_model.layers.19.mlp.experts.102", - "model.language_model.layers.19.mlp.experts.103", - "model.language_model.layers.19.mlp.experts.104", - "model.language_model.layers.19.mlp.experts.105", - "model.language_model.layers.19.mlp.experts.106", - "model.language_model.layers.19.mlp.experts.107", - "model.language_model.layers.19.mlp.experts.108", - "model.language_model.layers.19.mlp.experts.109", - "model.language_model.layers.19.mlp.experts.110", - "model.language_model.layers.19.mlp.experts.111", - "model.language_model.layers.19.mlp.experts.112", - "model.language_model.layers.19.mlp.experts.113", - "model.language_model.layers.19.mlp.experts.114", - "model.language_model.layers.19.mlp.experts.115", - "model.language_model.layers.19.mlp.experts.116", - "model.language_model.layers.19.mlp.experts.117", - "model.language_model.layers.19.mlp.experts.118", - "model.language_model.layers.19.mlp.experts.119", - "model.language_model.layers.19.mlp.experts.120", - "model.language_model.layers.19.mlp.experts.121", - "model.language_model.layers.19.mlp.experts.122", - "model.language_model.layers.19.mlp.experts.123", - "model.language_model.layers.19.mlp.experts.124", - "model.language_model.layers.19.mlp.experts.125", - "model.language_model.layers.19.mlp.experts.126", - "model.language_model.layers.19.mlp.experts.127", - "model.language_model.layers.19.mlp.experts.128", - "model.language_model.layers.19.mlp.experts.129", - "model.language_model.layers.19.mlp.experts.130", - "model.language_model.layers.19.mlp.experts.131", - "model.language_model.layers.19.mlp.experts.132", - "model.language_model.layers.19.mlp.experts.133", - "model.language_model.layers.19.mlp.experts.134", - "model.language_model.layers.19.mlp.experts.135", - "model.language_model.layers.19.mlp.experts.136", - "model.language_model.layers.19.mlp.experts.137", - "model.language_model.layers.19.mlp.experts.138", - "model.language_model.layers.19.mlp.experts.139", - "model.language_model.layers.19.mlp.experts.140", - "model.language_model.layers.19.mlp.experts.141", - "model.language_model.layers.19.mlp.experts.142", - "model.language_model.layers.19.mlp.experts.143", - "model.language_model.layers.19.mlp.experts.144", - "model.language_model.layers.19.mlp.experts.145", - "model.language_model.layers.19.mlp.experts.146", - "model.language_model.layers.19.mlp.experts.147", - "model.language_model.layers.19.mlp.experts.148", - "model.language_model.layers.19.mlp.experts.149", - "model.language_model.layers.19.mlp.experts.150", - "model.language_model.layers.19.mlp.experts.151", - "model.language_model.layers.19.mlp.experts.152", - "model.language_model.layers.19.mlp.experts.153", - "model.language_model.layers.19.mlp.experts.154", - "model.language_model.layers.19.mlp.experts.155", - "model.language_model.layers.19.mlp.experts.156", - "model.language_model.layers.19.mlp.experts.157", - "model.language_model.layers.19.mlp.experts.158", - "model.language_model.layers.19.mlp.experts.159", - "model.language_model.layers.19.mlp.experts.160", - "model.language_model.layers.19.mlp.experts.161", - "model.language_model.layers.19.mlp.experts.162", - "model.language_model.layers.19.mlp.experts.163", - "model.language_model.layers.19.mlp.experts.164", - "model.language_model.layers.19.mlp.experts.165", - "model.language_model.layers.19.mlp.experts.166", - "model.language_model.layers.19.mlp.experts.167", - "model.language_model.layers.19.mlp.experts.168", - "model.language_model.layers.19.mlp.experts.169", - "model.language_model.layers.19.mlp.experts.170", - "model.language_model.layers.19.mlp.experts.171", - "model.language_model.layers.19.mlp.experts.172", - "model.language_model.layers.19.mlp.experts.173", - "model.language_model.layers.19.mlp.experts.174", - "model.language_model.layers.19.mlp.experts.175", - "model.language_model.layers.19.mlp.experts.176", - "model.language_model.layers.19.mlp.experts.177", - "model.language_model.layers.19.mlp.experts.178", - "model.language_model.layers.19.mlp.experts.179", - "model.language_model.layers.19.mlp.experts.180", - "model.language_model.layers.19.mlp.experts.181", - "model.language_model.layers.19.mlp.experts.182", - "model.language_model.layers.19.mlp.experts.183", - "model.language_model.layers.19.mlp.experts.184", - "model.language_model.layers.19.mlp.experts.185", - "model.language_model.layers.19.mlp.experts.186", - "model.language_model.layers.19.mlp.experts.187", - "model.language_model.layers.19.mlp.experts.188", - "model.language_model.layers.19.mlp.experts.189", - "model.language_model.layers.19.mlp.experts.190", - "model.language_model.layers.19.mlp.experts.191", - "model.language_model.layers.19.mlp.experts.192", - "model.language_model.layers.19.mlp.experts.193", - "model.language_model.layers.19.mlp.experts.194", - "model.language_model.layers.19.mlp.experts.195", - "model.language_model.layers.19.mlp.experts.196", - "model.language_model.layers.19.mlp.experts.197", - "model.language_model.layers.19.mlp.experts.198", - "model.language_model.layers.19.mlp.experts.199", - "model.language_model.layers.19.mlp.experts.200", - "model.language_model.layers.19.mlp.experts.201", - "model.language_model.layers.19.mlp.experts.202", - "model.language_model.layers.19.mlp.experts.203", - "model.language_model.layers.19.mlp.experts.204", - "model.language_model.layers.19.mlp.experts.205", - "model.language_model.layers.19.mlp.experts.206", - "model.language_model.layers.19.mlp.experts.207", - "model.language_model.layers.19.mlp.experts.208", - "model.language_model.layers.19.mlp.experts.209", - "model.language_model.layers.19.mlp.experts.210", - "model.language_model.layers.19.mlp.experts.211", - "model.language_model.layers.19.mlp.experts.212", - "model.language_model.layers.19.mlp.experts.213", - "model.language_model.layers.19.mlp.experts.214", - "model.language_model.layers.19.mlp.experts.215", - "model.language_model.layers.19.mlp.experts.216", - "model.language_model.layers.19.mlp.experts.217", - "model.language_model.layers.19.mlp.experts.218", - "model.language_model.layers.19.mlp.experts.219", - "model.language_model.layers.19.mlp.experts.220", - "model.language_model.layers.19.mlp.experts.221", - "model.language_model.layers.19.mlp.experts.222", - "model.language_model.layers.19.mlp.experts.223", - "model.language_model.layers.19.mlp.experts.224", - "model.language_model.layers.19.mlp.experts.225", - "model.language_model.layers.19.mlp.experts.226", - "model.language_model.layers.19.mlp.experts.227", - "model.language_model.layers.19.mlp.experts.228", - "model.language_model.layers.19.mlp.experts.229", - "model.language_model.layers.19.mlp.experts.230", - "model.language_model.layers.19.mlp.experts.231", - "model.language_model.layers.19.mlp.experts.232", - "model.language_model.layers.19.mlp.experts.233", - "model.language_model.layers.19.mlp.experts.234", - "model.language_model.layers.19.mlp.experts.235", - "model.language_model.layers.19.mlp.experts.236", - "model.language_model.layers.19.mlp.experts.237", - "model.language_model.layers.19.mlp.experts.238", - "model.language_model.layers.19.mlp.experts.239", - "model.language_model.layers.19.mlp.experts.240", - "model.language_model.layers.19.mlp.experts.241", - "model.language_model.layers.19.mlp.experts.242", - "model.language_model.layers.19.mlp.experts.243", - "model.language_model.layers.19.mlp.experts.244", - "model.language_model.layers.19.mlp.experts.245", - "model.language_model.layers.19.mlp.experts.246", - "model.language_model.layers.19.mlp.experts.247", - "model.language_model.layers.19.mlp.experts.248", - "model.language_model.layers.19.mlp.experts.249", - "model.language_model.layers.19.mlp.experts.250", - "model.language_model.layers.19.mlp.experts.251", - "model.language_model.layers.19.mlp.experts.252", - "model.language_model.layers.19.mlp.experts.253", - "model.language_model.layers.19.mlp.experts.254", - "model.language_model.layers.19.mlp.experts.255", "model.language_model.layers.19.mlp.shared_expert_gate", - "model.language_model.layers.20.linear_attn", "model.language_model.layers.20.linear_attn.norm", "model.language_model.layers.20.linear_attn.in_proj_b", "model.language_model.layers.20.linear_attn.in_proj_a", "model.language_model.layers.20.mlp.gate", - "model.language_model.layers.20.mlp.experts.0", - "model.language_model.layers.20.mlp.experts.1", - "model.language_model.layers.20.mlp.experts.2", - "model.language_model.layers.20.mlp.experts.3", - "model.language_model.layers.20.mlp.experts.4", - "model.language_model.layers.20.mlp.experts.5", - "model.language_model.layers.20.mlp.experts.6", - "model.language_model.layers.20.mlp.experts.7", - "model.language_model.layers.20.mlp.experts.8", - "model.language_model.layers.20.mlp.experts.9", - "model.language_model.layers.20.mlp.experts.10", - "model.language_model.layers.20.mlp.experts.11", - "model.language_model.layers.20.mlp.experts.12", - "model.language_model.layers.20.mlp.experts.13", - "model.language_model.layers.20.mlp.experts.14", - "model.language_model.layers.20.mlp.experts.15", - "model.language_model.layers.20.mlp.experts.16", - "model.language_model.layers.20.mlp.experts.17", - "model.language_model.layers.20.mlp.experts.18", - "model.language_model.layers.20.mlp.experts.19", - "model.language_model.layers.20.mlp.experts.20", - "model.language_model.layers.20.mlp.experts.21", - "model.language_model.layers.20.mlp.experts.22", - "model.language_model.layers.20.mlp.experts.23", - "model.language_model.layers.20.mlp.experts.24", - "model.language_model.layers.20.mlp.experts.25", - "model.language_model.layers.20.mlp.experts.26", - "model.language_model.layers.20.mlp.experts.27", - "model.language_model.layers.20.mlp.experts.28", - "model.language_model.layers.20.mlp.experts.29", - "model.language_model.layers.20.mlp.experts.30", - "model.language_model.layers.20.mlp.experts.31", - "model.language_model.layers.20.mlp.experts.32", - "model.language_model.layers.20.mlp.experts.33", - "model.language_model.layers.20.mlp.experts.34", - "model.language_model.layers.20.mlp.experts.35", - "model.language_model.layers.20.mlp.experts.36", - "model.language_model.layers.20.mlp.experts.37", - "model.language_model.layers.20.mlp.experts.38", - "model.language_model.layers.20.mlp.experts.39", - "model.language_model.layers.20.mlp.experts.40", - "model.language_model.layers.20.mlp.experts.41", - "model.language_model.layers.20.mlp.experts.42", - "model.language_model.layers.20.mlp.experts.43", - "model.language_model.layers.20.mlp.experts.44", - "model.language_model.layers.20.mlp.experts.45", - "model.language_model.layers.20.mlp.experts.46", - "model.language_model.layers.20.mlp.experts.47", - "model.language_model.layers.20.mlp.experts.48", - "model.language_model.layers.20.mlp.experts.49", - "model.language_model.layers.20.mlp.experts.50", - "model.language_model.layers.20.mlp.experts.51", - "model.language_model.layers.20.mlp.experts.52", - "model.language_model.layers.20.mlp.experts.53", - "model.language_model.layers.20.mlp.experts.54", - "model.language_model.layers.20.mlp.experts.55", - "model.language_model.layers.20.mlp.experts.56", - "model.language_model.layers.20.mlp.experts.57", - "model.language_model.layers.20.mlp.experts.58", - "model.language_model.layers.20.mlp.experts.59", - "model.language_model.layers.20.mlp.experts.60", - "model.language_model.layers.20.mlp.experts.61", - "model.language_model.layers.20.mlp.experts.62", - "model.language_model.layers.20.mlp.experts.63", - "model.language_model.layers.20.mlp.experts.64", - "model.language_model.layers.20.mlp.experts.65", - "model.language_model.layers.20.mlp.experts.66", - "model.language_model.layers.20.mlp.experts.67", - "model.language_model.layers.20.mlp.experts.68", - "model.language_model.layers.20.mlp.experts.69", - "model.language_model.layers.20.mlp.experts.70", - "model.language_model.layers.20.mlp.experts.71", - "model.language_model.layers.20.mlp.experts.72", - "model.language_model.layers.20.mlp.experts.73", - "model.language_model.layers.20.mlp.experts.74", - "model.language_model.layers.20.mlp.experts.75", - "model.language_model.layers.20.mlp.experts.76", - "model.language_model.layers.20.mlp.experts.77", - "model.language_model.layers.20.mlp.experts.78", - "model.language_model.layers.20.mlp.experts.79", - "model.language_model.layers.20.mlp.experts.80", - "model.language_model.layers.20.mlp.experts.81", - "model.language_model.layers.20.mlp.experts.82", - "model.language_model.layers.20.mlp.experts.83", - "model.language_model.layers.20.mlp.experts.84", - "model.language_model.layers.20.mlp.experts.85", - "model.language_model.layers.20.mlp.experts.86", - "model.language_model.layers.20.mlp.experts.87", - "model.language_model.layers.20.mlp.experts.88", - "model.language_model.layers.20.mlp.experts.89", - "model.language_model.layers.20.mlp.experts.90", - "model.language_model.layers.20.mlp.experts.91", - "model.language_model.layers.20.mlp.experts.92", - "model.language_model.layers.20.mlp.experts.93", - "model.language_model.layers.20.mlp.experts.94", - "model.language_model.layers.20.mlp.experts.95", - "model.language_model.layers.20.mlp.experts.96", - "model.language_model.layers.20.mlp.experts.97", - "model.language_model.layers.20.mlp.experts.98", - "model.language_model.layers.20.mlp.experts.99", - "model.language_model.layers.20.mlp.experts.100", - "model.language_model.layers.20.mlp.experts.101", - "model.language_model.layers.20.mlp.experts.102", - "model.language_model.layers.20.mlp.experts.103", - "model.language_model.layers.20.mlp.experts.104", - "model.language_model.layers.20.mlp.experts.105", - "model.language_model.layers.20.mlp.experts.106", - "model.language_model.layers.20.mlp.experts.107", - "model.language_model.layers.20.mlp.experts.108", - "model.language_model.layers.20.mlp.experts.109", - "model.language_model.layers.20.mlp.experts.110", - "model.language_model.layers.20.mlp.experts.111", - "model.language_model.layers.20.mlp.experts.112", - "model.language_model.layers.20.mlp.experts.113", - "model.language_model.layers.20.mlp.experts.114", - "model.language_model.layers.20.mlp.experts.115", - "model.language_model.layers.20.mlp.experts.116", - "model.language_model.layers.20.mlp.experts.117", - "model.language_model.layers.20.mlp.experts.118", - "model.language_model.layers.20.mlp.experts.119", - "model.language_model.layers.20.mlp.experts.120", - "model.language_model.layers.20.mlp.experts.121", - "model.language_model.layers.20.mlp.experts.122", - "model.language_model.layers.20.mlp.experts.123", - "model.language_model.layers.20.mlp.experts.124", - "model.language_model.layers.20.mlp.experts.125", - "model.language_model.layers.20.mlp.experts.126", - "model.language_model.layers.20.mlp.experts.127", - "model.language_model.layers.20.mlp.experts.128", - "model.language_model.layers.20.mlp.experts.129", - "model.language_model.layers.20.mlp.experts.130", - "model.language_model.layers.20.mlp.experts.131", - "model.language_model.layers.20.mlp.experts.132", - "model.language_model.layers.20.mlp.experts.133", - "model.language_model.layers.20.mlp.experts.134", - "model.language_model.layers.20.mlp.experts.135", - "model.language_model.layers.20.mlp.experts.136", - "model.language_model.layers.20.mlp.experts.137", - "model.language_model.layers.20.mlp.experts.138", - "model.language_model.layers.20.mlp.experts.139", - "model.language_model.layers.20.mlp.experts.140", - "model.language_model.layers.20.mlp.experts.141", - "model.language_model.layers.20.mlp.experts.142", - "model.language_model.layers.20.mlp.experts.143", - "model.language_model.layers.20.mlp.experts.144", - "model.language_model.layers.20.mlp.experts.145", - "model.language_model.layers.20.mlp.experts.146", - "model.language_model.layers.20.mlp.experts.147", - "model.language_model.layers.20.mlp.experts.148", - "model.language_model.layers.20.mlp.experts.149", - "model.language_model.layers.20.mlp.experts.150", - "model.language_model.layers.20.mlp.experts.151", - "model.language_model.layers.20.mlp.experts.152", - "model.language_model.layers.20.mlp.experts.153", - "model.language_model.layers.20.mlp.experts.154", - "model.language_model.layers.20.mlp.experts.155", - "model.language_model.layers.20.mlp.experts.156", - "model.language_model.layers.20.mlp.experts.157", - "model.language_model.layers.20.mlp.experts.158", - "model.language_model.layers.20.mlp.experts.159", - "model.language_model.layers.20.mlp.experts.160", - "model.language_model.layers.20.mlp.experts.161", - "model.language_model.layers.20.mlp.experts.162", - "model.language_model.layers.20.mlp.experts.163", - "model.language_model.layers.20.mlp.experts.164", - "model.language_model.layers.20.mlp.experts.165", - "model.language_model.layers.20.mlp.experts.166", - "model.language_model.layers.20.mlp.experts.167", - "model.language_model.layers.20.mlp.experts.168", - "model.language_model.layers.20.mlp.experts.169", - "model.language_model.layers.20.mlp.experts.170", - "model.language_model.layers.20.mlp.experts.171", - "model.language_model.layers.20.mlp.experts.172", - "model.language_model.layers.20.mlp.experts.173", - "model.language_model.layers.20.mlp.experts.174", - "model.language_model.layers.20.mlp.experts.175", - "model.language_model.layers.20.mlp.experts.176", - "model.language_model.layers.20.mlp.experts.177", - "model.language_model.layers.20.mlp.experts.178", - "model.language_model.layers.20.mlp.experts.179", - "model.language_model.layers.20.mlp.experts.180", - "model.language_model.layers.20.mlp.experts.181", - "model.language_model.layers.20.mlp.experts.182", - "model.language_model.layers.20.mlp.experts.183", - "model.language_model.layers.20.mlp.experts.184", - "model.language_model.layers.20.mlp.experts.185", - "model.language_model.layers.20.mlp.experts.186", - "model.language_model.layers.20.mlp.experts.187", - "model.language_model.layers.20.mlp.experts.188", - "model.language_model.layers.20.mlp.experts.189", - "model.language_model.layers.20.mlp.experts.190", - "model.language_model.layers.20.mlp.experts.191", - "model.language_model.layers.20.mlp.experts.192", - "model.language_model.layers.20.mlp.experts.193", - "model.language_model.layers.20.mlp.experts.194", - "model.language_model.layers.20.mlp.experts.195", - "model.language_model.layers.20.mlp.experts.196", - "model.language_model.layers.20.mlp.experts.197", - "model.language_model.layers.20.mlp.experts.198", - "model.language_model.layers.20.mlp.experts.199", - "model.language_model.layers.20.mlp.experts.200", - "model.language_model.layers.20.mlp.experts.201", - "model.language_model.layers.20.mlp.experts.202", - "model.language_model.layers.20.mlp.experts.203", - "model.language_model.layers.20.mlp.experts.204", - "model.language_model.layers.20.mlp.experts.205", - "model.language_model.layers.20.mlp.experts.206", - "model.language_model.layers.20.mlp.experts.207", - "model.language_model.layers.20.mlp.experts.208", - "model.language_model.layers.20.mlp.experts.209", - "model.language_model.layers.20.mlp.experts.210", - "model.language_model.layers.20.mlp.experts.211", - "model.language_model.layers.20.mlp.experts.212", - "model.language_model.layers.20.mlp.experts.213", - "model.language_model.layers.20.mlp.experts.214", - "model.language_model.layers.20.mlp.experts.215", - "model.language_model.layers.20.mlp.experts.216", - "model.language_model.layers.20.mlp.experts.217", - "model.language_model.layers.20.mlp.experts.218", - "model.language_model.layers.20.mlp.experts.219", - "model.language_model.layers.20.mlp.experts.220", - "model.language_model.layers.20.mlp.experts.221", - "model.language_model.layers.20.mlp.experts.222", - "model.language_model.layers.20.mlp.experts.223", - "model.language_model.layers.20.mlp.experts.224", - "model.language_model.layers.20.mlp.experts.225", - "model.language_model.layers.20.mlp.experts.226", - "model.language_model.layers.20.mlp.experts.227", - "model.language_model.layers.20.mlp.experts.228", - "model.language_model.layers.20.mlp.experts.229", - "model.language_model.layers.20.mlp.experts.230", - "model.language_model.layers.20.mlp.experts.231", - "model.language_model.layers.20.mlp.experts.232", - "model.language_model.layers.20.mlp.experts.233", - "model.language_model.layers.20.mlp.experts.234", - "model.language_model.layers.20.mlp.experts.235", - "model.language_model.layers.20.mlp.experts.236", - "model.language_model.layers.20.mlp.experts.237", - "model.language_model.layers.20.mlp.experts.238", - "model.language_model.layers.20.mlp.experts.239", - "model.language_model.layers.20.mlp.experts.240", - "model.language_model.layers.20.mlp.experts.241", - "model.language_model.layers.20.mlp.experts.242", - "model.language_model.layers.20.mlp.experts.243", - "model.language_model.layers.20.mlp.experts.244", - "model.language_model.layers.20.mlp.experts.245", - "model.language_model.layers.20.mlp.experts.246", - "model.language_model.layers.20.mlp.experts.247", - "model.language_model.layers.20.mlp.experts.248", - "model.language_model.layers.20.mlp.experts.249", - "model.language_model.layers.20.mlp.experts.250", - "model.language_model.layers.20.mlp.experts.251", - "model.language_model.layers.20.mlp.experts.252", - "model.language_model.layers.20.mlp.experts.253", - "model.language_model.layers.20.mlp.experts.254", - "model.language_model.layers.20.mlp.experts.255", "model.language_model.layers.20.mlp.shared_expert_gate", - "model.language_model.layers.21.linear_attn", "model.language_model.layers.21.linear_attn.norm", "model.language_model.layers.21.linear_attn.in_proj_b", "model.language_model.layers.21.linear_attn.in_proj_a", "model.language_model.layers.21.mlp.gate", - "model.language_model.layers.21.mlp.experts.0", - "model.language_model.layers.21.mlp.experts.1", - "model.language_model.layers.21.mlp.experts.2", - "model.language_model.layers.21.mlp.experts.3", - "model.language_model.layers.21.mlp.experts.4", - "model.language_model.layers.21.mlp.experts.5", - "model.language_model.layers.21.mlp.experts.6", - "model.language_model.layers.21.mlp.experts.7", - "model.language_model.layers.21.mlp.experts.8", - "model.language_model.layers.21.mlp.experts.9", - "model.language_model.layers.21.mlp.experts.10", - "model.language_model.layers.21.mlp.experts.11", - "model.language_model.layers.21.mlp.experts.12", - "model.language_model.layers.21.mlp.experts.13", - "model.language_model.layers.21.mlp.experts.14", - "model.language_model.layers.21.mlp.experts.15", - "model.language_model.layers.21.mlp.experts.16", - "model.language_model.layers.21.mlp.experts.17", - "model.language_model.layers.21.mlp.experts.18", - "model.language_model.layers.21.mlp.experts.19", - "model.language_model.layers.21.mlp.experts.20", - "model.language_model.layers.21.mlp.experts.21", - "model.language_model.layers.21.mlp.experts.22", - "model.language_model.layers.21.mlp.experts.23", - "model.language_model.layers.21.mlp.experts.24", - "model.language_model.layers.21.mlp.experts.25", - "model.language_model.layers.21.mlp.experts.26", - "model.language_model.layers.21.mlp.experts.27", - "model.language_model.layers.21.mlp.experts.28", - "model.language_model.layers.21.mlp.experts.29", - "model.language_model.layers.21.mlp.experts.30", - "model.language_model.layers.21.mlp.experts.31", - "model.language_model.layers.21.mlp.experts.32", - "model.language_model.layers.21.mlp.experts.33", - "model.language_model.layers.21.mlp.experts.34", - "model.language_model.layers.21.mlp.experts.35", - "model.language_model.layers.21.mlp.experts.36", - "model.language_model.layers.21.mlp.experts.37", - "model.language_model.layers.21.mlp.experts.38", - "model.language_model.layers.21.mlp.experts.39", - "model.language_model.layers.21.mlp.experts.40", - "model.language_model.layers.21.mlp.experts.41", - "model.language_model.layers.21.mlp.experts.42", - "model.language_model.layers.21.mlp.experts.43", - "model.language_model.layers.21.mlp.experts.44", - "model.language_model.layers.21.mlp.experts.45", - "model.language_model.layers.21.mlp.experts.46", - "model.language_model.layers.21.mlp.experts.47", - "model.language_model.layers.21.mlp.experts.48", - "model.language_model.layers.21.mlp.experts.49", - "model.language_model.layers.21.mlp.experts.50", - "model.language_model.layers.21.mlp.experts.51", - "model.language_model.layers.21.mlp.experts.52", - "model.language_model.layers.21.mlp.experts.53", - "model.language_model.layers.21.mlp.experts.54", - "model.language_model.layers.21.mlp.experts.55", - "model.language_model.layers.21.mlp.experts.56", - "model.language_model.layers.21.mlp.experts.57", - "model.language_model.layers.21.mlp.experts.58", - "model.language_model.layers.21.mlp.experts.59", - "model.language_model.layers.21.mlp.experts.60", - "model.language_model.layers.21.mlp.experts.61", - "model.language_model.layers.21.mlp.experts.62", - "model.language_model.layers.21.mlp.experts.63", - "model.language_model.layers.21.mlp.experts.64", - "model.language_model.layers.21.mlp.experts.65", - "model.language_model.layers.21.mlp.experts.66", - "model.language_model.layers.21.mlp.experts.67", - "model.language_model.layers.21.mlp.experts.68", - "model.language_model.layers.21.mlp.experts.69", - "model.language_model.layers.21.mlp.experts.70", - "model.language_model.layers.21.mlp.experts.71", - "model.language_model.layers.21.mlp.experts.72", - "model.language_model.layers.21.mlp.experts.73", - "model.language_model.layers.21.mlp.experts.74", - "model.language_model.layers.21.mlp.experts.75", - "model.language_model.layers.21.mlp.experts.76", - "model.language_model.layers.21.mlp.experts.77", - "model.language_model.layers.21.mlp.experts.78", - "model.language_model.layers.21.mlp.experts.79", - "model.language_model.layers.21.mlp.experts.80", - "model.language_model.layers.21.mlp.experts.81", - "model.language_model.layers.21.mlp.experts.82", - "model.language_model.layers.21.mlp.experts.83", - "model.language_model.layers.21.mlp.experts.84", - "model.language_model.layers.21.mlp.experts.85", - "model.language_model.layers.21.mlp.experts.86", - "model.language_model.layers.21.mlp.experts.87", - "model.language_model.layers.21.mlp.experts.88", - "model.language_model.layers.21.mlp.experts.89", - "model.language_model.layers.21.mlp.experts.90", - "model.language_model.layers.21.mlp.experts.91", - "model.language_model.layers.21.mlp.experts.92", - "model.language_model.layers.21.mlp.experts.93", - "model.language_model.layers.21.mlp.experts.94", - "model.language_model.layers.21.mlp.experts.95", - "model.language_model.layers.21.mlp.experts.96", - "model.language_model.layers.21.mlp.experts.97", - "model.language_model.layers.21.mlp.experts.98", - "model.language_model.layers.21.mlp.experts.99", - "model.language_model.layers.21.mlp.experts.100", - "model.language_model.layers.21.mlp.experts.101", - "model.language_model.layers.21.mlp.experts.102", - "model.language_model.layers.21.mlp.experts.103", - "model.language_model.layers.21.mlp.experts.104", - "model.language_model.layers.21.mlp.experts.105", - "model.language_model.layers.21.mlp.experts.106", - "model.language_model.layers.21.mlp.experts.107", - "model.language_model.layers.21.mlp.experts.108", - "model.language_model.layers.21.mlp.experts.109", - "model.language_model.layers.21.mlp.experts.110", - "model.language_model.layers.21.mlp.experts.111", - "model.language_model.layers.21.mlp.experts.112", - "model.language_model.layers.21.mlp.experts.113", - "model.language_model.layers.21.mlp.experts.114", - "model.language_model.layers.21.mlp.experts.115", - "model.language_model.layers.21.mlp.experts.116", - "model.language_model.layers.21.mlp.experts.117", - "model.language_model.layers.21.mlp.experts.118", - "model.language_model.layers.21.mlp.experts.119", - "model.language_model.layers.21.mlp.experts.120", - "model.language_model.layers.21.mlp.experts.121", - "model.language_model.layers.21.mlp.experts.122", - "model.language_model.layers.21.mlp.experts.123", - "model.language_model.layers.21.mlp.experts.124", - "model.language_model.layers.21.mlp.experts.125", - "model.language_model.layers.21.mlp.experts.126", - "model.language_model.layers.21.mlp.experts.127", - "model.language_model.layers.21.mlp.experts.128", - "model.language_model.layers.21.mlp.experts.129", - "model.language_model.layers.21.mlp.experts.130", - "model.language_model.layers.21.mlp.experts.131", - "model.language_model.layers.21.mlp.experts.132", - "model.language_model.layers.21.mlp.experts.133", - "model.language_model.layers.21.mlp.experts.134", - "model.language_model.layers.21.mlp.experts.135", - "model.language_model.layers.21.mlp.experts.136", - "model.language_model.layers.21.mlp.experts.137", - "model.language_model.layers.21.mlp.experts.138", - "model.language_model.layers.21.mlp.experts.139", - "model.language_model.layers.21.mlp.experts.140", - "model.language_model.layers.21.mlp.experts.141", - "model.language_model.layers.21.mlp.experts.142", - "model.language_model.layers.21.mlp.experts.143", - "model.language_model.layers.21.mlp.experts.144", - "model.language_model.layers.21.mlp.experts.145", - "model.language_model.layers.21.mlp.experts.146", - "model.language_model.layers.21.mlp.experts.147", - "model.language_model.layers.21.mlp.experts.148", - "model.language_model.layers.21.mlp.experts.149", - "model.language_model.layers.21.mlp.experts.150", - "model.language_model.layers.21.mlp.experts.151", - "model.language_model.layers.21.mlp.experts.152", - "model.language_model.layers.21.mlp.experts.153", - "model.language_model.layers.21.mlp.experts.154", - "model.language_model.layers.21.mlp.experts.155", - "model.language_model.layers.21.mlp.experts.156", - "model.language_model.layers.21.mlp.experts.157", - "model.language_model.layers.21.mlp.experts.158", - "model.language_model.layers.21.mlp.experts.159", - "model.language_model.layers.21.mlp.experts.160", - "model.language_model.layers.21.mlp.experts.161", - "model.language_model.layers.21.mlp.experts.162", - "model.language_model.layers.21.mlp.experts.163", - "model.language_model.layers.21.mlp.experts.164", - "model.language_model.layers.21.mlp.experts.165", - "model.language_model.layers.21.mlp.experts.166", - "model.language_model.layers.21.mlp.experts.167", - "model.language_model.layers.21.mlp.experts.168", - "model.language_model.layers.21.mlp.experts.169", - "model.language_model.layers.21.mlp.experts.170", - "model.language_model.layers.21.mlp.experts.171", - "model.language_model.layers.21.mlp.experts.172", - "model.language_model.layers.21.mlp.experts.173", - "model.language_model.layers.21.mlp.experts.174", - "model.language_model.layers.21.mlp.experts.175", - "model.language_model.layers.21.mlp.experts.176", - "model.language_model.layers.21.mlp.experts.177", - "model.language_model.layers.21.mlp.experts.178", - "model.language_model.layers.21.mlp.experts.179", - "model.language_model.layers.21.mlp.experts.180", - "model.language_model.layers.21.mlp.experts.181", - "model.language_model.layers.21.mlp.experts.182", - "model.language_model.layers.21.mlp.experts.183", - "model.language_model.layers.21.mlp.experts.184", - "model.language_model.layers.21.mlp.experts.185", - "model.language_model.layers.21.mlp.experts.186", - "model.language_model.layers.21.mlp.experts.187", - "model.language_model.layers.21.mlp.experts.188", - "model.language_model.layers.21.mlp.experts.189", - "model.language_model.layers.21.mlp.experts.190", - "model.language_model.layers.21.mlp.experts.191", - "model.language_model.layers.21.mlp.experts.192", - "model.language_model.layers.21.mlp.experts.193", - "model.language_model.layers.21.mlp.experts.194", - "model.language_model.layers.21.mlp.experts.195", - "model.language_model.layers.21.mlp.experts.196", - "model.language_model.layers.21.mlp.experts.197", - "model.language_model.layers.21.mlp.experts.198", - "model.language_model.layers.21.mlp.experts.199", - "model.language_model.layers.21.mlp.experts.200", - "model.language_model.layers.21.mlp.experts.201", - "model.language_model.layers.21.mlp.experts.202", - "model.language_model.layers.21.mlp.experts.203", - "model.language_model.layers.21.mlp.experts.204", - "model.language_model.layers.21.mlp.experts.205", - "model.language_model.layers.21.mlp.experts.206", - "model.language_model.layers.21.mlp.experts.207", - "model.language_model.layers.21.mlp.experts.208", - "model.language_model.layers.21.mlp.experts.209", - "model.language_model.layers.21.mlp.experts.210", - "model.language_model.layers.21.mlp.experts.211", - "model.language_model.layers.21.mlp.experts.212", - "model.language_model.layers.21.mlp.experts.213", - "model.language_model.layers.21.mlp.experts.214", - "model.language_model.layers.21.mlp.experts.215", - "model.language_model.layers.21.mlp.experts.216", - "model.language_model.layers.21.mlp.experts.217", - "model.language_model.layers.21.mlp.experts.218", - "model.language_model.layers.21.mlp.experts.219", - "model.language_model.layers.21.mlp.experts.220", - "model.language_model.layers.21.mlp.experts.221", - "model.language_model.layers.21.mlp.experts.222", - "model.language_model.layers.21.mlp.experts.223", - "model.language_model.layers.21.mlp.experts.224", - "model.language_model.layers.21.mlp.experts.225", - "model.language_model.layers.21.mlp.experts.226", - "model.language_model.layers.21.mlp.experts.227", - "model.language_model.layers.21.mlp.experts.228", - "model.language_model.layers.21.mlp.experts.229", - "model.language_model.layers.21.mlp.experts.230", - "model.language_model.layers.21.mlp.experts.231", - "model.language_model.layers.21.mlp.experts.232", - "model.language_model.layers.21.mlp.experts.233", - "model.language_model.layers.21.mlp.experts.234", - "model.language_model.layers.21.mlp.experts.235", - "model.language_model.layers.21.mlp.experts.236", - "model.language_model.layers.21.mlp.experts.237", - "model.language_model.layers.21.mlp.experts.238", - "model.language_model.layers.21.mlp.experts.239", - "model.language_model.layers.21.mlp.experts.240", - "model.language_model.layers.21.mlp.experts.241", - "model.language_model.layers.21.mlp.experts.242", - "model.language_model.layers.21.mlp.experts.243", - "model.language_model.layers.21.mlp.experts.244", - "model.language_model.layers.21.mlp.experts.245", - "model.language_model.layers.21.mlp.experts.246", - "model.language_model.layers.21.mlp.experts.247", - "model.language_model.layers.21.mlp.experts.248", - "model.language_model.layers.21.mlp.experts.249", - "model.language_model.layers.21.mlp.experts.250", - "model.language_model.layers.21.mlp.experts.251", - "model.language_model.layers.21.mlp.experts.252", - "model.language_model.layers.21.mlp.experts.253", - "model.language_model.layers.21.mlp.experts.254", - "model.language_model.layers.21.mlp.experts.255", "model.language_model.layers.21.mlp.shared_expert_gate", - "model.language_model.layers.22.linear_attn", "model.language_model.layers.22.linear_attn.norm", "model.language_model.layers.22.linear_attn.in_proj_b", "model.language_model.layers.22.linear_attn.in_proj_a", "model.language_model.layers.22.mlp.gate", - "model.language_model.layers.22.mlp.experts.0", - "model.language_model.layers.22.mlp.experts.1", - "model.language_model.layers.22.mlp.experts.2", - "model.language_model.layers.22.mlp.experts.3", - "model.language_model.layers.22.mlp.experts.4", - "model.language_model.layers.22.mlp.experts.5", - "model.language_model.layers.22.mlp.experts.6", - "model.language_model.layers.22.mlp.experts.7", - "model.language_model.layers.22.mlp.experts.8", - "model.language_model.layers.22.mlp.experts.9", - "model.language_model.layers.22.mlp.experts.10", - "model.language_model.layers.22.mlp.experts.11", - "model.language_model.layers.22.mlp.experts.12", - "model.language_model.layers.22.mlp.experts.13", - "model.language_model.layers.22.mlp.experts.14", - "model.language_model.layers.22.mlp.experts.15", - "model.language_model.layers.22.mlp.experts.16", - "model.language_model.layers.22.mlp.experts.17", - "model.language_model.layers.22.mlp.experts.18", - "model.language_model.layers.22.mlp.experts.19", - "model.language_model.layers.22.mlp.experts.20", - "model.language_model.layers.22.mlp.experts.21", - "model.language_model.layers.22.mlp.experts.22", - "model.language_model.layers.22.mlp.experts.23", - "model.language_model.layers.22.mlp.experts.24", - "model.language_model.layers.22.mlp.experts.25", - "model.language_model.layers.22.mlp.experts.26", - "model.language_model.layers.22.mlp.experts.27", - "model.language_model.layers.22.mlp.experts.28", - "model.language_model.layers.22.mlp.experts.29", - "model.language_model.layers.22.mlp.experts.30", - "model.language_model.layers.22.mlp.experts.31", - "model.language_model.layers.22.mlp.experts.32", - "model.language_model.layers.22.mlp.experts.33", - "model.language_model.layers.22.mlp.experts.34", - "model.language_model.layers.22.mlp.experts.35", - "model.language_model.layers.22.mlp.experts.36", - "model.language_model.layers.22.mlp.experts.37", - "model.language_model.layers.22.mlp.experts.38", - "model.language_model.layers.22.mlp.experts.39", - "model.language_model.layers.22.mlp.experts.40", - "model.language_model.layers.22.mlp.experts.41", - "model.language_model.layers.22.mlp.experts.42", - "model.language_model.layers.22.mlp.experts.43", - "model.language_model.layers.22.mlp.experts.44", - "model.language_model.layers.22.mlp.experts.45", - "model.language_model.layers.22.mlp.experts.46", - "model.language_model.layers.22.mlp.experts.47", - "model.language_model.layers.22.mlp.experts.48", - "model.language_model.layers.22.mlp.experts.49", - "model.language_model.layers.22.mlp.experts.50", - "model.language_model.layers.22.mlp.experts.51", - "model.language_model.layers.22.mlp.experts.52", - "model.language_model.layers.22.mlp.experts.53", - "model.language_model.layers.22.mlp.experts.54", - "model.language_model.layers.22.mlp.experts.55", - "model.language_model.layers.22.mlp.experts.56", - "model.language_model.layers.22.mlp.experts.57", - "model.language_model.layers.22.mlp.experts.58", - "model.language_model.layers.22.mlp.experts.59", - "model.language_model.layers.22.mlp.experts.60", - "model.language_model.layers.22.mlp.experts.61", - "model.language_model.layers.22.mlp.experts.62", - "model.language_model.layers.22.mlp.experts.63", - "model.language_model.layers.22.mlp.experts.64", - "model.language_model.layers.22.mlp.experts.65", - "model.language_model.layers.22.mlp.experts.66", - "model.language_model.layers.22.mlp.experts.67", - "model.language_model.layers.22.mlp.experts.68", - "model.language_model.layers.22.mlp.experts.69", - "model.language_model.layers.22.mlp.experts.70", - "model.language_model.layers.22.mlp.experts.71", - "model.language_model.layers.22.mlp.experts.72", - "model.language_model.layers.22.mlp.experts.73", - "model.language_model.layers.22.mlp.experts.74", - "model.language_model.layers.22.mlp.experts.75", - "model.language_model.layers.22.mlp.experts.76", - "model.language_model.layers.22.mlp.experts.77", - "model.language_model.layers.22.mlp.experts.78", - "model.language_model.layers.22.mlp.experts.79", - "model.language_model.layers.22.mlp.experts.80", - "model.language_model.layers.22.mlp.experts.81", - "model.language_model.layers.22.mlp.experts.82", - "model.language_model.layers.22.mlp.experts.83", - "model.language_model.layers.22.mlp.experts.84", - "model.language_model.layers.22.mlp.experts.85", - "model.language_model.layers.22.mlp.experts.86", - "model.language_model.layers.22.mlp.experts.87", - "model.language_model.layers.22.mlp.experts.88", - "model.language_model.layers.22.mlp.experts.89", - "model.language_model.layers.22.mlp.experts.90", - "model.language_model.layers.22.mlp.experts.91", - "model.language_model.layers.22.mlp.experts.92", - "model.language_model.layers.22.mlp.experts.93", - "model.language_model.layers.22.mlp.experts.94", - "model.language_model.layers.22.mlp.experts.95", - "model.language_model.layers.22.mlp.experts.96", - "model.language_model.layers.22.mlp.experts.97", - "model.language_model.layers.22.mlp.experts.98", - "model.language_model.layers.22.mlp.experts.99", - "model.language_model.layers.22.mlp.experts.100", - "model.language_model.layers.22.mlp.experts.101", - "model.language_model.layers.22.mlp.experts.102", - "model.language_model.layers.22.mlp.experts.103", - "model.language_model.layers.22.mlp.experts.104", - "model.language_model.layers.22.mlp.experts.105", - "model.language_model.layers.22.mlp.experts.106", - "model.language_model.layers.22.mlp.experts.107", - "model.language_model.layers.22.mlp.experts.108", - "model.language_model.layers.22.mlp.experts.109", - "model.language_model.layers.22.mlp.experts.110", - "model.language_model.layers.22.mlp.experts.111", - "model.language_model.layers.22.mlp.experts.112", - "model.language_model.layers.22.mlp.experts.113", - "model.language_model.layers.22.mlp.experts.114", - "model.language_model.layers.22.mlp.experts.115", - "model.language_model.layers.22.mlp.experts.116", - "model.language_model.layers.22.mlp.experts.117", - "model.language_model.layers.22.mlp.experts.118", - "model.language_model.layers.22.mlp.experts.119", - "model.language_model.layers.22.mlp.experts.120", - "model.language_model.layers.22.mlp.experts.121", - "model.language_model.layers.22.mlp.experts.122", - "model.language_model.layers.22.mlp.experts.123", - "model.language_model.layers.22.mlp.experts.124", - "model.language_model.layers.22.mlp.experts.125", - "model.language_model.layers.22.mlp.experts.126", - "model.language_model.layers.22.mlp.experts.127", - "model.language_model.layers.22.mlp.experts.128", - "model.language_model.layers.22.mlp.experts.129", - "model.language_model.layers.22.mlp.experts.130", - "model.language_model.layers.22.mlp.experts.131", - "model.language_model.layers.22.mlp.experts.132", - "model.language_model.layers.22.mlp.experts.133", - "model.language_model.layers.22.mlp.experts.134", - "model.language_model.layers.22.mlp.experts.135", - "model.language_model.layers.22.mlp.experts.136", - "model.language_model.layers.22.mlp.experts.137", - "model.language_model.layers.22.mlp.experts.138", - "model.language_model.layers.22.mlp.experts.139", - "model.language_model.layers.22.mlp.experts.140", - "model.language_model.layers.22.mlp.experts.141", - "model.language_model.layers.22.mlp.experts.142", - "model.language_model.layers.22.mlp.experts.143", - "model.language_model.layers.22.mlp.experts.144", - "model.language_model.layers.22.mlp.experts.145", - "model.language_model.layers.22.mlp.experts.146", - "model.language_model.layers.22.mlp.experts.147", - "model.language_model.layers.22.mlp.experts.148", - "model.language_model.layers.22.mlp.experts.149", - "model.language_model.layers.22.mlp.experts.150", - "model.language_model.layers.22.mlp.experts.151", - "model.language_model.layers.22.mlp.experts.152", - "model.language_model.layers.22.mlp.experts.153", - "model.language_model.layers.22.mlp.experts.154", - "model.language_model.layers.22.mlp.experts.155", - "model.language_model.layers.22.mlp.experts.156", - "model.language_model.layers.22.mlp.experts.157", - "model.language_model.layers.22.mlp.experts.158", - "model.language_model.layers.22.mlp.experts.159", - "model.language_model.layers.22.mlp.experts.160", - "model.language_model.layers.22.mlp.experts.161", - "model.language_model.layers.22.mlp.experts.162", - "model.language_model.layers.22.mlp.experts.163", - "model.language_model.layers.22.mlp.experts.164", - "model.language_model.layers.22.mlp.experts.165", - "model.language_model.layers.22.mlp.experts.166", - "model.language_model.layers.22.mlp.experts.167", - "model.language_model.layers.22.mlp.experts.168", - "model.language_model.layers.22.mlp.experts.169", - "model.language_model.layers.22.mlp.experts.170", - "model.language_model.layers.22.mlp.experts.171", - "model.language_model.layers.22.mlp.experts.172", - "model.language_model.layers.22.mlp.experts.173", - "model.language_model.layers.22.mlp.experts.174", - "model.language_model.layers.22.mlp.experts.175", - "model.language_model.layers.22.mlp.experts.176", - "model.language_model.layers.22.mlp.experts.177", - "model.language_model.layers.22.mlp.experts.178", - "model.language_model.layers.22.mlp.experts.179", - "model.language_model.layers.22.mlp.experts.180", - "model.language_model.layers.22.mlp.experts.181", - "model.language_model.layers.22.mlp.experts.182", - "model.language_model.layers.22.mlp.experts.183", - "model.language_model.layers.22.mlp.experts.184", - "model.language_model.layers.22.mlp.experts.185", - "model.language_model.layers.22.mlp.experts.186", - "model.language_model.layers.22.mlp.experts.187", - "model.language_model.layers.22.mlp.experts.188", - "model.language_model.layers.22.mlp.experts.189", - "model.language_model.layers.22.mlp.experts.190", - "model.language_model.layers.22.mlp.experts.191", - "model.language_model.layers.22.mlp.experts.192", - "model.language_model.layers.22.mlp.experts.193", - "model.language_model.layers.22.mlp.experts.194", - "model.language_model.layers.22.mlp.experts.195", - "model.language_model.layers.22.mlp.experts.196", - "model.language_model.layers.22.mlp.experts.197", - "model.language_model.layers.22.mlp.experts.198", - "model.language_model.layers.22.mlp.experts.199", - "model.language_model.layers.22.mlp.experts.200", - "model.language_model.layers.22.mlp.experts.201", - "model.language_model.layers.22.mlp.experts.202", - "model.language_model.layers.22.mlp.experts.203", - "model.language_model.layers.22.mlp.experts.204", - "model.language_model.layers.22.mlp.experts.205", - "model.language_model.layers.22.mlp.experts.206", - "model.language_model.layers.22.mlp.experts.207", - "model.language_model.layers.22.mlp.experts.208", - "model.language_model.layers.22.mlp.experts.209", - "model.language_model.layers.22.mlp.experts.210", - "model.language_model.layers.22.mlp.experts.211", - "model.language_model.layers.22.mlp.experts.212", - "model.language_model.layers.22.mlp.experts.213", - "model.language_model.layers.22.mlp.experts.214", - "model.language_model.layers.22.mlp.experts.215", - "model.language_model.layers.22.mlp.experts.216", - "model.language_model.layers.22.mlp.experts.217", - "model.language_model.layers.22.mlp.experts.218", - "model.language_model.layers.22.mlp.experts.219", - "model.language_model.layers.22.mlp.experts.220", - "model.language_model.layers.22.mlp.experts.221", - "model.language_model.layers.22.mlp.experts.222", - "model.language_model.layers.22.mlp.experts.223", - "model.language_model.layers.22.mlp.experts.224", - "model.language_model.layers.22.mlp.experts.225", - "model.language_model.layers.22.mlp.experts.226", - "model.language_model.layers.22.mlp.experts.227", - "model.language_model.layers.22.mlp.experts.228", - "model.language_model.layers.22.mlp.experts.229", - "model.language_model.layers.22.mlp.experts.230", - "model.language_model.layers.22.mlp.experts.231", - "model.language_model.layers.22.mlp.experts.232", - "model.language_model.layers.22.mlp.experts.233", - "model.language_model.layers.22.mlp.experts.234", - "model.language_model.layers.22.mlp.experts.235", - "model.language_model.layers.22.mlp.experts.236", - "model.language_model.layers.22.mlp.experts.237", - "model.language_model.layers.22.mlp.experts.238", - "model.language_model.layers.22.mlp.experts.239", - "model.language_model.layers.22.mlp.experts.240", - "model.language_model.layers.22.mlp.experts.241", - "model.language_model.layers.22.mlp.experts.242", - "model.language_model.layers.22.mlp.experts.243", - "model.language_model.layers.22.mlp.experts.244", - "model.language_model.layers.22.mlp.experts.245", - "model.language_model.layers.22.mlp.experts.246", - "model.language_model.layers.22.mlp.experts.247", - "model.language_model.layers.22.mlp.experts.248", - "model.language_model.layers.22.mlp.experts.249", - "model.language_model.layers.22.mlp.experts.250", - "model.language_model.layers.22.mlp.experts.251", - "model.language_model.layers.22.mlp.experts.252", - "model.language_model.layers.22.mlp.experts.253", - "model.language_model.layers.22.mlp.experts.254", - "model.language_model.layers.22.mlp.experts.255", "model.language_model.layers.22.mlp.shared_expert_gate", "model.language_model.layers.23.mlp.gate", - "model.language_model.layers.23.mlp.experts.0", - "model.language_model.layers.23.mlp.experts.1", - "model.language_model.layers.23.mlp.experts.2", - "model.language_model.layers.23.mlp.experts.3", - "model.language_model.layers.23.mlp.experts.4", - "model.language_model.layers.23.mlp.experts.5", - "model.language_model.layers.23.mlp.experts.6", - "model.language_model.layers.23.mlp.experts.7", - "model.language_model.layers.23.mlp.experts.8", - "model.language_model.layers.23.mlp.experts.9", - "model.language_model.layers.23.mlp.experts.10", - "model.language_model.layers.23.mlp.experts.11", - "model.language_model.layers.23.mlp.experts.12", - "model.language_model.layers.23.mlp.experts.13", - "model.language_model.layers.23.mlp.experts.14", - "model.language_model.layers.23.mlp.experts.15", - "model.language_model.layers.23.mlp.experts.16", - "model.language_model.layers.23.mlp.experts.17", - "model.language_model.layers.23.mlp.experts.18", - "model.language_model.layers.23.mlp.experts.19", - "model.language_model.layers.23.mlp.experts.20", - "model.language_model.layers.23.mlp.experts.21", - "model.language_model.layers.23.mlp.experts.22", - "model.language_model.layers.23.mlp.experts.23", - "model.language_model.layers.23.mlp.experts.24", - "model.language_model.layers.23.mlp.experts.25", - "model.language_model.layers.23.mlp.experts.26", - "model.language_model.layers.23.mlp.experts.27", - "model.language_model.layers.23.mlp.experts.28", - "model.language_model.layers.23.mlp.experts.29", - "model.language_model.layers.23.mlp.experts.30", - "model.language_model.layers.23.mlp.experts.31", - "model.language_model.layers.23.mlp.experts.32", - "model.language_model.layers.23.mlp.experts.33", - "model.language_model.layers.23.mlp.experts.34", - "model.language_model.layers.23.mlp.experts.35", - "model.language_model.layers.23.mlp.experts.36", - "model.language_model.layers.23.mlp.experts.37", - "model.language_model.layers.23.mlp.experts.38", - "model.language_model.layers.23.mlp.experts.39", - "model.language_model.layers.23.mlp.experts.40", - "model.language_model.layers.23.mlp.experts.41", - "model.language_model.layers.23.mlp.experts.42", - "model.language_model.layers.23.mlp.experts.43", - "model.language_model.layers.23.mlp.experts.44", - "model.language_model.layers.23.mlp.experts.45", - "model.language_model.layers.23.mlp.experts.46", - "model.language_model.layers.23.mlp.experts.47", - "model.language_model.layers.23.mlp.experts.48", - "model.language_model.layers.23.mlp.experts.49", - "model.language_model.layers.23.mlp.experts.50", - "model.language_model.layers.23.mlp.experts.51", - "model.language_model.layers.23.mlp.experts.52", - "model.language_model.layers.23.mlp.experts.53", - "model.language_model.layers.23.mlp.experts.54", - "model.language_model.layers.23.mlp.experts.55", - "model.language_model.layers.23.mlp.experts.56", - "model.language_model.layers.23.mlp.experts.57", - "model.language_model.layers.23.mlp.experts.58", - "model.language_model.layers.23.mlp.experts.59", - "model.language_model.layers.23.mlp.experts.60", - "model.language_model.layers.23.mlp.experts.61", - "model.language_model.layers.23.mlp.experts.62", - "model.language_model.layers.23.mlp.experts.63", - "model.language_model.layers.23.mlp.experts.64", - "model.language_model.layers.23.mlp.experts.65", - "model.language_model.layers.23.mlp.experts.66", - "model.language_model.layers.23.mlp.experts.67", - "model.language_model.layers.23.mlp.experts.68", - "model.language_model.layers.23.mlp.experts.69", - "model.language_model.layers.23.mlp.experts.70", - "model.language_model.layers.23.mlp.experts.71", - "model.language_model.layers.23.mlp.experts.72", - "model.language_model.layers.23.mlp.experts.73", - "model.language_model.layers.23.mlp.experts.74", - "model.language_model.layers.23.mlp.experts.75", - "model.language_model.layers.23.mlp.experts.76", - "model.language_model.layers.23.mlp.experts.77", - "model.language_model.layers.23.mlp.experts.78", - "model.language_model.layers.23.mlp.experts.79", - "model.language_model.layers.23.mlp.experts.80", - "model.language_model.layers.23.mlp.experts.81", - "model.language_model.layers.23.mlp.experts.82", - "model.language_model.layers.23.mlp.experts.83", - "model.language_model.layers.23.mlp.experts.84", - "model.language_model.layers.23.mlp.experts.85", - "model.language_model.layers.23.mlp.experts.86", - "model.language_model.layers.23.mlp.experts.87", - "model.language_model.layers.23.mlp.experts.88", - "model.language_model.layers.23.mlp.experts.89", - "model.language_model.layers.23.mlp.experts.90", - "model.language_model.layers.23.mlp.experts.91", - "model.language_model.layers.23.mlp.experts.92", - "model.language_model.layers.23.mlp.experts.93", - "model.language_model.layers.23.mlp.experts.94", - "model.language_model.layers.23.mlp.experts.95", - "model.language_model.layers.23.mlp.experts.96", - "model.language_model.layers.23.mlp.experts.97", - "model.language_model.layers.23.mlp.experts.98", - "model.language_model.layers.23.mlp.experts.99", - "model.language_model.layers.23.mlp.experts.100", - "model.language_model.layers.23.mlp.experts.101", - "model.language_model.layers.23.mlp.experts.102", - "model.language_model.layers.23.mlp.experts.103", - "model.language_model.layers.23.mlp.experts.104", - "model.language_model.layers.23.mlp.experts.105", - "model.language_model.layers.23.mlp.experts.106", - "model.language_model.layers.23.mlp.experts.107", - "model.language_model.layers.23.mlp.experts.108", - "model.language_model.layers.23.mlp.experts.109", - "model.language_model.layers.23.mlp.experts.110", - "model.language_model.layers.23.mlp.experts.111", - "model.language_model.layers.23.mlp.experts.112", - "model.language_model.layers.23.mlp.experts.113", - "model.language_model.layers.23.mlp.experts.114", - "model.language_model.layers.23.mlp.experts.115", - "model.language_model.layers.23.mlp.experts.116", - "model.language_model.layers.23.mlp.experts.117", - "model.language_model.layers.23.mlp.experts.118", - "model.language_model.layers.23.mlp.experts.119", - "model.language_model.layers.23.mlp.experts.120", - "model.language_model.layers.23.mlp.experts.121", - "model.language_model.layers.23.mlp.experts.122", - "model.language_model.layers.23.mlp.experts.123", - "model.language_model.layers.23.mlp.experts.124", - "model.language_model.layers.23.mlp.experts.125", - "model.language_model.layers.23.mlp.experts.126", - "model.language_model.layers.23.mlp.experts.127", - "model.language_model.layers.23.mlp.experts.128", - "model.language_model.layers.23.mlp.experts.129", - "model.language_model.layers.23.mlp.experts.130", - "model.language_model.layers.23.mlp.experts.131", - "model.language_model.layers.23.mlp.experts.132", - "model.language_model.layers.23.mlp.experts.133", - "model.language_model.layers.23.mlp.experts.134", - "model.language_model.layers.23.mlp.experts.135", - "model.language_model.layers.23.mlp.experts.136", - "model.language_model.layers.23.mlp.experts.137", - "model.language_model.layers.23.mlp.experts.138", - "model.language_model.layers.23.mlp.experts.139", - "model.language_model.layers.23.mlp.experts.140", - "model.language_model.layers.23.mlp.experts.141", - "model.language_model.layers.23.mlp.experts.142", - "model.language_model.layers.23.mlp.experts.143", - "model.language_model.layers.23.mlp.experts.144", - "model.language_model.layers.23.mlp.experts.145", - "model.language_model.layers.23.mlp.experts.146", - "model.language_model.layers.23.mlp.experts.147", - "model.language_model.layers.23.mlp.experts.148", - "model.language_model.layers.23.mlp.experts.149", - "model.language_model.layers.23.mlp.experts.150", - "model.language_model.layers.23.mlp.experts.151", - "model.language_model.layers.23.mlp.experts.152", - "model.language_model.layers.23.mlp.experts.153", - "model.language_model.layers.23.mlp.experts.154", - "model.language_model.layers.23.mlp.experts.155", - "model.language_model.layers.23.mlp.experts.156", - "model.language_model.layers.23.mlp.experts.157", - "model.language_model.layers.23.mlp.experts.158", - "model.language_model.layers.23.mlp.experts.159", - "model.language_model.layers.23.mlp.experts.160", - "model.language_model.layers.23.mlp.experts.161", - "model.language_model.layers.23.mlp.experts.162", - "model.language_model.layers.23.mlp.experts.163", - "model.language_model.layers.23.mlp.experts.164", - "model.language_model.layers.23.mlp.experts.165", - "model.language_model.layers.23.mlp.experts.166", - "model.language_model.layers.23.mlp.experts.167", - "model.language_model.layers.23.mlp.experts.168", - "model.language_model.layers.23.mlp.experts.169", - "model.language_model.layers.23.mlp.experts.170", - "model.language_model.layers.23.mlp.experts.171", - "model.language_model.layers.23.mlp.experts.172", - "model.language_model.layers.23.mlp.experts.173", - "model.language_model.layers.23.mlp.experts.174", - "model.language_model.layers.23.mlp.experts.175", - "model.language_model.layers.23.mlp.experts.176", - "model.language_model.layers.23.mlp.experts.177", - "model.language_model.layers.23.mlp.experts.178", - "model.language_model.layers.23.mlp.experts.179", - "model.language_model.layers.23.mlp.experts.180", - "model.language_model.layers.23.mlp.experts.181", - "model.language_model.layers.23.mlp.experts.182", - "model.language_model.layers.23.mlp.experts.183", - "model.language_model.layers.23.mlp.experts.184", - "model.language_model.layers.23.mlp.experts.185", - "model.language_model.layers.23.mlp.experts.186", - "model.language_model.layers.23.mlp.experts.187", - "model.language_model.layers.23.mlp.experts.188", - "model.language_model.layers.23.mlp.experts.189", - "model.language_model.layers.23.mlp.experts.190", - "model.language_model.layers.23.mlp.experts.191", - "model.language_model.layers.23.mlp.experts.192", - "model.language_model.layers.23.mlp.experts.193", - "model.language_model.layers.23.mlp.experts.194", - "model.language_model.layers.23.mlp.experts.195", - "model.language_model.layers.23.mlp.experts.196", - "model.language_model.layers.23.mlp.experts.197", - "model.language_model.layers.23.mlp.experts.198", - "model.language_model.layers.23.mlp.experts.199", - "model.language_model.layers.23.mlp.experts.200", - "model.language_model.layers.23.mlp.experts.201", - "model.language_model.layers.23.mlp.experts.202", - "model.language_model.layers.23.mlp.experts.203", - "model.language_model.layers.23.mlp.experts.204", - "model.language_model.layers.23.mlp.experts.205", - "model.language_model.layers.23.mlp.experts.206", - "model.language_model.layers.23.mlp.experts.207", - "model.language_model.layers.23.mlp.experts.208", - "model.language_model.layers.23.mlp.experts.209", - "model.language_model.layers.23.mlp.experts.210", - "model.language_model.layers.23.mlp.experts.211", - "model.language_model.layers.23.mlp.experts.212", - "model.language_model.layers.23.mlp.experts.213", - "model.language_model.layers.23.mlp.experts.214", - "model.language_model.layers.23.mlp.experts.215", - "model.language_model.layers.23.mlp.experts.216", - "model.language_model.layers.23.mlp.experts.217", - "model.language_model.layers.23.mlp.experts.218", - "model.language_model.layers.23.mlp.experts.219", - "model.language_model.layers.23.mlp.experts.220", - "model.language_model.layers.23.mlp.experts.221", - "model.language_model.layers.23.mlp.experts.222", - "model.language_model.layers.23.mlp.experts.223", - "model.language_model.layers.23.mlp.experts.224", - "model.language_model.layers.23.mlp.experts.225", - "model.language_model.layers.23.mlp.experts.226", - "model.language_model.layers.23.mlp.experts.227", - "model.language_model.layers.23.mlp.experts.228", - "model.language_model.layers.23.mlp.experts.229", - "model.language_model.layers.23.mlp.experts.230", - "model.language_model.layers.23.mlp.experts.231", - "model.language_model.layers.23.mlp.experts.232", - "model.language_model.layers.23.mlp.experts.233", - "model.language_model.layers.23.mlp.experts.234", - "model.language_model.layers.23.mlp.experts.235", - "model.language_model.layers.23.mlp.experts.236", - "model.language_model.layers.23.mlp.experts.237", - "model.language_model.layers.23.mlp.experts.238", - "model.language_model.layers.23.mlp.experts.239", - "model.language_model.layers.23.mlp.experts.240", - "model.language_model.layers.23.mlp.experts.241", - "model.language_model.layers.23.mlp.experts.242", - "model.language_model.layers.23.mlp.experts.243", - "model.language_model.layers.23.mlp.experts.244", - "model.language_model.layers.23.mlp.experts.245", - "model.language_model.layers.23.mlp.experts.246", - "model.language_model.layers.23.mlp.experts.247", - "model.language_model.layers.23.mlp.experts.248", - "model.language_model.layers.23.mlp.experts.249", - "model.language_model.layers.23.mlp.experts.250", - "model.language_model.layers.23.mlp.experts.251", - "model.language_model.layers.23.mlp.experts.252", - "model.language_model.layers.23.mlp.experts.253", - "model.language_model.layers.23.mlp.experts.254", - "model.language_model.layers.23.mlp.experts.255", "model.language_model.layers.23.mlp.shared_expert_gate", - "model.language_model.layers.24.linear_attn", "model.language_model.layers.24.linear_attn.norm", "model.language_model.layers.24.linear_attn.in_proj_b", "model.language_model.layers.24.linear_attn.in_proj_a", "model.language_model.layers.24.mlp.gate", - "model.language_model.layers.24.mlp.experts.0", - "model.language_model.layers.24.mlp.experts.1", - "model.language_model.layers.24.mlp.experts.2", - "model.language_model.layers.24.mlp.experts.3", - "model.language_model.layers.24.mlp.experts.4", - "model.language_model.layers.24.mlp.experts.5", - "model.language_model.layers.24.mlp.experts.6", - "model.language_model.layers.24.mlp.experts.7", - "model.language_model.layers.24.mlp.experts.8", - "model.language_model.layers.24.mlp.experts.9", - "model.language_model.layers.24.mlp.experts.10", - "model.language_model.layers.24.mlp.experts.11", - "model.language_model.layers.24.mlp.experts.12", - "model.language_model.layers.24.mlp.experts.13", - "model.language_model.layers.24.mlp.experts.14", - "model.language_model.layers.24.mlp.experts.15", - "model.language_model.layers.24.mlp.experts.16", - "model.language_model.layers.24.mlp.experts.17", - "model.language_model.layers.24.mlp.experts.18", - "model.language_model.layers.24.mlp.experts.19", - "model.language_model.layers.24.mlp.experts.20", - "model.language_model.layers.24.mlp.experts.21", - "model.language_model.layers.24.mlp.experts.22", - "model.language_model.layers.24.mlp.experts.23", - "model.language_model.layers.24.mlp.experts.24", - "model.language_model.layers.24.mlp.experts.25", - "model.language_model.layers.24.mlp.experts.26", - "model.language_model.layers.24.mlp.experts.27", - "model.language_model.layers.24.mlp.experts.28", - "model.language_model.layers.24.mlp.experts.29", - "model.language_model.layers.24.mlp.experts.30", - "model.language_model.layers.24.mlp.experts.31", - "model.language_model.layers.24.mlp.experts.32", - "model.language_model.layers.24.mlp.experts.33", - "model.language_model.layers.24.mlp.experts.34", - "model.language_model.layers.24.mlp.experts.35", - "model.language_model.layers.24.mlp.experts.36", - "model.language_model.layers.24.mlp.experts.37", - "model.language_model.layers.24.mlp.experts.38", - "model.language_model.layers.24.mlp.experts.39", - "model.language_model.layers.24.mlp.experts.40", - "model.language_model.layers.24.mlp.experts.41", - "model.language_model.layers.24.mlp.experts.42", - "model.language_model.layers.24.mlp.experts.43", - "model.language_model.layers.24.mlp.experts.44", - "model.language_model.layers.24.mlp.experts.45", - "model.language_model.layers.24.mlp.experts.46", - "model.language_model.layers.24.mlp.experts.47", - "model.language_model.layers.24.mlp.experts.48", - "model.language_model.layers.24.mlp.experts.49", - "model.language_model.layers.24.mlp.experts.50", - "model.language_model.layers.24.mlp.experts.51", - "model.language_model.layers.24.mlp.experts.52", - "model.language_model.layers.24.mlp.experts.53", - "model.language_model.layers.24.mlp.experts.54", - "model.language_model.layers.24.mlp.experts.55", - "model.language_model.layers.24.mlp.experts.56", - "model.language_model.layers.24.mlp.experts.57", - "model.language_model.layers.24.mlp.experts.58", - "model.language_model.layers.24.mlp.experts.59", - "model.language_model.layers.24.mlp.experts.60", - "model.language_model.layers.24.mlp.experts.61", - "model.language_model.layers.24.mlp.experts.62", - "model.language_model.layers.24.mlp.experts.63", - "model.language_model.layers.24.mlp.experts.64", - "model.language_model.layers.24.mlp.experts.65", - "model.language_model.layers.24.mlp.experts.66", - "model.language_model.layers.24.mlp.experts.67", - "model.language_model.layers.24.mlp.experts.68", - "model.language_model.layers.24.mlp.experts.69", - "model.language_model.layers.24.mlp.experts.70", - "model.language_model.layers.24.mlp.experts.71", - "model.language_model.layers.24.mlp.experts.72", - "model.language_model.layers.24.mlp.experts.73", - "model.language_model.layers.24.mlp.experts.74", - "model.language_model.layers.24.mlp.experts.75", - "model.language_model.layers.24.mlp.experts.76", - "model.language_model.layers.24.mlp.experts.77", - "model.language_model.layers.24.mlp.experts.78", - "model.language_model.layers.24.mlp.experts.79", - "model.language_model.layers.24.mlp.experts.80", - "model.language_model.layers.24.mlp.experts.81", - "model.language_model.layers.24.mlp.experts.82", - "model.language_model.layers.24.mlp.experts.83", - "model.language_model.layers.24.mlp.experts.84", - "model.language_model.layers.24.mlp.experts.85", - "model.language_model.layers.24.mlp.experts.86", - "model.language_model.layers.24.mlp.experts.87", - "model.language_model.layers.24.mlp.experts.88", - "model.language_model.layers.24.mlp.experts.89", - "model.language_model.layers.24.mlp.experts.90", - "model.language_model.layers.24.mlp.experts.91", - "model.language_model.layers.24.mlp.experts.92", - "model.language_model.layers.24.mlp.experts.93", - "model.language_model.layers.24.mlp.experts.94", - "model.language_model.layers.24.mlp.experts.95", - "model.language_model.layers.24.mlp.experts.96", - "model.language_model.layers.24.mlp.experts.97", - "model.language_model.layers.24.mlp.experts.98", - "model.language_model.layers.24.mlp.experts.99", - "model.language_model.layers.24.mlp.experts.100", - "model.language_model.layers.24.mlp.experts.101", - "model.language_model.layers.24.mlp.experts.102", - "model.language_model.layers.24.mlp.experts.103", - "model.language_model.layers.24.mlp.experts.104", - "model.language_model.layers.24.mlp.experts.105", - "model.language_model.layers.24.mlp.experts.106", - "model.language_model.layers.24.mlp.experts.107", - "model.language_model.layers.24.mlp.experts.108", - "model.language_model.layers.24.mlp.experts.109", - "model.language_model.layers.24.mlp.experts.110", - "model.language_model.layers.24.mlp.experts.111", - "model.language_model.layers.24.mlp.experts.112", - "model.language_model.layers.24.mlp.experts.113", - "model.language_model.layers.24.mlp.experts.114", - "model.language_model.layers.24.mlp.experts.115", - "model.language_model.layers.24.mlp.experts.116", - "model.language_model.layers.24.mlp.experts.117", - "model.language_model.layers.24.mlp.experts.118", - "model.language_model.layers.24.mlp.experts.119", - "model.language_model.layers.24.mlp.experts.120", - "model.language_model.layers.24.mlp.experts.121", - "model.language_model.layers.24.mlp.experts.122", - "model.language_model.layers.24.mlp.experts.123", - "model.language_model.layers.24.mlp.experts.124", - "model.language_model.layers.24.mlp.experts.125", - "model.language_model.layers.24.mlp.experts.126", - "model.language_model.layers.24.mlp.experts.127", - "model.language_model.layers.24.mlp.experts.128", - "model.language_model.layers.24.mlp.experts.129", - "model.language_model.layers.24.mlp.experts.130", - "model.language_model.layers.24.mlp.experts.131", - "model.language_model.layers.24.mlp.experts.132", - "model.language_model.layers.24.mlp.experts.133", - "model.language_model.layers.24.mlp.experts.134", - "model.language_model.layers.24.mlp.experts.135", - "model.language_model.layers.24.mlp.experts.136", - "model.language_model.layers.24.mlp.experts.137", - "model.language_model.layers.24.mlp.experts.138", - "model.language_model.layers.24.mlp.experts.139", - "model.language_model.layers.24.mlp.experts.140", - "model.language_model.layers.24.mlp.experts.141", - "model.language_model.layers.24.mlp.experts.142", - "model.language_model.layers.24.mlp.experts.143", - "model.language_model.layers.24.mlp.experts.144", - "model.language_model.layers.24.mlp.experts.145", - "model.language_model.layers.24.mlp.experts.146", - "model.language_model.layers.24.mlp.experts.147", - "model.language_model.layers.24.mlp.experts.148", - "model.language_model.layers.24.mlp.experts.149", - "model.language_model.layers.24.mlp.experts.150", - "model.language_model.layers.24.mlp.experts.151", - "model.language_model.layers.24.mlp.experts.152", - "model.language_model.layers.24.mlp.experts.153", - "model.language_model.layers.24.mlp.experts.154", - "model.language_model.layers.24.mlp.experts.155", - "model.language_model.layers.24.mlp.experts.156", - "model.language_model.layers.24.mlp.experts.157", - "model.language_model.layers.24.mlp.experts.158", - "model.language_model.layers.24.mlp.experts.159", - "model.language_model.layers.24.mlp.experts.160", - "model.language_model.layers.24.mlp.experts.161", - "model.language_model.layers.24.mlp.experts.162", - "model.language_model.layers.24.mlp.experts.163", - "model.language_model.layers.24.mlp.experts.164", - "model.language_model.layers.24.mlp.experts.165", - "model.language_model.layers.24.mlp.experts.166", - "model.language_model.layers.24.mlp.experts.167", - "model.language_model.layers.24.mlp.experts.168", - "model.language_model.layers.24.mlp.experts.169", - "model.language_model.layers.24.mlp.experts.170", - "model.language_model.layers.24.mlp.experts.171", - "model.language_model.layers.24.mlp.experts.172", - "model.language_model.layers.24.mlp.experts.173", - "model.language_model.layers.24.mlp.experts.174", - "model.language_model.layers.24.mlp.experts.175", - "model.language_model.layers.24.mlp.experts.176", - "model.language_model.layers.24.mlp.experts.177", - "model.language_model.layers.24.mlp.experts.178", - "model.language_model.layers.24.mlp.experts.179", - "model.language_model.layers.24.mlp.experts.180", - "model.language_model.layers.24.mlp.experts.181", - "model.language_model.layers.24.mlp.experts.182", - "model.language_model.layers.24.mlp.experts.183", - "model.language_model.layers.24.mlp.experts.184", - "model.language_model.layers.24.mlp.experts.185", - "model.language_model.layers.24.mlp.experts.186", - "model.language_model.layers.24.mlp.experts.187", - "model.language_model.layers.24.mlp.experts.188", - "model.language_model.layers.24.mlp.experts.189", - "model.language_model.layers.24.mlp.experts.190", - "model.language_model.layers.24.mlp.experts.191", - "model.language_model.layers.24.mlp.experts.192", - "model.language_model.layers.24.mlp.experts.193", - "model.language_model.layers.24.mlp.experts.194", - "model.language_model.layers.24.mlp.experts.195", - "model.language_model.layers.24.mlp.experts.196", - "model.language_model.layers.24.mlp.experts.197", - "model.language_model.layers.24.mlp.experts.198", - "model.language_model.layers.24.mlp.experts.199", - "model.language_model.layers.24.mlp.experts.200", - "model.language_model.layers.24.mlp.experts.201", - "model.language_model.layers.24.mlp.experts.202", - "model.language_model.layers.24.mlp.experts.203", - "model.language_model.layers.24.mlp.experts.204", - "model.language_model.layers.24.mlp.experts.205", - "model.language_model.layers.24.mlp.experts.206", - "model.language_model.layers.24.mlp.experts.207", - "model.language_model.layers.24.mlp.experts.208", - "model.language_model.layers.24.mlp.experts.209", - "model.language_model.layers.24.mlp.experts.210", - "model.language_model.layers.24.mlp.experts.211", - "model.language_model.layers.24.mlp.experts.212", - "model.language_model.layers.24.mlp.experts.213", - "model.language_model.layers.24.mlp.experts.214", - "model.language_model.layers.24.mlp.experts.215", - "model.language_model.layers.24.mlp.experts.216", - "model.language_model.layers.24.mlp.experts.217", - "model.language_model.layers.24.mlp.experts.218", - "model.language_model.layers.24.mlp.experts.219", - "model.language_model.layers.24.mlp.experts.220", - "model.language_model.layers.24.mlp.experts.221", - "model.language_model.layers.24.mlp.experts.222", - "model.language_model.layers.24.mlp.experts.223", - "model.language_model.layers.24.mlp.experts.224", - "model.language_model.layers.24.mlp.experts.225", - "model.language_model.layers.24.mlp.experts.226", - "model.language_model.layers.24.mlp.experts.227", - "model.language_model.layers.24.mlp.experts.228", - "model.language_model.layers.24.mlp.experts.229", - "model.language_model.layers.24.mlp.experts.230", - "model.language_model.layers.24.mlp.experts.231", - "model.language_model.layers.24.mlp.experts.232", - "model.language_model.layers.24.mlp.experts.233", - "model.language_model.layers.24.mlp.experts.234", - "model.language_model.layers.24.mlp.experts.235", - "model.language_model.layers.24.mlp.experts.236", - "model.language_model.layers.24.mlp.experts.237", - "model.language_model.layers.24.mlp.experts.238", - "model.language_model.layers.24.mlp.experts.239", - "model.language_model.layers.24.mlp.experts.240", - "model.language_model.layers.24.mlp.experts.241", - "model.language_model.layers.24.mlp.experts.242", - "model.language_model.layers.24.mlp.experts.243", - "model.language_model.layers.24.mlp.experts.244", - "model.language_model.layers.24.mlp.experts.245", - "model.language_model.layers.24.mlp.experts.246", - "model.language_model.layers.24.mlp.experts.247", - "model.language_model.layers.24.mlp.experts.248", - "model.language_model.layers.24.mlp.experts.249", - "model.language_model.layers.24.mlp.experts.250", - "model.language_model.layers.24.mlp.experts.251", - "model.language_model.layers.24.mlp.experts.252", - "model.language_model.layers.24.mlp.experts.253", - "model.language_model.layers.24.mlp.experts.254", - "model.language_model.layers.24.mlp.experts.255", "model.language_model.layers.24.mlp.shared_expert_gate", - "model.language_model.layers.25.linear_attn", "model.language_model.layers.25.linear_attn.norm", "model.language_model.layers.25.linear_attn.in_proj_b", "model.language_model.layers.25.linear_attn.in_proj_a", "model.language_model.layers.25.mlp.gate", - "model.language_model.layers.25.mlp.experts.0", - "model.language_model.layers.25.mlp.experts.1", - "model.language_model.layers.25.mlp.experts.2", - "model.language_model.layers.25.mlp.experts.3", - "model.language_model.layers.25.mlp.experts.4", - "model.language_model.layers.25.mlp.experts.5", - "model.language_model.layers.25.mlp.experts.6", - "model.language_model.layers.25.mlp.experts.7", - "model.language_model.layers.25.mlp.experts.8", - "model.language_model.layers.25.mlp.experts.9", - "model.language_model.layers.25.mlp.experts.10", - "model.language_model.layers.25.mlp.experts.11", - "model.language_model.layers.25.mlp.experts.12", - "model.language_model.layers.25.mlp.experts.13", - "model.language_model.layers.25.mlp.experts.14", - "model.language_model.layers.25.mlp.experts.15", - "model.language_model.layers.25.mlp.experts.16", - "model.language_model.layers.25.mlp.experts.17", - "model.language_model.layers.25.mlp.experts.18", - "model.language_model.layers.25.mlp.experts.19", - "model.language_model.layers.25.mlp.experts.20", - "model.language_model.layers.25.mlp.experts.21", - "model.language_model.layers.25.mlp.experts.22", - "model.language_model.layers.25.mlp.experts.23", - "model.language_model.layers.25.mlp.experts.24", - "model.language_model.layers.25.mlp.experts.25", - "model.language_model.layers.25.mlp.experts.26", - "model.language_model.layers.25.mlp.experts.27", - "model.language_model.layers.25.mlp.experts.28", - "model.language_model.layers.25.mlp.experts.29", - "model.language_model.layers.25.mlp.experts.30", - "model.language_model.layers.25.mlp.experts.31", - "model.language_model.layers.25.mlp.experts.32", - "model.language_model.layers.25.mlp.experts.33", - "model.language_model.layers.25.mlp.experts.34", - "model.language_model.layers.25.mlp.experts.35", - "model.language_model.layers.25.mlp.experts.36", - "model.language_model.layers.25.mlp.experts.37", - "model.language_model.layers.25.mlp.experts.38", - "model.language_model.layers.25.mlp.experts.39", - "model.language_model.layers.25.mlp.experts.40", - "model.language_model.layers.25.mlp.experts.41", - "model.language_model.layers.25.mlp.experts.42", - "model.language_model.layers.25.mlp.experts.43", - "model.language_model.layers.25.mlp.experts.44", - "model.language_model.layers.25.mlp.experts.45", - "model.language_model.layers.25.mlp.experts.46", - "model.language_model.layers.25.mlp.experts.47", - "model.language_model.layers.25.mlp.experts.48", - "model.language_model.layers.25.mlp.experts.49", - "model.language_model.layers.25.mlp.experts.50", - "model.language_model.layers.25.mlp.experts.51", - "model.language_model.layers.25.mlp.experts.52", - "model.language_model.layers.25.mlp.experts.53", - "model.language_model.layers.25.mlp.experts.54", - "model.language_model.layers.25.mlp.experts.55", - "model.language_model.layers.25.mlp.experts.56", - "model.language_model.layers.25.mlp.experts.57", - "model.language_model.layers.25.mlp.experts.58", - "model.language_model.layers.25.mlp.experts.59", - "model.language_model.layers.25.mlp.experts.60", - "model.language_model.layers.25.mlp.experts.61", - "model.language_model.layers.25.mlp.experts.62", - "model.language_model.layers.25.mlp.experts.63", - "model.language_model.layers.25.mlp.experts.64", - "model.language_model.layers.25.mlp.experts.65", - "model.language_model.layers.25.mlp.experts.66", - "model.language_model.layers.25.mlp.experts.67", - "model.language_model.layers.25.mlp.experts.68", - "model.language_model.layers.25.mlp.experts.69", - "model.language_model.layers.25.mlp.experts.70", - "model.language_model.layers.25.mlp.experts.71", - "model.language_model.layers.25.mlp.experts.72", - "model.language_model.layers.25.mlp.experts.73", - "model.language_model.layers.25.mlp.experts.74", - "model.language_model.layers.25.mlp.experts.75", - "model.language_model.layers.25.mlp.experts.76", - "model.language_model.layers.25.mlp.experts.77", - "model.language_model.layers.25.mlp.experts.78", - "model.language_model.layers.25.mlp.experts.79", - "model.language_model.layers.25.mlp.experts.80", - "model.language_model.layers.25.mlp.experts.81", - "model.language_model.layers.25.mlp.experts.82", - "model.language_model.layers.25.mlp.experts.83", - "model.language_model.layers.25.mlp.experts.84", - "model.language_model.layers.25.mlp.experts.85", - "model.language_model.layers.25.mlp.experts.86", - "model.language_model.layers.25.mlp.experts.87", - "model.language_model.layers.25.mlp.experts.88", - "model.language_model.layers.25.mlp.experts.89", - "model.language_model.layers.25.mlp.experts.90", - "model.language_model.layers.25.mlp.experts.91", - "model.language_model.layers.25.mlp.experts.92", - "model.language_model.layers.25.mlp.experts.93", - "model.language_model.layers.25.mlp.experts.94", - "model.language_model.layers.25.mlp.experts.95", - "model.language_model.layers.25.mlp.experts.96", - "model.language_model.layers.25.mlp.experts.97", - "model.language_model.layers.25.mlp.experts.98", - "model.language_model.layers.25.mlp.experts.99", - "model.language_model.layers.25.mlp.experts.100", - "model.language_model.layers.25.mlp.experts.101", - "model.language_model.layers.25.mlp.experts.102", - "model.language_model.layers.25.mlp.experts.103", - "model.language_model.layers.25.mlp.experts.104", - "model.language_model.layers.25.mlp.experts.105", - "model.language_model.layers.25.mlp.experts.106", - "model.language_model.layers.25.mlp.experts.107", - "model.language_model.layers.25.mlp.experts.108", - "model.language_model.layers.25.mlp.experts.109", - "model.language_model.layers.25.mlp.experts.110", - "model.language_model.layers.25.mlp.experts.111", - "model.language_model.layers.25.mlp.experts.112", - "model.language_model.layers.25.mlp.experts.113", - "model.language_model.layers.25.mlp.experts.114", - "model.language_model.layers.25.mlp.experts.115", - "model.language_model.layers.25.mlp.experts.116", - "model.language_model.layers.25.mlp.experts.117", - "model.language_model.layers.25.mlp.experts.118", - "model.language_model.layers.25.mlp.experts.119", - "model.language_model.layers.25.mlp.experts.120", - "model.language_model.layers.25.mlp.experts.121", - "model.language_model.layers.25.mlp.experts.122", - "model.language_model.layers.25.mlp.experts.123", - "model.language_model.layers.25.mlp.experts.124", - "model.language_model.layers.25.mlp.experts.125", - "model.language_model.layers.25.mlp.experts.126", - "model.language_model.layers.25.mlp.experts.127", - "model.language_model.layers.25.mlp.experts.128", - "model.language_model.layers.25.mlp.experts.129", - "model.language_model.layers.25.mlp.experts.130", - "model.language_model.layers.25.mlp.experts.131", - "model.language_model.layers.25.mlp.experts.132", - "model.language_model.layers.25.mlp.experts.133", - "model.language_model.layers.25.mlp.experts.134", - "model.language_model.layers.25.mlp.experts.135", - "model.language_model.layers.25.mlp.experts.136", - "model.language_model.layers.25.mlp.experts.137", - "model.language_model.layers.25.mlp.experts.138", - "model.language_model.layers.25.mlp.experts.139", - "model.language_model.layers.25.mlp.experts.140", - "model.language_model.layers.25.mlp.experts.141", - "model.language_model.layers.25.mlp.experts.142", - "model.language_model.layers.25.mlp.experts.143", - "model.language_model.layers.25.mlp.experts.144", - "model.language_model.layers.25.mlp.experts.145", - "model.language_model.layers.25.mlp.experts.146", - "model.language_model.layers.25.mlp.experts.147", - "model.language_model.layers.25.mlp.experts.148", - "model.language_model.layers.25.mlp.experts.149", - "model.language_model.layers.25.mlp.experts.150", - "model.language_model.layers.25.mlp.experts.151", - "model.language_model.layers.25.mlp.experts.152", - "model.language_model.layers.25.mlp.experts.153", - "model.language_model.layers.25.mlp.experts.154", - "model.language_model.layers.25.mlp.experts.155", - "model.language_model.layers.25.mlp.experts.156", - "model.language_model.layers.25.mlp.experts.157", - "model.language_model.layers.25.mlp.experts.158", - "model.language_model.layers.25.mlp.experts.159", - "model.language_model.layers.25.mlp.experts.160", - "model.language_model.layers.25.mlp.experts.161", - "model.language_model.layers.25.mlp.experts.162", - "model.language_model.layers.25.mlp.experts.163", - "model.language_model.layers.25.mlp.experts.164", - "model.language_model.layers.25.mlp.experts.165", - "model.language_model.layers.25.mlp.experts.166", - "model.language_model.layers.25.mlp.experts.167", - "model.language_model.layers.25.mlp.experts.168", - "model.language_model.layers.25.mlp.experts.169", - "model.language_model.layers.25.mlp.experts.170", - "model.language_model.layers.25.mlp.experts.171", - "model.language_model.layers.25.mlp.experts.172", - "model.language_model.layers.25.mlp.experts.173", - "model.language_model.layers.25.mlp.experts.174", - "model.language_model.layers.25.mlp.experts.175", - "model.language_model.layers.25.mlp.experts.176", - "model.language_model.layers.25.mlp.experts.177", - "model.language_model.layers.25.mlp.experts.178", - "model.language_model.layers.25.mlp.experts.179", - "model.language_model.layers.25.mlp.experts.180", - "model.language_model.layers.25.mlp.experts.181", - "model.language_model.layers.25.mlp.experts.182", - "model.language_model.layers.25.mlp.experts.183", - "model.language_model.layers.25.mlp.experts.184", - "model.language_model.layers.25.mlp.experts.185", - "model.language_model.layers.25.mlp.experts.186", - "model.language_model.layers.25.mlp.experts.187", - "model.language_model.layers.25.mlp.experts.188", - "model.language_model.layers.25.mlp.experts.189", - "model.language_model.layers.25.mlp.experts.190", - "model.language_model.layers.25.mlp.experts.191", - "model.language_model.layers.25.mlp.experts.192", - "model.language_model.layers.25.mlp.experts.193", - "model.language_model.layers.25.mlp.experts.194", - "model.language_model.layers.25.mlp.experts.195", - "model.language_model.layers.25.mlp.experts.196", - "model.language_model.layers.25.mlp.experts.197", - "model.language_model.layers.25.mlp.experts.198", - "model.language_model.layers.25.mlp.experts.199", - "model.language_model.layers.25.mlp.experts.200", - "model.language_model.layers.25.mlp.experts.201", - "model.language_model.layers.25.mlp.experts.202", - "model.language_model.layers.25.mlp.experts.203", - "model.language_model.layers.25.mlp.experts.204", - "model.language_model.layers.25.mlp.experts.205", - "model.language_model.layers.25.mlp.experts.206", - "model.language_model.layers.25.mlp.experts.207", - "model.language_model.layers.25.mlp.experts.208", - "model.language_model.layers.25.mlp.experts.209", - "model.language_model.layers.25.mlp.experts.210", - "model.language_model.layers.25.mlp.experts.211", - "model.language_model.layers.25.mlp.experts.212", - "model.language_model.layers.25.mlp.experts.213", - "model.language_model.layers.25.mlp.experts.214", - "model.language_model.layers.25.mlp.experts.215", - "model.language_model.layers.25.mlp.experts.216", - "model.language_model.layers.25.mlp.experts.217", - "model.language_model.layers.25.mlp.experts.218", - "model.language_model.layers.25.mlp.experts.219", - "model.language_model.layers.25.mlp.experts.220", - "model.language_model.layers.25.mlp.experts.221", - "model.language_model.layers.25.mlp.experts.222", - "model.language_model.layers.25.mlp.experts.223", - "model.language_model.layers.25.mlp.experts.224", - "model.language_model.layers.25.mlp.experts.225", - "model.language_model.layers.25.mlp.experts.226", - "model.language_model.layers.25.mlp.experts.227", - "model.language_model.layers.25.mlp.experts.228", - "model.language_model.layers.25.mlp.experts.229", - "model.language_model.layers.25.mlp.experts.230", - "model.language_model.layers.25.mlp.experts.231", - "model.language_model.layers.25.mlp.experts.232", - "model.language_model.layers.25.mlp.experts.233", - "model.language_model.layers.25.mlp.experts.234", - "model.language_model.layers.25.mlp.experts.235", - "model.language_model.layers.25.mlp.experts.236", - "model.language_model.layers.25.mlp.experts.237", - "model.language_model.layers.25.mlp.experts.238", - "model.language_model.layers.25.mlp.experts.239", - "model.language_model.layers.25.mlp.experts.240", - "model.language_model.layers.25.mlp.experts.241", - "model.language_model.layers.25.mlp.experts.242", - "model.language_model.layers.25.mlp.experts.243", - "model.language_model.layers.25.mlp.experts.244", - "model.language_model.layers.25.mlp.experts.245", - "model.language_model.layers.25.mlp.experts.246", - "model.language_model.layers.25.mlp.experts.247", - "model.language_model.layers.25.mlp.experts.248", - "model.language_model.layers.25.mlp.experts.249", - "model.language_model.layers.25.mlp.experts.250", - "model.language_model.layers.25.mlp.experts.251", - "model.language_model.layers.25.mlp.experts.252", - "model.language_model.layers.25.mlp.experts.253", - "model.language_model.layers.25.mlp.experts.254", - "model.language_model.layers.25.mlp.experts.255", "model.language_model.layers.25.mlp.shared_expert_gate", - "model.language_model.layers.26.linear_attn", "model.language_model.layers.26.linear_attn.norm", "model.language_model.layers.26.linear_attn.in_proj_b", "model.language_model.layers.26.linear_attn.in_proj_a", "model.language_model.layers.26.mlp.gate", - "model.language_model.layers.26.mlp.experts.0", - "model.language_model.layers.26.mlp.experts.1", - "model.language_model.layers.26.mlp.experts.2", - "model.language_model.layers.26.mlp.experts.3", - "model.language_model.layers.26.mlp.experts.4", - "model.language_model.layers.26.mlp.experts.5", - "model.language_model.layers.26.mlp.experts.6", - "model.language_model.layers.26.mlp.experts.7", - "model.language_model.layers.26.mlp.experts.8", - "model.language_model.layers.26.mlp.experts.9", - "model.language_model.layers.26.mlp.experts.10", - "model.language_model.layers.26.mlp.experts.11", - "model.language_model.layers.26.mlp.experts.12", - "model.language_model.layers.26.mlp.experts.13", - "model.language_model.layers.26.mlp.experts.14", - "model.language_model.layers.26.mlp.experts.15", - "model.language_model.layers.26.mlp.experts.16", - "model.language_model.layers.26.mlp.experts.17", - "model.language_model.layers.26.mlp.experts.18", - "model.language_model.layers.26.mlp.experts.19", - "model.language_model.layers.26.mlp.experts.20", - "model.language_model.layers.26.mlp.experts.21", - "model.language_model.layers.26.mlp.experts.22", - "model.language_model.layers.26.mlp.experts.23", - "model.language_model.layers.26.mlp.experts.24", - "model.language_model.layers.26.mlp.experts.25", - "model.language_model.layers.26.mlp.experts.26", - "model.language_model.layers.26.mlp.experts.27", - "model.language_model.layers.26.mlp.experts.28", - "model.language_model.layers.26.mlp.experts.29", - "model.language_model.layers.26.mlp.experts.30", - "model.language_model.layers.26.mlp.experts.31", - "model.language_model.layers.26.mlp.experts.32", - "model.language_model.layers.26.mlp.experts.33", - "model.language_model.layers.26.mlp.experts.34", - "model.language_model.layers.26.mlp.experts.35", - "model.language_model.layers.26.mlp.experts.36", - "model.language_model.layers.26.mlp.experts.37", - "model.language_model.layers.26.mlp.experts.38", - "model.language_model.layers.26.mlp.experts.39", - "model.language_model.layers.26.mlp.experts.40", - "model.language_model.layers.26.mlp.experts.41", - "model.language_model.layers.26.mlp.experts.42", - "model.language_model.layers.26.mlp.experts.43", - "model.language_model.layers.26.mlp.experts.44", - "model.language_model.layers.26.mlp.experts.45", - "model.language_model.layers.26.mlp.experts.46", - "model.language_model.layers.26.mlp.experts.47", - "model.language_model.layers.26.mlp.experts.48", - "model.language_model.layers.26.mlp.experts.49", - "model.language_model.layers.26.mlp.experts.50", - "model.language_model.layers.26.mlp.experts.51", - "model.language_model.layers.26.mlp.experts.52", - "model.language_model.layers.26.mlp.experts.53", - "model.language_model.layers.26.mlp.experts.54", - "model.language_model.layers.26.mlp.experts.55", - "model.language_model.layers.26.mlp.experts.56", - "model.language_model.layers.26.mlp.experts.57", - "model.language_model.layers.26.mlp.experts.58", - "model.language_model.layers.26.mlp.experts.59", - "model.language_model.layers.26.mlp.experts.60", - "model.language_model.layers.26.mlp.experts.61", - "model.language_model.layers.26.mlp.experts.62", - "model.language_model.layers.26.mlp.experts.63", - "model.language_model.layers.26.mlp.experts.64", - "model.language_model.layers.26.mlp.experts.65", - "model.language_model.layers.26.mlp.experts.66", - "model.language_model.layers.26.mlp.experts.67", - "model.language_model.layers.26.mlp.experts.68", - "model.language_model.layers.26.mlp.experts.69", - "model.language_model.layers.26.mlp.experts.70", - "model.language_model.layers.26.mlp.experts.71", - "model.language_model.layers.26.mlp.experts.72", - "model.language_model.layers.26.mlp.experts.73", - "model.language_model.layers.26.mlp.experts.74", - "model.language_model.layers.26.mlp.experts.75", - "model.language_model.layers.26.mlp.experts.76", - "model.language_model.layers.26.mlp.experts.77", - "model.language_model.layers.26.mlp.experts.78", - "model.language_model.layers.26.mlp.experts.79", - "model.language_model.layers.26.mlp.experts.80", - "model.language_model.layers.26.mlp.experts.81", - "model.language_model.layers.26.mlp.experts.82", - "model.language_model.layers.26.mlp.experts.83", - "model.language_model.layers.26.mlp.experts.84", - "model.language_model.layers.26.mlp.experts.85", - "model.language_model.layers.26.mlp.experts.86", - "model.language_model.layers.26.mlp.experts.87", - "model.language_model.layers.26.mlp.experts.88", - "model.language_model.layers.26.mlp.experts.89", - "model.language_model.layers.26.mlp.experts.90", - "model.language_model.layers.26.mlp.experts.91", - "model.language_model.layers.26.mlp.experts.92", - "model.language_model.layers.26.mlp.experts.93", - "model.language_model.layers.26.mlp.experts.94", - "model.language_model.layers.26.mlp.experts.95", - "model.language_model.layers.26.mlp.experts.96", - "model.language_model.layers.26.mlp.experts.97", - "model.language_model.layers.26.mlp.experts.98", - "model.language_model.layers.26.mlp.experts.99", - "model.language_model.layers.26.mlp.experts.100", - "model.language_model.layers.26.mlp.experts.101", - "model.language_model.layers.26.mlp.experts.102", - "model.language_model.layers.26.mlp.experts.103", - "model.language_model.layers.26.mlp.experts.104", - "model.language_model.layers.26.mlp.experts.105", - "model.language_model.layers.26.mlp.experts.106", - "model.language_model.layers.26.mlp.experts.107", - "model.language_model.layers.26.mlp.experts.108", - "model.language_model.layers.26.mlp.experts.109", - "model.language_model.layers.26.mlp.experts.110", - "model.language_model.layers.26.mlp.experts.111", - "model.language_model.layers.26.mlp.experts.112", - "model.language_model.layers.26.mlp.experts.113", - "model.language_model.layers.26.mlp.experts.114", - "model.language_model.layers.26.mlp.experts.115", - "model.language_model.layers.26.mlp.experts.116", - "model.language_model.layers.26.mlp.experts.117", - "model.language_model.layers.26.mlp.experts.118", - "model.language_model.layers.26.mlp.experts.119", - "model.language_model.layers.26.mlp.experts.120", - "model.language_model.layers.26.mlp.experts.121", - "model.language_model.layers.26.mlp.experts.122", - "model.language_model.layers.26.mlp.experts.123", - "model.language_model.layers.26.mlp.experts.124", - "model.language_model.layers.26.mlp.experts.125", - "model.language_model.layers.26.mlp.experts.126", - "model.language_model.layers.26.mlp.experts.127", - "model.language_model.layers.26.mlp.experts.128", - "model.language_model.layers.26.mlp.experts.129", - "model.language_model.layers.26.mlp.experts.130", - "model.language_model.layers.26.mlp.experts.131", - "model.language_model.layers.26.mlp.experts.132", - "model.language_model.layers.26.mlp.experts.133", - "model.language_model.layers.26.mlp.experts.134", - "model.language_model.layers.26.mlp.experts.135", - "model.language_model.layers.26.mlp.experts.136", - "model.language_model.layers.26.mlp.experts.137", - "model.language_model.layers.26.mlp.experts.138", - "model.language_model.layers.26.mlp.experts.139", - "model.language_model.layers.26.mlp.experts.140", - "model.language_model.layers.26.mlp.experts.141", - "model.language_model.layers.26.mlp.experts.142", - "model.language_model.layers.26.mlp.experts.143", - "model.language_model.layers.26.mlp.experts.144", - "model.language_model.layers.26.mlp.experts.145", - "model.language_model.layers.26.mlp.experts.146", - "model.language_model.layers.26.mlp.experts.147", - "model.language_model.layers.26.mlp.experts.148", - "model.language_model.layers.26.mlp.experts.149", - "model.language_model.layers.26.mlp.experts.150", - "model.language_model.layers.26.mlp.experts.151", - "model.language_model.layers.26.mlp.experts.152", - "model.language_model.layers.26.mlp.experts.153", - "model.language_model.layers.26.mlp.experts.154", - "model.language_model.layers.26.mlp.experts.155", - "model.language_model.layers.26.mlp.experts.156", - "model.language_model.layers.26.mlp.experts.157", - "model.language_model.layers.26.mlp.experts.158", - "model.language_model.layers.26.mlp.experts.159", - "model.language_model.layers.26.mlp.experts.160", - "model.language_model.layers.26.mlp.experts.161", - "model.language_model.layers.26.mlp.experts.162", - "model.language_model.layers.26.mlp.experts.163", - "model.language_model.layers.26.mlp.experts.164", - "model.language_model.layers.26.mlp.experts.165", - "model.language_model.layers.26.mlp.experts.166", - "model.language_model.layers.26.mlp.experts.167", - "model.language_model.layers.26.mlp.experts.168", - "model.language_model.layers.26.mlp.experts.169", - "model.language_model.layers.26.mlp.experts.170", - "model.language_model.layers.26.mlp.experts.171", - "model.language_model.layers.26.mlp.experts.172", - "model.language_model.layers.26.mlp.experts.173", - "model.language_model.layers.26.mlp.experts.174", - "model.language_model.layers.26.mlp.experts.175", - "model.language_model.layers.26.mlp.experts.176", - "model.language_model.layers.26.mlp.experts.177", - "model.language_model.layers.26.mlp.experts.178", - "model.language_model.layers.26.mlp.experts.179", - "model.language_model.layers.26.mlp.experts.180", - "model.language_model.layers.26.mlp.experts.181", - "model.language_model.layers.26.mlp.experts.182", - "model.language_model.layers.26.mlp.experts.183", - "model.language_model.layers.26.mlp.experts.184", - "model.language_model.layers.26.mlp.experts.185", - "model.language_model.layers.26.mlp.experts.186", - "model.language_model.layers.26.mlp.experts.187", - "model.language_model.layers.26.mlp.experts.188", - "model.language_model.layers.26.mlp.experts.189", - "model.language_model.layers.26.mlp.experts.190", - "model.language_model.layers.26.mlp.experts.191", - "model.language_model.layers.26.mlp.experts.192", - "model.language_model.layers.26.mlp.experts.193", - "model.language_model.layers.26.mlp.experts.194", - "model.language_model.layers.26.mlp.experts.195", - "model.language_model.layers.26.mlp.experts.196", - "model.language_model.layers.26.mlp.experts.197", - "model.language_model.layers.26.mlp.experts.198", - "model.language_model.layers.26.mlp.experts.199", - "model.language_model.layers.26.mlp.experts.200", - "model.language_model.layers.26.mlp.experts.201", - "model.language_model.layers.26.mlp.experts.202", - "model.language_model.layers.26.mlp.experts.203", - "model.language_model.layers.26.mlp.experts.204", - "model.language_model.layers.26.mlp.experts.205", - "model.language_model.layers.26.mlp.experts.206", - "model.language_model.layers.26.mlp.experts.207", - "model.language_model.layers.26.mlp.experts.208", - "model.language_model.layers.26.mlp.experts.209", - "model.language_model.layers.26.mlp.experts.210", - "model.language_model.layers.26.mlp.experts.211", - "model.language_model.layers.26.mlp.experts.212", - "model.language_model.layers.26.mlp.experts.213", - "model.language_model.layers.26.mlp.experts.214", - "model.language_model.layers.26.mlp.experts.215", - "model.language_model.layers.26.mlp.experts.216", - "model.language_model.layers.26.mlp.experts.217", - "model.language_model.layers.26.mlp.experts.218", - "model.language_model.layers.26.mlp.experts.219", - "model.language_model.layers.26.mlp.experts.220", - "model.language_model.layers.26.mlp.experts.221", - "model.language_model.layers.26.mlp.experts.222", - "model.language_model.layers.26.mlp.experts.223", - "model.language_model.layers.26.mlp.experts.224", - "model.language_model.layers.26.mlp.experts.225", - "model.language_model.layers.26.mlp.experts.226", - "model.language_model.layers.26.mlp.experts.227", - "model.language_model.layers.26.mlp.experts.228", - "model.language_model.layers.26.mlp.experts.229", - "model.language_model.layers.26.mlp.experts.230", - "model.language_model.layers.26.mlp.experts.231", - "model.language_model.layers.26.mlp.experts.232", - "model.language_model.layers.26.mlp.experts.233", - "model.language_model.layers.26.mlp.experts.234", - "model.language_model.layers.26.mlp.experts.235", - "model.language_model.layers.26.mlp.experts.236", - "model.language_model.layers.26.mlp.experts.237", - "model.language_model.layers.26.mlp.experts.238", - "model.language_model.layers.26.mlp.experts.239", - "model.language_model.layers.26.mlp.experts.240", - "model.language_model.layers.26.mlp.experts.241", - "model.language_model.layers.26.mlp.experts.242", - "model.language_model.layers.26.mlp.experts.243", - "model.language_model.layers.26.mlp.experts.244", - "model.language_model.layers.26.mlp.experts.245", - "model.language_model.layers.26.mlp.experts.246", - "model.language_model.layers.26.mlp.experts.247", - "model.language_model.layers.26.mlp.experts.248", - "model.language_model.layers.26.mlp.experts.249", - "model.language_model.layers.26.mlp.experts.250", - "model.language_model.layers.26.mlp.experts.251", - "model.language_model.layers.26.mlp.experts.252", - "model.language_model.layers.26.mlp.experts.253", - "model.language_model.layers.26.mlp.experts.254", - "model.language_model.layers.26.mlp.experts.255", "model.language_model.layers.26.mlp.shared_expert_gate", "model.language_model.layers.27.mlp.gate", - "model.language_model.layers.27.mlp.experts.0", - "model.language_model.layers.27.mlp.experts.1", - "model.language_model.layers.27.mlp.experts.2", - "model.language_model.layers.27.mlp.experts.3", - "model.language_model.layers.27.mlp.experts.4", - "model.language_model.layers.27.mlp.experts.5", - "model.language_model.layers.27.mlp.experts.6", - "model.language_model.layers.27.mlp.experts.7", - "model.language_model.layers.27.mlp.experts.8", - "model.language_model.layers.27.mlp.experts.9", - "model.language_model.layers.27.mlp.experts.10", - "model.language_model.layers.27.mlp.experts.11", - "model.language_model.layers.27.mlp.experts.12", - "model.language_model.layers.27.mlp.experts.13", - "model.language_model.layers.27.mlp.experts.14", - "model.language_model.layers.27.mlp.experts.15", - "model.language_model.layers.27.mlp.experts.16", - "model.language_model.layers.27.mlp.experts.17", - "model.language_model.layers.27.mlp.experts.18", - "model.language_model.layers.27.mlp.experts.19", - "model.language_model.layers.27.mlp.experts.20", - "model.language_model.layers.27.mlp.experts.21", - "model.language_model.layers.27.mlp.experts.22", - "model.language_model.layers.27.mlp.experts.23", - "model.language_model.layers.27.mlp.experts.24", - "model.language_model.layers.27.mlp.experts.25", - "model.language_model.layers.27.mlp.experts.26", - "model.language_model.layers.27.mlp.experts.27", - "model.language_model.layers.27.mlp.experts.28", - "model.language_model.layers.27.mlp.experts.29", - "model.language_model.layers.27.mlp.experts.30", - "model.language_model.layers.27.mlp.experts.31", - "model.language_model.layers.27.mlp.experts.32", - "model.language_model.layers.27.mlp.experts.33", - "model.language_model.layers.27.mlp.experts.34", - "model.language_model.layers.27.mlp.experts.35", - "model.language_model.layers.27.mlp.experts.36", - "model.language_model.layers.27.mlp.experts.37", - "model.language_model.layers.27.mlp.experts.38", - "model.language_model.layers.27.mlp.experts.39", - "model.language_model.layers.27.mlp.experts.40", - "model.language_model.layers.27.mlp.experts.41", - "model.language_model.layers.27.mlp.experts.42", - "model.language_model.layers.27.mlp.experts.43", - "model.language_model.layers.27.mlp.experts.44", - "model.language_model.layers.27.mlp.experts.45", - "model.language_model.layers.27.mlp.experts.46", - "model.language_model.layers.27.mlp.experts.47", - "model.language_model.layers.27.mlp.experts.48", - "model.language_model.layers.27.mlp.experts.49", - "model.language_model.layers.27.mlp.experts.50", - "model.language_model.layers.27.mlp.experts.51", - "model.language_model.layers.27.mlp.experts.52", - "model.language_model.layers.27.mlp.experts.53", - "model.language_model.layers.27.mlp.experts.54", - "model.language_model.layers.27.mlp.experts.55", - "model.language_model.layers.27.mlp.experts.56", - "model.language_model.layers.27.mlp.experts.57", - "model.language_model.layers.27.mlp.experts.58", - "model.language_model.layers.27.mlp.experts.59", - "model.language_model.layers.27.mlp.experts.60", - "model.language_model.layers.27.mlp.experts.61", - "model.language_model.layers.27.mlp.experts.62", - "model.language_model.layers.27.mlp.experts.63", - "model.language_model.layers.27.mlp.experts.64", - "model.language_model.layers.27.mlp.experts.65", - "model.language_model.layers.27.mlp.experts.66", - "model.language_model.layers.27.mlp.experts.67", - "model.language_model.layers.27.mlp.experts.68", - "model.language_model.layers.27.mlp.experts.69", - "model.language_model.layers.27.mlp.experts.70", - "model.language_model.layers.27.mlp.experts.71", - "model.language_model.layers.27.mlp.experts.72", - "model.language_model.layers.27.mlp.experts.73", - "model.language_model.layers.27.mlp.experts.74", - "model.language_model.layers.27.mlp.experts.75", - "model.language_model.layers.27.mlp.experts.76", - "model.language_model.layers.27.mlp.experts.77", - "model.language_model.layers.27.mlp.experts.78", - "model.language_model.layers.27.mlp.experts.79", - "model.language_model.layers.27.mlp.experts.80", - "model.language_model.layers.27.mlp.experts.81", - "model.language_model.layers.27.mlp.experts.82", - "model.language_model.layers.27.mlp.experts.83", - "model.language_model.layers.27.mlp.experts.84", - "model.language_model.layers.27.mlp.experts.85", - "model.language_model.layers.27.mlp.experts.86", - "model.language_model.layers.27.mlp.experts.87", - "model.language_model.layers.27.mlp.experts.88", - "model.language_model.layers.27.mlp.experts.89", - "model.language_model.layers.27.mlp.experts.90", - "model.language_model.layers.27.mlp.experts.91", - "model.language_model.layers.27.mlp.experts.92", - "model.language_model.layers.27.mlp.experts.93", - "model.language_model.layers.27.mlp.experts.94", - "model.language_model.layers.27.mlp.experts.95", - "model.language_model.layers.27.mlp.experts.96", - "model.language_model.layers.27.mlp.experts.97", - "model.language_model.layers.27.mlp.experts.98", - "model.language_model.layers.27.mlp.experts.99", - "model.language_model.layers.27.mlp.experts.100", - "model.language_model.layers.27.mlp.experts.101", - "model.language_model.layers.27.mlp.experts.102", - "model.language_model.layers.27.mlp.experts.103", - "model.language_model.layers.27.mlp.experts.104", - "model.language_model.layers.27.mlp.experts.105", - "model.language_model.layers.27.mlp.experts.106", - "model.language_model.layers.27.mlp.experts.107", - "model.language_model.layers.27.mlp.experts.108", - "model.language_model.layers.27.mlp.experts.109", - "model.language_model.layers.27.mlp.experts.110", - "model.language_model.layers.27.mlp.experts.111", - "model.language_model.layers.27.mlp.experts.112", - "model.language_model.layers.27.mlp.experts.113", - "model.language_model.layers.27.mlp.experts.114", - "model.language_model.layers.27.mlp.experts.115", - "model.language_model.layers.27.mlp.experts.116", - "model.language_model.layers.27.mlp.experts.117", - "model.language_model.layers.27.mlp.experts.118", - "model.language_model.layers.27.mlp.experts.119", - "model.language_model.layers.27.mlp.experts.120", - "model.language_model.layers.27.mlp.experts.121", - "model.language_model.layers.27.mlp.experts.122", - "model.language_model.layers.27.mlp.experts.123", - "model.language_model.layers.27.mlp.experts.124", - "model.language_model.layers.27.mlp.experts.125", - "model.language_model.layers.27.mlp.experts.126", - "model.language_model.layers.27.mlp.experts.127", - "model.language_model.layers.27.mlp.experts.128", - "model.language_model.layers.27.mlp.experts.129", - "model.language_model.layers.27.mlp.experts.130", - "model.language_model.layers.27.mlp.experts.131", - "model.language_model.layers.27.mlp.experts.132", - "model.language_model.layers.27.mlp.experts.133", - "model.language_model.layers.27.mlp.experts.134", - "model.language_model.layers.27.mlp.experts.135", - "model.language_model.layers.27.mlp.experts.136", - "model.language_model.layers.27.mlp.experts.137", - "model.language_model.layers.27.mlp.experts.138", - "model.language_model.layers.27.mlp.experts.139", - "model.language_model.layers.27.mlp.experts.140", - "model.language_model.layers.27.mlp.experts.141", - "model.language_model.layers.27.mlp.experts.142", - "model.language_model.layers.27.mlp.experts.143", - "model.language_model.layers.27.mlp.experts.144", - "model.language_model.layers.27.mlp.experts.145", - "model.language_model.layers.27.mlp.experts.146", - "model.language_model.layers.27.mlp.experts.147", - "model.language_model.layers.27.mlp.experts.148", - "model.language_model.layers.27.mlp.experts.149", - "model.language_model.layers.27.mlp.experts.150", - "model.language_model.layers.27.mlp.experts.151", - "model.language_model.layers.27.mlp.experts.152", - "model.language_model.layers.27.mlp.experts.153", - "model.language_model.layers.27.mlp.experts.154", - "model.language_model.layers.27.mlp.experts.155", - "model.language_model.layers.27.mlp.experts.156", - "model.language_model.layers.27.mlp.experts.157", - "model.language_model.layers.27.mlp.experts.158", - "model.language_model.layers.27.mlp.experts.159", - "model.language_model.layers.27.mlp.experts.160", - "model.language_model.layers.27.mlp.experts.161", - "model.language_model.layers.27.mlp.experts.162", - "model.language_model.layers.27.mlp.experts.163", - "model.language_model.layers.27.mlp.experts.164", - "model.language_model.layers.27.mlp.experts.165", - "model.language_model.layers.27.mlp.experts.166", - "model.language_model.layers.27.mlp.experts.167", - "model.language_model.layers.27.mlp.experts.168", - "model.language_model.layers.27.mlp.experts.169", - "model.language_model.layers.27.mlp.experts.170", - "model.language_model.layers.27.mlp.experts.171", - "model.language_model.layers.27.mlp.experts.172", - "model.language_model.layers.27.mlp.experts.173", - "model.language_model.layers.27.mlp.experts.174", - "model.language_model.layers.27.mlp.experts.175", - "model.language_model.layers.27.mlp.experts.176", - "model.language_model.layers.27.mlp.experts.177", - "model.language_model.layers.27.mlp.experts.178", - "model.language_model.layers.27.mlp.experts.179", - "model.language_model.layers.27.mlp.experts.180", - "model.language_model.layers.27.mlp.experts.181", - "model.language_model.layers.27.mlp.experts.182", - "model.language_model.layers.27.mlp.experts.183", - "model.language_model.layers.27.mlp.experts.184", - "model.language_model.layers.27.mlp.experts.185", - "model.language_model.layers.27.mlp.experts.186", - "model.language_model.layers.27.mlp.experts.187", - "model.language_model.layers.27.mlp.experts.188", - "model.language_model.layers.27.mlp.experts.189", - "model.language_model.layers.27.mlp.experts.190", - "model.language_model.layers.27.mlp.experts.191", - "model.language_model.layers.27.mlp.experts.192", - "model.language_model.layers.27.mlp.experts.193", - "model.language_model.layers.27.mlp.experts.194", - "model.language_model.layers.27.mlp.experts.195", - "model.language_model.layers.27.mlp.experts.196", - "model.language_model.layers.27.mlp.experts.197", - "model.language_model.layers.27.mlp.experts.198", - "model.language_model.layers.27.mlp.experts.199", - "model.language_model.layers.27.mlp.experts.200", - "model.language_model.layers.27.mlp.experts.201", - "model.language_model.layers.27.mlp.experts.202", - "model.language_model.layers.27.mlp.experts.203", - "model.language_model.layers.27.mlp.experts.204", - "model.language_model.layers.27.mlp.experts.205", - "model.language_model.layers.27.mlp.experts.206", - "model.language_model.layers.27.mlp.experts.207", - "model.language_model.layers.27.mlp.experts.208", - "model.language_model.layers.27.mlp.experts.209", - "model.language_model.layers.27.mlp.experts.210", - "model.language_model.layers.27.mlp.experts.211", - "model.language_model.layers.27.mlp.experts.212", - "model.language_model.layers.27.mlp.experts.213", - "model.language_model.layers.27.mlp.experts.214", - "model.language_model.layers.27.mlp.experts.215", - "model.language_model.layers.27.mlp.experts.216", - "model.language_model.layers.27.mlp.experts.217", - "model.language_model.layers.27.mlp.experts.218", - "model.language_model.layers.27.mlp.experts.219", - "model.language_model.layers.27.mlp.experts.220", - "model.language_model.layers.27.mlp.experts.221", - "model.language_model.layers.27.mlp.experts.222", - "model.language_model.layers.27.mlp.experts.223", - "model.language_model.layers.27.mlp.experts.224", - "model.language_model.layers.27.mlp.experts.225", - "model.language_model.layers.27.mlp.experts.226", - "model.language_model.layers.27.mlp.experts.227", - "model.language_model.layers.27.mlp.experts.228", - "model.language_model.layers.27.mlp.experts.229", - "model.language_model.layers.27.mlp.experts.230", - "model.language_model.layers.27.mlp.experts.231", - "model.language_model.layers.27.mlp.experts.232", - "model.language_model.layers.27.mlp.experts.233", - "model.language_model.layers.27.mlp.experts.234", - "model.language_model.layers.27.mlp.experts.235", - "model.language_model.layers.27.mlp.experts.236", - "model.language_model.layers.27.mlp.experts.237", - "model.language_model.layers.27.mlp.experts.238", - "model.language_model.layers.27.mlp.experts.239", - "model.language_model.layers.27.mlp.experts.240", - "model.language_model.layers.27.mlp.experts.241", - "model.language_model.layers.27.mlp.experts.242", - "model.language_model.layers.27.mlp.experts.243", - "model.language_model.layers.27.mlp.experts.244", - "model.language_model.layers.27.mlp.experts.245", - "model.language_model.layers.27.mlp.experts.246", - "model.language_model.layers.27.mlp.experts.247", - "model.language_model.layers.27.mlp.experts.248", - "model.language_model.layers.27.mlp.experts.249", - "model.language_model.layers.27.mlp.experts.250", - "model.language_model.layers.27.mlp.experts.251", - "model.language_model.layers.27.mlp.experts.252", - "model.language_model.layers.27.mlp.experts.253", - "model.language_model.layers.27.mlp.experts.254", - "model.language_model.layers.27.mlp.experts.255", "model.language_model.layers.27.mlp.shared_expert_gate", - "model.language_model.layers.28.linear_attn", "model.language_model.layers.28.linear_attn.norm", "model.language_model.layers.28.linear_attn.in_proj_b", "model.language_model.layers.28.linear_attn.in_proj_a", "model.language_model.layers.28.mlp.gate", - "model.language_model.layers.28.mlp.experts.0", - "model.language_model.layers.28.mlp.experts.1", - "model.language_model.layers.28.mlp.experts.2", - "model.language_model.layers.28.mlp.experts.3", - "model.language_model.layers.28.mlp.experts.4", - "model.language_model.layers.28.mlp.experts.5", - "model.language_model.layers.28.mlp.experts.6", - "model.language_model.layers.28.mlp.experts.7", - "model.language_model.layers.28.mlp.experts.8", - "model.language_model.layers.28.mlp.experts.9", - "model.language_model.layers.28.mlp.experts.10", - "model.language_model.layers.28.mlp.experts.11", - "model.language_model.layers.28.mlp.experts.12", - "model.language_model.layers.28.mlp.experts.13", - "model.language_model.layers.28.mlp.experts.14", - "model.language_model.layers.28.mlp.experts.15", - "model.language_model.layers.28.mlp.experts.16", - "model.language_model.layers.28.mlp.experts.17", - "model.language_model.layers.28.mlp.experts.18", - "model.language_model.layers.28.mlp.experts.19", - "model.language_model.layers.28.mlp.experts.20", - "model.language_model.layers.28.mlp.experts.21", - "model.language_model.layers.28.mlp.experts.22", - "model.language_model.layers.28.mlp.experts.23", - "model.language_model.layers.28.mlp.experts.24", - "model.language_model.layers.28.mlp.experts.25", - "model.language_model.layers.28.mlp.experts.26", - "model.language_model.layers.28.mlp.experts.27", - "model.language_model.layers.28.mlp.experts.28", - "model.language_model.layers.28.mlp.experts.29", - "model.language_model.layers.28.mlp.experts.30", - "model.language_model.layers.28.mlp.experts.31", - "model.language_model.layers.28.mlp.experts.32", - "model.language_model.layers.28.mlp.experts.33", - "model.language_model.layers.28.mlp.experts.34", - "model.language_model.layers.28.mlp.experts.35", - "model.language_model.layers.28.mlp.experts.36", - "model.language_model.layers.28.mlp.experts.37", - "model.language_model.layers.28.mlp.experts.38", - "model.language_model.layers.28.mlp.experts.39", - "model.language_model.layers.28.mlp.experts.40", - "model.language_model.layers.28.mlp.experts.41", - "model.language_model.layers.28.mlp.experts.42", - "model.language_model.layers.28.mlp.experts.43", - "model.language_model.layers.28.mlp.experts.44", - "model.language_model.layers.28.mlp.experts.45", - "model.language_model.layers.28.mlp.experts.46", - "model.language_model.layers.28.mlp.experts.47", - "model.language_model.layers.28.mlp.experts.48", - "model.language_model.layers.28.mlp.experts.49", - "model.language_model.layers.28.mlp.experts.50", - "model.language_model.layers.28.mlp.experts.51", - "model.language_model.layers.28.mlp.experts.52", - "model.language_model.layers.28.mlp.experts.53", - "model.language_model.layers.28.mlp.experts.54", - "model.language_model.layers.28.mlp.experts.55", - "model.language_model.layers.28.mlp.experts.56", - "model.language_model.layers.28.mlp.experts.57", - "model.language_model.layers.28.mlp.experts.58", - "model.language_model.layers.28.mlp.experts.59", - "model.language_model.layers.28.mlp.experts.60", - "model.language_model.layers.28.mlp.experts.61", - "model.language_model.layers.28.mlp.experts.62", - "model.language_model.layers.28.mlp.experts.63", - "model.language_model.layers.28.mlp.experts.64", - "model.language_model.layers.28.mlp.experts.65", - "model.language_model.layers.28.mlp.experts.66", - "model.language_model.layers.28.mlp.experts.67", - "model.language_model.layers.28.mlp.experts.68", - "model.language_model.layers.28.mlp.experts.69", - "model.language_model.layers.28.mlp.experts.70", - "model.language_model.layers.28.mlp.experts.71", - "model.language_model.layers.28.mlp.experts.72", - "model.language_model.layers.28.mlp.experts.73", - "model.language_model.layers.28.mlp.experts.74", - "model.language_model.layers.28.mlp.experts.75", - "model.language_model.layers.28.mlp.experts.76", - "model.language_model.layers.28.mlp.experts.77", - "model.language_model.layers.28.mlp.experts.78", - "model.language_model.layers.28.mlp.experts.79", - "model.language_model.layers.28.mlp.experts.80", - "model.language_model.layers.28.mlp.experts.81", - "model.language_model.layers.28.mlp.experts.82", - "model.language_model.layers.28.mlp.experts.83", - "model.language_model.layers.28.mlp.experts.84", - "model.language_model.layers.28.mlp.experts.85", - "model.language_model.layers.28.mlp.experts.86", - "model.language_model.layers.28.mlp.experts.87", - "model.language_model.layers.28.mlp.experts.88", - "model.language_model.layers.28.mlp.experts.89", - "model.language_model.layers.28.mlp.experts.90", - "model.language_model.layers.28.mlp.experts.91", - "model.language_model.layers.28.mlp.experts.92", - "model.language_model.layers.28.mlp.experts.93", - "model.language_model.layers.28.mlp.experts.94", - "model.language_model.layers.28.mlp.experts.95", - "model.language_model.layers.28.mlp.experts.96", - "model.language_model.layers.28.mlp.experts.97", - "model.language_model.layers.28.mlp.experts.98", - "model.language_model.layers.28.mlp.experts.99", - "model.language_model.layers.28.mlp.experts.100", - "model.language_model.layers.28.mlp.experts.101", - "model.language_model.layers.28.mlp.experts.102", - "model.language_model.layers.28.mlp.experts.103", - "model.language_model.layers.28.mlp.experts.104", - "model.language_model.layers.28.mlp.experts.105", - "model.language_model.layers.28.mlp.experts.106", - "model.language_model.layers.28.mlp.experts.107", - "model.language_model.layers.28.mlp.experts.108", - "model.language_model.layers.28.mlp.experts.109", - "model.language_model.layers.28.mlp.experts.110", - "model.language_model.layers.28.mlp.experts.111", - "model.language_model.layers.28.mlp.experts.112", - "model.language_model.layers.28.mlp.experts.113", - "model.language_model.layers.28.mlp.experts.114", - "model.language_model.layers.28.mlp.experts.115", - "model.language_model.layers.28.mlp.experts.116", - "model.language_model.layers.28.mlp.experts.117", - "model.language_model.layers.28.mlp.experts.118", - "model.language_model.layers.28.mlp.experts.119", - "model.language_model.layers.28.mlp.experts.120", - "model.language_model.layers.28.mlp.experts.121", - "model.language_model.layers.28.mlp.experts.122", - "model.language_model.layers.28.mlp.experts.123", - "model.language_model.layers.28.mlp.experts.124", - "model.language_model.layers.28.mlp.experts.125", - "model.language_model.layers.28.mlp.experts.126", - "model.language_model.layers.28.mlp.experts.127", - "model.language_model.layers.28.mlp.experts.128", - "model.language_model.layers.28.mlp.experts.129", - "model.language_model.layers.28.mlp.experts.130", - "model.language_model.layers.28.mlp.experts.131", - "model.language_model.layers.28.mlp.experts.132", - "model.language_model.layers.28.mlp.experts.133", - "model.language_model.layers.28.mlp.experts.134", - "model.language_model.layers.28.mlp.experts.135", - "model.language_model.layers.28.mlp.experts.136", - "model.language_model.layers.28.mlp.experts.137", - "model.language_model.layers.28.mlp.experts.138", - "model.language_model.layers.28.mlp.experts.139", - "model.language_model.layers.28.mlp.experts.140", - "model.language_model.layers.28.mlp.experts.141", - "model.language_model.layers.28.mlp.experts.142", - "model.language_model.layers.28.mlp.experts.143", - "model.language_model.layers.28.mlp.experts.144", - "model.language_model.layers.28.mlp.experts.145", - "model.language_model.layers.28.mlp.experts.146", - "model.language_model.layers.28.mlp.experts.147", - "model.language_model.layers.28.mlp.experts.148", - "model.language_model.layers.28.mlp.experts.149", - "model.language_model.layers.28.mlp.experts.150", - "model.language_model.layers.28.mlp.experts.151", - "model.language_model.layers.28.mlp.experts.152", - "model.language_model.layers.28.mlp.experts.153", - "model.language_model.layers.28.mlp.experts.154", - "model.language_model.layers.28.mlp.experts.155", - "model.language_model.layers.28.mlp.experts.156", - "model.language_model.layers.28.mlp.experts.157", - "model.language_model.layers.28.mlp.experts.158", - "model.language_model.layers.28.mlp.experts.159", - "model.language_model.layers.28.mlp.experts.160", - "model.language_model.layers.28.mlp.experts.161", - "model.language_model.layers.28.mlp.experts.162", - "model.language_model.layers.28.mlp.experts.163", - "model.language_model.layers.28.mlp.experts.164", - "model.language_model.layers.28.mlp.experts.165", - "model.language_model.layers.28.mlp.experts.166", - "model.language_model.layers.28.mlp.experts.167", - "model.language_model.layers.28.mlp.experts.168", - "model.language_model.layers.28.mlp.experts.169", - "model.language_model.layers.28.mlp.experts.170", - "model.language_model.layers.28.mlp.experts.171", - "model.language_model.layers.28.mlp.experts.172", - "model.language_model.layers.28.mlp.experts.173", - "model.language_model.layers.28.mlp.experts.174", - "model.language_model.layers.28.mlp.experts.175", - "model.language_model.layers.28.mlp.experts.176", - "model.language_model.layers.28.mlp.experts.177", - "model.language_model.layers.28.mlp.experts.178", - "model.language_model.layers.28.mlp.experts.179", - "model.language_model.layers.28.mlp.experts.180", - "model.language_model.layers.28.mlp.experts.181", - "model.language_model.layers.28.mlp.experts.182", - "model.language_model.layers.28.mlp.experts.183", - "model.language_model.layers.28.mlp.experts.184", - "model.language_model.layers.28.mlp.experts.185", - "model.language_model.layers.28.mlp.experts.186", - "model.language_model.layers.28.mlp.experts.187", - "model.language_model.layers.28.mlp.experts.188", - "model.language_model.layers.28.mlp.experts.189", - "model.language_model.layers.28.mlp.experts.190", - "model.language_model.layers.28.mlp.experts.191", - "model.language_model.layers.28.mlp.experts.192", - "model.language_model.layers.28.mlp.experts.193", - "model.language_model.layers.28.mlp.experts.194", - "model.language_model.layers.28.mlp.experts.195", - "model.language_model.layers.28.mlp.experts.196", - "model.language_model.layers.28.mlp.experts.197", - "model.language_model.layers.28.mlp.experts.198", - "model.language_model.layers.28.mlp.experts.199", - "model.language_model.layers.28.mlp.experts.200", - "model.language_model.layers.28.mlp.experts.201", - "model.language_model.layers.28.mlp.experts.202", - "model.language_model.layers.28.mlp.experts.203", - "model.language_model.layers.28.mlp.experts.204", - "model.language_model.layers.28.mlp.experts.205", - "model.language_model.layers.28.mlp.experts.206", - "model.language_model.layers.28.mlp.experts.207", - "model.language_model.layers.28.mlp.experts.208", - "model.language_model.layers.28.mlp.experts.209", - "model.language_model.layers.28.mlp.experts.210", - "model.language_model.layers.28.mlp.experts.211", - "model.language_model.layers.28.mlp.experts.212", - "model.language_model.layers.28.mlp.experts.213", - "model.language_model.layers.28.mlp.experts.214", - "model.language_model.layers.28.mlp.experts.215", - "model.language_model.layers.28.mlp.experts.216", - "model.language_model.layers.28.mlp.experts.217", - "model.language_model.layers.28.mlp.experts.218", - "model.language_model.layers.28.mlp.experts.219", - "model.language_model.layers.28.mlp.experts.220", - "model.language_model.layers.28.mlp.experts.221", - "model.language_model.layers.28.mlp.experts.222", - "model.language_model.layers.28.mlp.experts.223", - "model.language_model.layers.28.mlp.experts.224", - "model.language_model.layers.28.mlp.experts.225", - "model.language_model.layers.28.mlp.experts.226", - "model.language_model.layers.28.mlp.experts.227", - "model.language_model.layers.28.mlp.experts.228", - "model.language_model.layers.28.mlp.experts.229", - "model.language_model.layers.28.mlp.experts.230", - "model.language_model.layers.28.mlp.experts.231", - "model.language_model.layers.28.mlp.experts.232", - "model.language_model.layers.28.mlp.experts.233", - "model.language_model.layers.28.mlp.experts.234", - "model.language_model.layers.28.mlp.experts.235", - "model.language_model.layers.28.mlp.experts.236", - "model.language_model.layers.28.mlp.experts.237", - "model.language_model.layers.28.mlp.experts.238", - "model.language_model.layers.28.mlp.experts.239", - "model.language_model.layers.28.mlp.experts.240", - "model.language_model.layers.28.mlp.experts.241", - "model.language_model.layers.28.mlp.experts.242", - "model.language_model.layers.28.mlp.experts.243", - "model.language_model.layers.28.mlp.experts.244", - "model.language_model.layers.28.mlp.experts.245", - "model.language_model.layers.28.mlp.experts.246", - "model.language_model.layers.28.mlp.experts.247", - "model.language_model.layers.28.mlp.experts.248", - "model.language_model.layers.28.mlp.experts.249", - "model.language_model.layers.28.mlp.experts.250", - "model.language_model.layers.28.mlp.experts.251", - "model.language_model.layers.28.mlp.experts.252", - "model.language_model.layers.28.mlp.experts.253", - "model.language_model.layers.28.mlp.experts.254", - "model.language_model.layers.28.mlp.experts.255", "model.language_model.layers.28.mlp.shared_expert_gate", - "model.language_model.layers.29.linear_attn", "model.language_model.layers.29.linear_attn.norm", "model.language_model.layers.29.linear_attn.in_proj_b", "model.language_model.layers.29.linear_attn.in_proj_a", "model.language_model.layers.29.mlp.gate", - "model.language_model.layers.29.mlp.experts.0", - "model.language_model.layers.29.mlp.experts.1", - "model.language_model.layers.29.mlp.experts.2", - "model.language_model.layers.29.mlp.experts.3", - "model.language_model.layers.29.mlp.experts.4", - "model.language_model.layers.29.mlp.experts.5", - "model.language_model.layers.29.mlp.experts.6", - "model.language_model.layers.29.mlp.experts.7", - "model.language_model.layers.29.mlp.experts.8", - "model.language_model.layers.29.mlp.experts.9", - "model.language_model.layers.29.mlp.experts.10", - "model.language_model.layers.29.mlp.experts.11", - "model.language_model.layers.29.mlp.experts.12", - "model.language_model.layers.29.mlp.experts.13", - "model.language_model.layers.29.mlp.experts.14", - "model.language_model.layers.29.mlp.experts.15", - "model.language_model.layers.29.mlp.experts.16", - "model.language_model.layers.29.mlp.experts.17", - "model.language_model.layers.29.mlp.experts.18", - "model.language_model.layers.29.mlp.experts.19", - "model.language_model.layers.29.mlp.experts.20", - "model.language_model.layers.29.mlp.experts.21", - "model.language_model.layers.29.mlp.experts.22", - "model.language_model.layers.29.mlp.experts.23", - "model.language_model.layers.29.mlp.experts.24", - "model.language_model.layers.29.mlp.experts.25", - "model.language_model.layers.29.mlp.experts.26", - "model.language_model.layers.29.mlp.experts.27", - "model.language_model.layers.29.mlp.experts.28", - "model.language_model.layers.29.mlp.experts.29", - "model.language_model.layers.29.mlp.experts.30", - "model.language_model.layers.29.mlp.experts.31", - "model.language_model.layers.29.mlp.experts.32", - "model.language_model.layers.29.mlp.experts.33", - "model.language_model.layers.29.mlp.experts.34", - "model.language_model.layers.29.mlp.experts.35", - "model.language_model.layers.29.mlp.experts.36", - "model.language_model.layers.29.mlp.experts.37", - "model.language_model.layers.29.mlp.experts.38", - "model.language_model.layers.29.mlp.experts.39", - "model.language_model.layers.29.mlp.experts.40", - "model.language_model.layers.29.mlp.experts.41", - "model.language_model.layers.29.mlp.experts.42", - "model.language_model.layers.29.mlp.experts.43", - "model.language_model.layers.29.mlp.experts.44", - "model.language_model.layers.29.mlp.experts.45", - "model.language_model.layers.29.mlp.experts.46", - "model.language_model.layers.29.mlp.experts.47", - "model.language_model.layers.29.mlp.experts.48", - "model.language_model.layers.29.mlp.experts.49", - "model.language_model.layers.29.mlp.experts.50", - "model.language_model.layers.29.mlp.experts.51", - "model.language_model.layers.29.mlp.experts.52", - "model.language_model.layers.29.mlp.experts.53", - "model.language_model.layers.29.mlp.experts.54", - "model.language_model.layers.29.mlp.experts.55", - "model.language_model.layers.29.mlp.experts.56", - "model.language_model.layers.29.mlp.experts.57", - "model.language_model.layers.29.mlp.experts.58", - "model.language_model.layers.29.mlp.experts.59", - "model.language_model.layers.29.mlp.experts.60", - "model.language_model.layers.29.mlp.experts.61", - "model.language_model.layers.29.mlp.experts.62", - "model.language_model.layers.29.mlp.experts.63", - "model.language_model.layers.29.mlp.experts.64", - "model.language_model.layers.29.mlp.experts.65", - "model.language_model.layers.29.mlp.experts.66", - "model.language_model.layers.29.mlp.experts.67", - "model.language_model.layers.29.mlp.experts.68", - "model.language_model.layers.29.mlp.experts.69", - "model.language_model.layers.29.mlp.experts.70", - "model.language_model.layers.29.mlp.experts.71", - "model.language_model.layers.29.mlp.experts.72", - "model.language_model.layers.29.mlp.experts.73", - "model.language_model.layers.29.mlp.experts.74", - "model.language_model.layers.29.mlp.experts.75", - "model.language_model.layers.29.mlp.experts.76", - "model.language_model.layers.29.mlp.experts.77", - "model.language_model.layers.29.mlp.experts.78", - "model.language_model.layers.29.mlp.experts.79", - "model.language_model.layers.29.mlp.experts.80", - "model.language_model.layers.29.mlp.experts.81", - "model.language_model.layers.29.mlp.experts.82", - "model.language_model.layers.29.mlp.experts.83", - "model.language_model.layers.29.mlp.experts.84", - "model.language_model.layers.29.mlp.experts.85", - "model.language_model.layers.29.mlp.experts.86", - "model.language_model.layers.29.mlp.experts.87", - "model.language_model.layers.29.mlp.experts.88", - "model.language_model.layers.29.mlp.experts.89", - "model.language_model.layers.29.mlp.experts.90", - "model.language_model.layers.29.mlp.experts.91", - "model.language_model.layers.29.mlp.experts.92", - "model.language_model.layers.29.mlp.experts.93", - "model.language_model.layers.29.mlp.experts.94", - "model.language_model.layers.29.mlp.experts.95", - "model.language_model.layers.29.mlp.experts.96", - "model.language_model.layers.29.mlp.experts.97", - "model.language_model.layers.29.mlp.experts.98", - "model.language_model.layers.29.mlp.experts.99", - "model.language_model.layers.29.mlp.experts.100", - "model.language_model.layers.29.mlp.experts.101", - "model.language_model.layers.29.mlp.experts.102", - "model.language_model.layers.29.mlp.experts.103", - "model.language_model.layers.29.mlp.experts.104", - "model.language_model.layers.29.mlp.experts.105", - "model.language_model.layers.29.mlp.experts.106", - "model.language_model.layers.29.mlp.experts.107", - "model.language_model.layers.29.mlp.experts.108", - "model.language_model.layers.29.mlp.experts.109", - "model.language_model.layers.29.mlp.experts.110", - "model.language_model.layers.29.mlp.experts.111", - "model.language_model.layers.29.mlp.experts.112", - "model.language_model.layers.29.mlp.experts.113", - "model.language_model.layers.29.mlp.experts.114", - "model.language_model.layers.29.mlp.experts.115", - "model.language_model.layers.29.mlp.experts.116", - "model.language_model.layers.29.mlp.experts.117", - "model.language_model.layers.29.mlp.experts.118", - "model.language_model.layers.29.mlp.experts.119", - "model.language_model.layers.29.mlp.experts.120", - "model.language_model.layers.29.mlp.experts.121", - "model.language_model.layers.29.mlp.experts.122", - "model.language_model.layers.29.mlp.experts.123", - "model.language_model.layers.29.mlp.experts.124", - "model.language_model.layers.29.mlp.experts.125", - "model.language_model.layers.29.mlp.experts.126", - "model.language_model.layers.29.mlp.experts.127", - "model.language_model.layers.29.mlp.experts.128", - "model.language_model.layers.29.mlp.experts.129", - "model.language_model.layers.29.mlp.experts.130", - "model.language_model.layers.29.mlp.experts.131", - "model.language_model.layers.29.mlp.experts.132", - "model.language_model.layers.29.mlp.experts.133", - "model.language_model.layers.29.mlp.experts.134", - "model.language_model.layers.29.mlp.experts.135", - "model.language_model.layers.29.mlp.experts.136", - "model.language_model.layers.29.mlp.experts.137", - "model.language_model.layers.29.mlp.experts.138", - "model.language_model.layers.29.mlp.experts.139", - "model.language_model.layers.29.mlp.experts.140", - "model.language_model.layers.29.mlp.experts.141", - "model.language_model.layers.29.mlp.experts.142", - "model.language_model.layers.29.mlp.experts.143", - "model.language_model.layers.29.mlp.experts.144", - "model.language_model.layers.29.mlp.experts.145", - "model.language_model.layers.29.mlp.experts.146", - "model.language_model.layers.29.mlp.experts.147", - "model.language_model.layers.29.mlp.experts.148", - "model.language_model.layers.29.mlp.experts.149", - "model.language_model.layers.29.mlp.experts.150", - "model.language_model.layers.29.mlp.experts.151", - "model.language_model.layers.29.mlp.experts.152", - "model.language_model.layers.29.mlp.experts.153", - "model.language_model.layers.29.mlp.experts.154", - "model.language_model.layers.29.mlp.experts.155", - "model.language_model.layers.29.mlp.experts.156", - "model.language_model.layers.29.mlp.experts.157", - "model.language_model.layers.29.mlp.experts.158", - "model.language_model.layers.29.mlp.experts.159", - "model.language_model.layers.29.mlp.experts.160", - "model.language_model.layers.29.mlp.experts.161", - "model.language_model.layers.29.mlp.experts.162", - "model.language_model.layers.29.mlp.experts.163", - "model.language_model.layers.29.mlp.experts.164", - "model.language_model.layers.29.mlp.experts.165", - "model.language_model.layers.29.mlp.experts.166", - "model.language_model.layers.29.mlp.experts.167", - "model.language_model.layers.29.mlp.experts.168", - "model.language_model.layers.29.mlp.experts.169", - "model.language_model.layers.29.mlp.experts.170", - "model.language_model.layers.29.mlp.experts.171", - "model.language_model.layers.29.mlp.experts.172", - "model.language_model.layers.29.mlp.experts.173", - "model.language_model.layers.29.mlp.experts.174", - "model.language_model.layers.29.mlp.experts.175", - "model.language_model.layers.29.mlp.experts.176", - "model.language_model.layers.29.mlp.experts.177", - "model.language_model.layers.29.mlp.experts.178", - "model.language_model.layers.29.mlp.experts.179", - "model.language_model.layers.29.mlp.experts.180", - "model.language_model.layers.29.mlp.experts.181", - "model.language_model.layers.29.mlp.experts.182", - "model.language_model.layers.29.mlp.experts.183", - "model.language_model.layers.29.mlp.experts.184", - "model.language_model.layers.29.mlp.experts.185", - "model.language_model.layers.29.mlp.experts.186", - "model.language_model.layers.29.mlp.experts.187", - "model.language_model.layers.29.mlp.experts.188", - "model.language_model.layers.29.mlp.experts.189", - "model.language_model.layers.29.mlp.experts.190", - "model.language_model.layers.29.mlp.experts.191", - "model.language_model.layers.29.mlp.experts.192", - "model.language_model.layers.29.mlp.experts.193", - "model.language_model.layers.29.mlp.experts.194", - "model.language_model.layers.29.mlp.experts.195", - "model.language_model.layers.29.mlp.experts.196", - "model.language_model.layers.29.mlp.experts.197", - "model.language_model.layers.29.mlp.experts.198", - "model.language_model.layers.29.mlp.experts.199", - "model.language_model.layers.29.mlp.experts.200", - "model.language_model.layers.29.mlp.experts.201", - "model.language_model.layers.29.mlp.experts.202", - "model.language_model.layers.29.mlp.experts.203", - "model.language_model.layers.29.mlp.experts.204", - "model.language_model.layers.29.mlp.experts.205", - "model.language_model.layers.29.mlp.experts.206", - "model.language_model.layers.29.mlp.experts.207", - "model.language_model.layers.29.mlp.experts.208", - "model.language_model.layers.29.mlp.experts.209", - "model.language_model.layers.29.mlp.experts.210", - "model.language_model.layers.29.mlp.experts.211", - "model.language_model.layers.29.mlp.experts.212", - "model.language_model.layers.29.mlp.experts.213", - "model.language_model.layers.29.mlp.experts.214", - "model.language_model.layers.29.mlp.experts.215", - "model.language_model.layers.29.mlp.experts.216", - "model.language_model.layers.29.mlp.experts.217", - "model.language_model.layers.29.mlp.experts.218", - "model.language_model.layers.29.mlp.experts.219", - "model.language_model.layers.29.mlp.experts.220", - "model.language_model.layers.29.mlp.experts.221", - "model.language_model.layers.29.mlp.experts.222", - "model.language_model.layers.29.mlp.experts.223", - "model.language_model.layers.29.mlp.experts.224", - "model.language_model.layers.29.mlp.experts.225", - "model.language_model.layers.29.mlp.experts.226", - "model.language_model.layers.29.mlp.experts.227", - "model.language_model.layers.29.mlp.experts.228", - "model.language_model.layers.29.mlp.experts.229", - "model.language_model.layers.29.mlp.experts.230", - "model.language_model.layers.29.mlp.experts.231", - "model.language_model.layers.29.mlp.experts.232", - "model.language_model.layers.29.mlp.experts.233", - "model.language_model.layers.29.mlp.experts.234", - "model.language_model.layers.29.mlp.experts.235", - "model.language_model.layers.29.mlp.experts.236", - "model.language_model.layers.29.mlp.experts.237", - "model.language_model.layers.29.mlp.experts.238", - "model.language_model.layers.29.mlp.experts.239", - "model.language_model.layers.29.mlp.experts.240", - "model.language_model.layers.29.mlp.experts.241", - "model.language_model.layers.29.mlp.experts.242", - "model.language_model.layers.29.mlp.experts.243", - "model.language_model.layers.29.mlp.experts.244", - "model.language_model.layers.29.mlp.experts.245", - "model.language_model.layers.29.mlp.experts.246", - "model.language_model.layers.29.mlp.experts.247", - "model.language_model.layers.29.mlp.experts.248", - "model.language_model.layers.29.mlp.experts.249", - "model.language_model.layers.29.mlp.experts.250", - "model.language_model.layers.29.mlp.experts.251", - "model.language_model.layers.29.mlp.experts.252", - "model.language_model.layers.29.mlp.experts.253", - "model.language_model.layers.29.mlp.experts.254", - "model.language_model.layers.29.mlp.experts.255", "model.language_model.layers.29.mlp.shared_expert_gate", - "model.language_model.layers.30.linear_attn", "model.language_model.layers.30.linear_attn.norm", "model.language_model.layers.30.linear_attn.in_proj_b", "model.language_model.layers.30.linear_attn.in_proj_a", "model.language_model.layers.30.mlp.gate", - "model.language_model.layers.30.mlp.experts.0", - "model.language_model.layers.30.mlp.experts.1", - "model.language_model.layers.30.mlp.experts.2", - "model.language_model.layers.30.mlp.experts.3", - "model.language_model.layers.30.mlp.experts.4", - "model.language_model.layers.30.mlp.experts.5", - "model.language_model.layers.30.mlp.experts.6", - "model.language_model.layers.30.mlp.experts.7", - "model.language_model.layers.30.mlp.experts.8", - "model.language_model.layers.30.mlp.experts.9", - "model.language_model.layers.30.mlp.experts.10", - "model.language_model.layers.30.mlp.experts.11", - "model.language_model.layers.30.mlp.experts.12", - "model.language_model.layers.30.mlp.experts.13", - "model.language_model.layers.30.mlp.experts.14", - "model.language_model.layers.30.mlp.experts.15", - "model.language_model.layers.30.mlp.experts.16", - "model.language_model.layers.30.mlp.experts.17", - "model.language_model.layers.30.mlp.experts.18", - "model.language_model.layers.30.mlp.experts.19", - "model.language_model.layers.30.mlp.experts.20", - "model.language_model.layers.30.mlp.experts.21", - "model.language_model.layers.30.mlp.experts.22", - "model.language_model.layers.30.mlp.experts.23", - "model.language_model.layers.30.mlp.experts.24", - "model.language_model.layers.30.mlp.experts.25", - "model.language_model.layers.30.mlp.experts.26", - "model.language_model.layers.30.mlp.experts.27", - "model.language_model.layers.30.mlp.experts.28", - "model.language_model.layers.30.mlp.experts.29", - "model.language_model.layers.30.mlp.experts.30", - "model.language_model.layers.30.mlp.experts.31", - "model.language_model.layers.30.mlp.experts.32", - "model.language_model.layers.30.mlp.experts.33", - "model.language_model.layers.30.mlp.experts.34", - "model.language_model.layers.30.mlp.experts.35", - "model.language_model.layers.30.mlp.experts.36", - "model.language_model.layers.30.mlp.experts.37", - "model.language_model.layers.30.mlp.experts.38", - "model.language_model.layers.30.mlp.experts.39", - "model.language_model.layers.30.mlp.experts.40", - "model.language_model.layers.30.mlp.experts.41", - "model.language_model.layers.30.mlp.experts.42", - "model.language_model.layers.30.mlp.experts.43", - "model.language_model.layers.30.mlp.experts.44", - "model.language_model.layers.30.mlp.experts.45", - "model.language_model.layers.30.mlp.experts.46", - "model.language_model.layers.30.mlp.experts.47", - "model.language_model.layers.30.mlp.experts.48", - "model.language_model.layers.30.mlp.experts.49", - "model.language_model.layers.30.mlp.experts.50", - "model.language_model.layers.30.mlp.experts.51", - "model.language_model.layers.30.mlp.experts.52", - "model.language_model.layers.30.mlp.experts.53", - "model.language_model.layers.30.mlp.experts.54", - "model.language_model.layers.30.mlp.experts.55", - "model.language_model.layers.30.mlp.experts.56", - "model.language_model.layers.30.mlp.experts.57", - "model.language_model.layers.30.mlp.experts.58", - "model.language_model.layers.30.mlp.experts.59", - "model.language_model.layers.30.mlp.experts.60", - "model.language_model.layers.30.mlp.experts.61", - "model.language_model.layers.30.mlp.experts.62", - "model.language_model.layers.30.mlp.experts.63", - "model.language_model.layers.30.mlp.experts.64", - "model.language_model.layers.30.mlp.experts.65", - "model.language_model.layers.30.mlp.experts.66", - "model.language_model.layers.30.mlp.experts.67", - "model.language_model.layers.30.mlp.experts.68", - "model.language_model.layers.30.mlp.experts.69", - "model.language_model.layers.30.mlp.experts.70", - "model.language_model.layers.30.mlp.experts.71", - "model.language_model.layers.30.mlp.experts.72", - "model.language_model.layers.30.mlp.experts.73", - "model.language_model.layers.30.mlp.experts.74", - "model.language_model.layers.30.mlp.experts.75", - "model.language_model.layers.30.mlp.experts.76", - "model.language_model.layers.30.mlp.experts.77", - "model.language_model.layers.30.mlp.experts.78", - "model.language_model.layers.30.mlp.experts.79", - "model.language_model.layers.30.mlp.experts.80", - "model.language_model.layers.30.mlp.experts.81", - "model.language_model.layers.30.mlp.experts.82", - "model.language_model.layers.30.mlp.experts.83", - "model.language_model.layers.30.mlp.experts.84", - "model.language_model.layers.30.mlp.experts.85", - "model.language_model.layers.30.mlp.experts.86", - "model.language_model.layers.30.mlp.experts.87", - "model.language_model.layers.30.mlp.experts.88", - "model.language_model.layers.30.mlp.experts.89", - "model.language_model.layers.30.mlp.experts.90", - "model.language_model.layers.30.mlp.experts.91", - "model.language_model.layers.30.mlp.experts.92", - "model.language_model.layers.30.mlp.experts.93", - "model.language_model.layers.30.mlp.experts.94", - "model.language_model.layers.30.mlp.experts.95", - "model.language_model.layers.30.mlp.experts.96", - "model.language_model.layers.30.mlp.experts.97", - "model.language_model.layers.30.mlp.experts.98", - "model.language_model.layers.30.mlp.experts.99", - "model.language_model.layers.30.mlp.experts.100", - "model.language_model.layers.30.mlp.experts.101", - "model.language_model.layers.30.mlp.experts.102", - "model.language_model.layers.30.mlp.experts.103", - "model.language_model.layers.30.mlp.experts.104", - "model.language_model.layers.30.mlp.experts.105", - "model.language_model.layers.30.mlp.experts.106", - "model.language_model.layers.30.mlp.experts.107", - "model.language_model.layers.30.mlp.experts.108", - "model.language_model.layers.30.mlp.experts.109", - "model.language_model.layers.30.mlp.experts.110", - "model.language_model.layers.30.mlp.experts.111", - "model.language_model.layers.30.mlp.experts.112", - "model.language_model.layers.30.mlp.experts.113", - "model.language_model.layers.30.mlp.experts.114", - "model.language_model.layers.30.mlp.experts.115", - "model.language_model.layers.30.mlp.experts.116", - "model.language_model.layers.30.mlp.experts.117", - "model.language_model.layers.30.mlp.experts.118", - "model.language_model.layers.30.mlp.experts.119", - "model.language_model.layers.30.mlp.experts.120", - "model.language_model.layers.30.mlp.experts.121", - "model.language_model.layers.30.mlp.experts.122", - "model.language_model.layers.30.mlp.experts.123", - "model.language_model.layers.30.mlp.experts.124", - "model.language_model.layers.30.mlp.experts.125", - "model.language_model.layers.30.mlp.experts.126", - "model.language_model.layers.30.mlp.experts.127", - "model.language_model.layers.30.mlp.experts.128", - "model.language_model.layers.30.mlp.experts.129", - "model.language_model.layers.30.mlp.experts.130", - "model.language_model.layers.30.mlp.experts.131", - "model.language_model.layers.30.mlp.experts.132", - "model.language_model.layers.30.mlp.experts.133", - "model.language_model.layers.30.mlp.experts.134", - "model.language_model.layers.30.mlp.experts.135", - "model.language_model.layers.30.mlp.experts.136", - "model.language_model.layers.30.mlp.experts.137", - "model.language_model.layers.30.mlp.experts.138", - "model.language_model.layers.30.mlp.experts.139", - "model.language_model.layers.30.mlp.experts.140", - "model.language_model.layers.30.mlp.experts.141", - "model.language_model.layers.30.mlp.experts.142", - "model.language_model.layers.30.mlp.experts.143", - "model.language_model.layers.30.mlp.experts.144", - "model.language_model.layers.30.mlp.experts.145", - "model.language_model.layers.30.mlp.experts.146", - "model.language_model.layers.30.mlp.experts.147", - "model.language_model.layers.30.mlp.experts.148", - "model.language_model.layers.30.mlp.experts.149", - "model.language_model.layers.30.mlp.experts.150", - "model.language_model.layers.30.mlp.experts.151", - "model.language_model.layers.30.mlp.experts.152", - "model.language_model.layers.30.mlp.experts.153", - "model.language_model.layers.30.mlp.experts.154", - "model.language_model.layers.30.mlp.experts.155", - "model.language_model.layers.30.mlp.experts.156", - "model.language_model.layers.30.mlp.experts.157", - "model.language_model.layers.30.mlp.experts.158", - "model.language_model.layers.30.mlp.experts.159", - "model.language_model.layers.30.mlp.experts.160", - "model.language_model.layers.30.mlp.experts.161", - "model.language_model.layers.30.mlp.experts.162", - "model.language_model.layers.30.mlp.experts.163", - "model.language_model.layers.30.mlp.experts.164", - "model.language_model.layers.30.mlp.experts.165", - "model.language_model.layers.30.mlp.experts.166", - "model.language_model.layers.30.mlp.experts.167", - "model.language_model.layers.30.mlp.experts.168", - "model.language_model.layers.30.mlp.experts.169", - "model.language_model.layers.30.mlp.experts.170", - "model.language_model.layers.30.mlp.experts.171", - "model.language_model.layers.30.mlp.experts.172", - "model.language_model.layers.30.mlp.experts.173", - "model.language_model.layers.30.mlp.experts.174", - "model.language_model.layers.30.mlp.experts.175", - "model.language_model.layers.30.mlp.experts.176", - "model.language_model.layers.30.mlp.experts.177", - "model.language_model.layers.30.mlp.experts.178", - "model.language_model.layers.30.mlp.experts.179", - "model.language_model.layers.30.mlp.experts.180", - "model.language_model.layers.30.mlp.experts.181", - "model.language_model.layers.30.mlp.experts.182", - "model.language_model.layers.30.mlp.experts.183", - "model.language_model.layers.30.mlp.experts.184", - "model.language_model.layers.30.mlp.experts.185", - "model.language_model.layers.30.mlp.experts.186", - "model.language_model.layers.30.mlp.experts.187", - "model.language_model.layers.30.mlp.experts.188", - "model.language_model.layers.30.mlp.experts.189", - "model.language_model.layers.30.mlp.experts.190", - "model.language_model.layers.30.mlp.experts.191", - "model.language_model.layers.30.mlp.experts.192", - "model.language_model.layers.30.mlp.experts.193", - "model.language_model.layers.30.mlp.experts.194", - "model.language_model.layers.30.mlp.experts.195", - "model.language_model.layers.30.mlp.experts.196", - "model.language_model.layers.30.mlp.experts.197", - "model.language_model.layers.30.mlp.experts.198", - "model.language_model.layers.30.mlp.experts.199", - "model.language_model.layers.30.mlp.experts.200", - "model.language_model.layers.30.mlp.experts.201", - "model.language_model.layers.30.mlp.experts.202", - "model.language_model.layers.30.mlp.experts.203", - "model.language_model.layers.30.mlp.experts.204", - "model.language_model.layers.30.mlp.experts.205", - "model.language_model.layers.30.mlp.experts.206", - "model.language_model.layers.30.mlp.experts.207", - "model.language_model.layers.30.mlp.experts.208", - "model.language_model.layers.30.mlp.experts.209", - "model.language_model.layers.30.mlp.experts.210", - "model.language_model.layers.30.mlp.experts.211", - "model.language_model.layers.30.mlp.experts.212", - "model.language_model.layers.30.mlp.experts.213", - "model.language_model.layers.30.mlp.experts.214", - "model.language_model.layers.30.mlp.experts.215", - "model.language_model.layers.30.mlp.experts.216", - "model.language_model.layers.30.mlp.experts.217", - "model.language_model.layers.30.mlp.experts.218", - "model.language_model.layers.30.mlp.experts.219", - "model.language_model.layers.30.mlp.experts.220", - "model.language_model.layers.30.mlp.experts.221", - "model.language_model.layers.30.mlp.experts.222", - "model.language_model.layers.30.mlp.experts.223", - "model.language_model.layers.30.mlp.experts.224", - "model.language_model.layers.30.mlp.experts.225", - "model.language_model.layers.30.mlp.experts.226", - "model.language_model.layers.30.mlp.experts.227", - "model.language_model.layers.30.mlp.experts.228", - "model.language_model.layers.30.mlp.experts.229", - "model.language_model.layers.30.mlp.experts.230", - "model.language_model.layers.30.mlp.experts.231", - "model.language_model.layers.30.mlp.experts.232", - "model.language_model.layers.30.mlp.experts.233", - "model.language_model.layers.30.mlp.experts.234", - "model.language_model.layers.30.mlp.experts.235", - "model.language_model.layers.30.mlp.experts.236", - "model.language_model.layers.30.mlp.experts.237", - "model.language_model.layers.30.mlp.experts.238", - "model.language_model.layers.30.mlp.experts.239", - "model.language_model.layers.30.mlp.experts.240", - "model.language_model.layers.30.mlp.experts.241", - "model.language_model.layers.30.mlp.experts.242", - "model.language_model.layers.30.mlp.experts.243", - "model.language_model.layers.30.mlp.experts.244", - "model.language_model.layers.30.mlp.experts.245", - "model.language_model.layers.30.mlp.experts.246", - "model.language_model.layers.30.mlp.experts.247", - "model.language_model.layers.30.mlp.experts.248", - "model.language_model.layers.30.mlp.experts.249", - "model.language_model.layers.30.mlp.experts.250", - "model.language_model.layers.30.mlp.experts.251", - "model.language_model.layers.30.mlp.experts.252", - "model.language_model.layers.30.mlp.experts.253", - "model.language_model.layers.30.mlp.experts.254", - "model.language_model.layers.30.mlp.experts.255", "model.language_model.layers.30.mlp.shared_expert_gate", "model.language_model.layers.31.mlp.gate", - "model.language_model.layers.31.mlp.experts.0", - "model.language_model.layers.31.mlp.experts.1", - "model.language_model.layers.31.mlp.experts.2", - "model.language_model.layers.31.mlp.experts.3", - "model.language_model.layers.31.mlp.experts.4", - "model.language_model.layers.31.mlp.experts.5", - "model.language_model.layers.31.mlp.experts.6", - "model.language_model.layers.31.mlp.experts.7", - "model.language_model.layers.31.mlp.experts.8", - "model.language_model.layers.31.mlp.experts.9", - "model.language_model.layers.31.mlp.experts.10", - "model.language_model.layers.31.mlp.experts.11", - "model.language_model.layers.31.mlp.experts.12", - "model.language_model.layers.31.mlp.experts.13", - "model.language_model.layers.31.mlp.experts.14", - "model.language_model.layers.31.mlp.experts.15", - "model.language_model.layers.31.mlp.experts.16", - "model.language_model.layers.31.mlp.experts.17", - "model.language_model.layers.31.mlp.experts.18", - "model.language_model.layers.31.mlp.experts.19", - "model.language_model.layers.31.mlp.experts.20", - "model.language_model.layers.31.mlp.experts.21", - "model.language_model.layers.31.mlp.experts.22", - "model.language_model.layers.31.mlp.experts.23", - "model.language_model.layers.31.mlp.experts.24", - "model.language_model.layers.31.mlp.experts.25", - "model.language_model.layers.31.mlp.experts.26", - "model.language_model.layers.31.mlp.experts.27", - "model.language_model.layers.31.mlp.experts.28", - "model.language_model.layers.31.mlp.experts.29", - "model.language_model.layers.31.mlp.experts.30", - "model.language_model.layers.31.mlp.experts.31", - "model.language_model.layers.31.mlp.experts.32", - "model.language_model.layers.31.mlp.experts.33", - "model.language_model.layers.31.mlp.experts.34", - "model.language_model.layers.31.mlp.experts.35", - "model.language_model.layers.31.mlp.experts.36", - "model.language_model.layers.31.mlp.experts.37", - "model.language_model.layers.31.mlp.experts.38", - "model.language_model.layers.31.mlp.experts.39", - "model.language_model.layers.31.mlp.experts.40", - "model.language_model.layers.31.mlp.experts.41", - "model.language_model.layers.31.mlp.experts.42", - "model.language_model.layers.31.mlp.experts.43", - "model.language_model.layers.31.mlp.experts.44", - "model.language_model.layers.31.mlp.experts.45", - "model.language_model.layers.31.mlp.experts.46", - "model.language_model.layers.31.mlp.experts.47", - "model.language_model.layers.31.mlp.experts.48", - "model.language_model.layers.31.mlp.experts.49", - "model.language_model.layers.31.mlp.experts.50", - "model.language_model.layers.31.mlp.experts.51", - "model.language_model.layers.31.mlp.experts.52", - "model.language_model.layers.31.mlp.experts.53", - "model.language_model.layers.31.mlp.experts.54", - "model.language_model.layers.31.mlp.experts.55", - "model.language_model.layers.31.mlp.experts.56", - "model.language_model.layers.31.mlp.experts.57", - "model.language_model.layers.31.mlp.experts.58", - "model.language_model.layers.31.mlp.experts.59", - "model.language_model.layers.31.mlp.experts.60", - "model.language_model.layers.31.mlp.experts.61", - "model.language_model.layers.31.mlp.experts.62", - "model.language_model.layers.31.mlp.experts.63", - "model.language_model.layers.31.mlp.experts.64", - "model.language_model.layers.31.mlp.experts.65", - "model.language_model.layers.31.mlp.experts.66", - "model.language_model.layers.31.mlp.experts.67", - "model.language_model.layers.31.mlp.experts.68", - "model.language_model.layers.31.mlp.experts.69", - "model.language_model.layers.31.mlp.experts.70", - "model.language_model.layers.31.mlp.experts.71", - "model.language_model.layers.31.mlp.experts.72", - "model.language_model.layers.31.mlp.experts.73", - "model.language_model.layers.31.mlp.experts.74", - "model.language_model.layers.31.mlp.experts.75", - "model.language_model.layers.31.mlp.experts.76", - "model.language_model.layers.31.mlp.experts.77", - "model.language_model.layers.31.mlp.experts.78", - "model.language_model.layers.31.mlp.experts.79", - "model.language_model.layers.31.mlp.experts.80", - "model.language_model.layers.31.mlp.experts.81", - "model.language_model.layers.31.mlp.experts.82", - "model.language_model.layers.31.mlp.experts.83", - "model.language_model.layers.31.mlp.experts.84", - "model.language_model.layers.31.mlp.experts.85", - "model.language_model.layers.31.mlp.experts.86", - "model.language_model.layers.31.mlp.experts.87", - "model.language_model.layers.31.mlp.experts.88", - "model.language_model.layers.31.mlp.experts.89", - "model.language_model.layers.31.mlp.experts.90", - "model.language_model.layers.31.mlp.experts.91", - "model.language_model.layers.31.mlp.experts.92", - "model.language_model.layers.31.mlp.experts.93", - "model.language_model.layers.31.mlp.experts.94", - "model.language_model.layers.31.mlp.experts.95", - "model.language_model.layers.31.mlp.experts.96", - "model.language_model.layers.31.mlp.experts.97", - "model.language_model.layers.31.mlp.experts.98", - "model.language_model.layers.31.mlp.experts.99", - "model.language_model.layers.31.mlp.experts.100", - "model.language_model.layers.31.mlp.experts.101", - "model.language_model.layers.31.mlp.experts.102", - "model.language_model.layers.31.mlp.experts.103", - "model.language_model.layers.31.mlp.experts.104", - "model.language_model.layers.31.mlp.experts.105", - "model.language_model.layers.31.mlp.experts.106", - "model.language_model.layers.31.mlp.experts.107", - "model.language_model.layers.31.mlp.experts.108", - "model.language_model.layers.31.mlp.experts.109", - "model.language_model.layers.31.mlp.experts.110", - "model.language_model.layers.31.mlp.experts.111", - "model.language_model.layers.31.mlp.experts.112", - "model.language_model.layers.31.mlp.experts.113", - "model.language_model.layers.31.mlp.experts.114", - "model.language_model.layers.31.mlp.experts.115", - "model.language_model.layers.31.mlp.experts.116", - "model.language_model.layers.31.mlp.experts.117", - "model.language_model.layers.31.mlp.experts.118", - "model.language_model.layers.31.mlp.experts.119", - "model.language_model.layers.31.mlp.experts.120", - "model.language_model.layers.31.mlp.experts.121", - "model.language_model.layers.31.mlp.experts.122", - "model.language_model.layers.31.mlp.experts.123", - "model.language_model.layers.31.mlp.experts.124", - "model.language_model.layers.31.mlp.experts.125", - "model.language_model.layers.31.mlp.experts.126", - "model.language_model.layers.31.mlp.experts.127", - "model.language_model.layers.31.mlp.experts.128", - "model.language_model.layers.31.mlp.experts.129", - "model.language_model.layers.31.mlp.experts.130", - "model.language_model.layers.31.mlp.experts.131", - "model.language_model.layers.31.mlp.experts.132", - "model.language_model.layers.31.mlp.experts.133", - "model.language_model.layers.31.mlp.experts.134", - "model.language_model.layers.31.mlp.experts.135", - "model.language_model.layers.31.mlp.experts.136", - "model.language_model.layers.31.mlp.experts.137", - "model.language_model.layers.31.mlp.experts.138", - "model.language_model.layers.31.mlp.experts.139", - "model.language_model.layers.31.mlp.experts.140", - "model.language_model.layers.31.mlp.experts.141", - "model.language_model.layers.31.mlp.experts.142", - "model.language_model.layers.31.mlp.experts.143", - "model.language_model.layers.31.mlp.experts.144", - "model.language_model.layers.31.mlp.experts.145", - "model.language_model.layers.31.mlp.experts.146", - "model.language_model.layers.31.mlp.experts.147", - "model.language_model.layers.31.mlp.experts.148", - "model.language_model.layers.31.mlp.experts.149", - "model.language_model.layers.31.mlp.experts.150", - "model.language_model.layers.31.mlp.experts.151", - "model.language_model.layers.31.mlp.experts.152", - "model.language_model.layers.31.mlp.experts.153", - "model.language_model.layers.31.mlp.experts.154", - "model.language_model.layers.31.mlp.experts.155", - "model.language_model.layers.31.mlp.experts.156", - "model.language_model.layers.31.mlp.experts.157", - "model.language_model.layers.31.mlp.experts.158", - "model.language_model.layers.31.mlp.experts.159", - "model.language_model.layers.31.mlp.experts.160", - "model.language_model.layers.31.mlp.experts.161", - "model.language_model.layers.31.mlp.experts.162", - "model.language_model.layers.31.mlp.experts.163", - "model.language_model.layers.31.mlp.experts.164", - "model.language_model.layers.31.mlp.experts.165", - "model.language_model.layers.31.mlp.experts.166", - "model.language_model.layers.31.mlp.experts.167", - "model.language_model.layers.31.mlp.experts.168", - "model.language_model.layers.31.mlp.experts.169", - "model.language_model.layers.31.mlp.experts.170", - "model.language_model.layers.31.mlp.experts.171", - "model.language_model.layers.31.mlp.experts.172", - "model.language_model.layers.31.mlp.experts.173", - "model.language_model.layers.31.mlp.experts.174", - "model.language_model.layers.31.mlp.experts.175", - "model.language_model.layers.31.mlp.experts.176", - "model.language_model.layers.31.mlp.experts.177", - "model.language_model.layers.31.mlp.experts.178", - "model.language_model.layers.31.mlp.experts.179", - "model.language_model.layers.31.mlp.experts.180", - "model.language_model.layers.31.mlp.experts.181", - "model.language_model.layers.31.mlp.experts.182", - "model.language_model.layers.31.mlp.experts.183", - "model.language_model.layers.31.mlp.experts.184", - "model.language_model.layers.31.mlp.experts.185", - "model.language_model.layers.31.mlp.experts.186", - "model.language_model.layers.31.mlp.experts.187", - "model.language_model.layers.31.mlp.experts.188", - "model.language_model.layers.31.mlp.experts.189", - "model.language_model.layers.31.mlp.experts.190", - "model.language_model.layers.31.mlp.experts.191", - "model.language_model.layers.31.mlp.experts.192", - "model.language_model.layers.31.mlp.experts.193", - "model.language_model.layers.31.mlp.experts.194", - "model.language_model.layers.31.mlp.experts.195", - "model.language_model.layers.31.mlp.experts.196", - "model.language_model.layers.31.mlp.experts.197", - "model.language_model.layers.31.mlp.experts.198", - "model.language_model.layers.31.mlp.experts.199", - "model.language_model.layers.31.mlp.experts.200", - "model.language_model.layers.31.mlp.experts.201", - "model.language_model.layers.31.mlp.experts.202", - "model.language_model.layers.31.mlp.experts.203", - "model.language_model.layers.31.mlp.experts.204", - "model.language_model.layers.31.mlp.experts.205", - "model.language_model.layers.31.mlp.experts.206", - "model.language_model.layers.31.mlp.experts.207", - "model.language_model.layers.31.mlp.experts.208", - "model.language_model.layers.31.mlp.experts.209", - "model.language_model.layers.31.mlp.experts.210", - "model.language_model.layers.31.mlp.experts.211", - "model.language_model.layers.31.mlp.experts.212", - "model.language_model.layers.31.mlp.experts.213", - "model.language_model.layers.31.mlp.experts.214", - "model.language_model.layers.31.mlp.experts.215", - "model.language_model.layers.31.mlp.experts.216", - "model.language_model.layers.31.mlp.experts.217", - "model.language_model.layers.31.mlp.experts.218", - "model.language_model.layers.31.mlp.experts.219", - "model.language_model.layers.31.mlp.experts.220", - "model.language_model.layers.31.mlp.experts.221", - "model.language_model.layers.31.mlp.experts.222", - "model.language_model.layers.31.mlp.experts.223", - "model.language_model.layers.31.mlp.experts.224", - "model.language_model.layers.31.mlp.experts.225", - "model.language_model.layers.31.mlp.experts.226", - "model.language_model.layers.31.mlp.experts.227", - "model.language_model.layers.31.mlp.experts.228", - "model.language_model.layers.31.mlp.experts.229", - "model.language_model.layers.31.mlp.experts.230", - "model.language_model.layers.31.mlp.experts.231", - "model.language_model.layers.31.mlp.experts.232", - "model.language_model.layers.31.mlp.experts.233", - "model.language_model.layers.31.mlp.experts.234", - "model.language_model.layers.31.mlp.experts.235", - "model.language_model.layers.31.mlp.experts.236", - "model.language_model.layers.31.mlp.experts.237", - "model.language_model.layers.31.mlp.experts.238", - "model.language_model.layers.31.mlp.experts.239", - "model.language_model.layers.31.mlp.experts.240", - "model.language_model.layers.31.mlp.experts.241", - "model.language_model.layers.31.mlp.experts.242", - "model.language_model.layers.31.mlp.experts.243", - "model.language_model.layers.31.mlp.experts.244", - "model.language_model.layers.31.mlp.experts.245", - "model.language_model.layers.31.mlp.experts.246", - "model.language_model.layers.31.mlp.experts.247", - "model.language_model.layers.31.mlp.experts.248", - "model.language_model.layers.31.mlp.experts.249", - "model.language_model.layers.31.mlp.experts.250", - "model.language_model.layers.31.mlp.experts.251", - "model.language_model.layers.31.mlp.experts.252", - "model.language_model.layers.31.mlp.experts.253", - "model.language_model.layers.31.mlp.experts.254", - "model.language_model.layers.31.mlp.experts.255", "model.language_model.layers.31.mlp.shared_expert_gate", - "model.language_model.layers.32.linear_attn", "model.language_model.layers.32.linear_attn.norm", "model.language_model.layers.32.linear_attn.in_proj_b", "model.language_model.layers.32.linear_attn.in_proj_a", "model.language_model.layers.32.mlp.gate", - "model.language_model.layers.32.mlp.experts.0", - "model.language_model.layers.32.mlp.experts.1", - "model.language_model.layers.32.mlp.experts.2", - "model.language_model.layers.32.mlp.experts.3", - "model.language_model.layers.32.mlp.experts.4", - "model.language_model.layers.32.mlp.experts.5", - "model.language_model.layers.32.mlp.experts.6", - "model.language_model.layers.32.mlp.experts.7", - "model.language_model.layers.32.mlp.experts.8", - "model.language_model.layers.32.mlp.experts.9", - "model.language_model.layers.32.mlp.experts.10", - "model.language_model.layers.32.mlp.experts.11", - "model.language_model.layers.32.mlp.experts.12", - "model.language_model.layers.32.mlp.experts.13", - "model.language_model.layers.32.mlp.experts.14", - "model.language_model.layers.32.mlp.experts.15", - "model.language_model.layers.32.mlp.experts.16", - "model.language_model.layers.32.mlp.experts.17", - "model.language_model.layers.32.mlp.experts.18", - "model.language_model.layers.32.mlp.experts.19", - "model.language_model.layers.32.mlp.experts.20", - "model.language_model.layers.32.mlp.experts.21", - "model.language_model.layers.32.mlp.experts.22", - "model.language_model.layers.32.mlp.experts.23", - "model.language_model.layers.32.mlp.experts.24", - "model.language_model.layers.32.mlp.experts.25", - "model.language_model.layers.32.mlp.experts.26", - "model.language_model.layers.32.mlp.experts.27", - "model.language_model.layers.32.mlp.experts.28", - "model.language_model.layers.32.mlp.experts.29", - "model.language_model.layers.32.mlp.experts.30", - "model.language_model.layers.32.mlp.experts.31", - "model.language_model.layers.32.mlp.experts.32", - "model.language_model.layers.32.mlp.experts.33", - "model.language_model.layers.32.mlp.experts.34", - "model.language_model.layers.32.mlp.experts.35", - "model.language_model.layers.32.mlp.experts.36", - "model.language_model.layers.32.mlp.experts.37", - "model.language_model.layers.32.mlp.experts.38", - "model.language_model.layers.32.mlp.experts.39", - "model.language_model.layers.32.mlp.experts.40", - "model.language_model.layers.32.mlp.experts.41", - "model.language_model.layers.32.mlp.experts.42", - "model.language_model.layers.32.mlp.experts.43", - "model.language_model.layers.32.mlp.experts.44", - "model.language_model.layers.32.mlp.experts.45", - "model.language_model.layers.32.mlp.experts.46", - "model.language_model.layers.32.mlp.experts.47", - "model.language_model.layers.32.mlp.experts.48", - "model.language_model.layers.32.mlp.experts.49", - "model.language_model.layers.32.mlp.experts.50", - "model.language_model.layers.32.mlp.experts.51", - "model.language_model.layers.32.mlp.experts.52", - "model.language_model.layers.32.mlp.experts.53", - "model.language_model.layers.32.mlp.experts.54", - "model.language_model.layers.32.mlp.experts.55", - "model.language_model.layers.32.mlp.experts.56", - "model.language_model.layers.32.mlp.experts.57", - "model.language_model.layers.32.mlp.experts.58", - "model.language_model.layers.32.mlp.experts.59", - "model.language_model.layers.32.mlp.experts.60", - "model.language_model.layers.32.mlp.experts.61", - "model.language_model.layers.32.mlp.experts.62", - "model.language_model.layers.32.mlp.experts.63", - "model.language_model.layers.32.mlp.experts.64", - "model.language_model.layers.32.mlp.experts.65", - "model.language_model.layers.32.mlp.experts.66", - "model.language_model.layers.32.mlp.experts.67", - "model.language_model.layers.32.mlp.experts.68", - "model.language_model.layers.32.mlp.experts.69", - "model.language_model.layers.32.mlp.experts.70", - "model.language_model.layers.32.mlp.experts.71", - "model.language_model.layers.32.mlp.experts.72", - "model.language_model.layers.32.mlp.experts.73", - "model.language_model.layers.32.mlp.experts.74", - "model.language_model.layers.32.mlp.experts.75", - "model.language_model.layers.32.mlp.experts.76", - "model.language_model.layers.32.mlp.experts.77", - "model.language_model.layers.32.mlp.experts.78", - "model.language_model.layers.32.mlp.experts.79", - "model.language_model.layers.32.mlp.experts.80", - "model.language_model.layers.32.mlp.experts.81", - "model.language_model.layers.32.mlp.experts.82", - "model.language_model.layers.32.mlp.experts.83", - "model.language_model.layers.32.mlp.experts.84", - "model.language_model.layers.32.mlp.experts.85", - "model.language_model.layers.32.mlp.experts.86", - "model.language_model.layers.32.mlp.experts.87", - "model.language_model.layers.32.mlp.experts.88", - "model.language_model.layers.32.mlp.experts.89", - "model.language_model.layers.32.mlp.experts.90", - "model.language_model.layers.32.mlp.experts.91", - "model.language_model.layers.32.mlp.experts.92", - "model.language_model.layers.32.mlp.experts.93", - "model.language_model.layers.32.mlp.experts.94", - "model.language_model.layers.32.mlp.experts.95", - "model.language_model.layers.32.mlp.experts.96", - "model.language_model.layers.32.mlp.experts.97", - "model.language_model.layers.32.mlp.experts.98", - "model.language_model.layers.32.mlp.experts.99", - "model.language_model.layers.32.mlp.experts.100", - "model.language_model.layers.32.mlp.experts.101", - "model.language_model.layers.32.mlp.experts.102", - "model.language_model.layers.32.mlp.experts.103", - "model.language_model.layers.32.mlp.experts.104", - "model.language_model.layers.32.mlp.experts.105", - "model.language_model.layers.32.mlp.experts.106", - "model.language_model.layers.32.mlp.experts.107", - "model.language_model.layers.32.mlp.experts.108", - "model.language_model.layers.32.mlp.experts.109", - "model.language_model.layers.32.mlp.experts.110", - "model.language_model.layers.32.mlp.experts.111", - "model.language_model.layers.32.mlp.experts.112", - "model.language_model.layers.32.mlp.experts.113", - "model.language_model.layers.32.mlp.experts.114", - "model.language_model.layers.32.mlp.experts.115", - "model.language_model.layers.32.mlp.experts.116", - "model.language_model.layers.32.mlp.experts.117", - "model.language_model.layers.32.mlp.experts.118", - "model.language_model.layers.32.mlp.experts.119", - "model.language_model.layers.32.mlp.experts.120", - "model.language_model.layers.32.mlp.experts.121", - "model.language_model.layers.32.mlp.experts.122", - "model.language_model.layers.32.mlp.experts.123", - "model.language_model.layers.32.mlp.experts.124", - "model.language_model.layers.32.mlp.experts.125", - "model.language_model.layers.32.mlp.experts.126", - "model.language_model.layers.32.mlp.experts.127", - "model.language_model.layers.32.mlp.experts.128", - "model.language_model.layers.32.mlp.experts.129", - "model.language_model.layers.32.mlp.experts.130", - "model.language_model.layers.32.mlp.experts.131", - "model.language_model.layers.32.mlp.experts.132", - "model.language_model.layers.32.mlp.experts.133", - "model.language_model.layers.32.mlp.experts.134", - "model.language_model.layers.32.mlp.experts.135", - "model.language_model.layers.32.mlp.experts.136", - "model.language_model.layers.32.mlp.experts.137", - "model.language_model.layers.32.mlp.experts.138", - "model.language_model.layers.32.mlp.experts.139", - "model.language_model.layers.32.mlp.experts.140", - "model.language_model.layers.32.mlp.experts.141", - "model.language_model.layers.32.mlp.experts.142", - "model.language_model.layers.32.mlp.experts.143", - "model.language_model.layers.32.mlp.experts.144", - "model.language_model.layers.32.mlp.experts.145", - "model.language_model.layers.32.mlp.experts.146", - "model.language_model.layers.32.mlp.experts.147", - "model.language_model.layers.32.mlp.experts.148", - "model.language_model.layers.32.mlp.experts.149", - "model.language_model.layers.32.mlp.experts.150", - "model.language_model.layers.32.mlp.experts.151", - "model.language_model.layers.32.mlp.experts.152", - "model.language_model.layers.32.mlp.experts.153", - "model.language_model.layers.32.mlp.experts.154", - "model.language_model.layers.32.mlp.experts.155", - "model.language_model.layers.32.mlp.experts.156", - "model.language_model.layers.32.mlp.experts.157", - "model.language_model.layers.32.mlp.experts.158", - "model.language_model.layers.32.mlp.experts.159", - "model.language_model.layers.32.mlp.experts.160", - "model.language_model.layers.32.mlp.experts.161", - "model.language_model.layers.32.mlp.experts.162", - "model.language_model.layers.32.mlp.experts.163", - "model.language_model.layers.32.mlp.experts.164", - "model.language_model.layers.32.mlp.experts.165", - "model.language_model.layers.32.mlp.experts.166", - "model.language_model.layers.32.mlp.experts.167", - "model.language_model.layers.32.mlp.experts.168", - "model.language_model.layers.32.mlp.experts.169", - "model.language_model.layers.32.mlp.experts.170", - "model.language_model.layers.32.mlp.experts.171", - "model.language_model.layers.32.mlp.experts.172", - "model.language_model.layers.32.mlp.experts.173", - "model.language_model.layers.32.mlp.experts.174", - "model.language_model.layers.32.mlp.experts.175", - "model.language_model.layers.32.mlp.experts.176", - "model.language_model.layers.32.mlp.experts.177", - "model.language_model.layers.32.mlp.experts.178", - "model.language_model.layers.32.mlp.experts.179", - "model.language_model.layers.32.mlp.experts.180", - "model.language_model.layers.32.mlp.experts.181", - "model.language_model.layers.32.mlp.experts.182", - "model.language_model.layers.32.mlp.experts.183", - "model.language_model.layers.32.mlp.experts.184", - "model.language_model.layers.32.mlp.experts.185", - "model.language_model.layers.32.mlp.experts.186", - "model.language_model.layers.32.mlp.experts.187", - "model.language_model.layers.32.mlp.experts.188", - "model.language_model.layers.32.mlp.experts.189", - "model.language_model.layers.32.mlp.experts.190", - "model.language_model.layers.32.mlp.experts.191", - "model.language_model.layers.32.mlp.experts.192", - "model.language_model.layers.32.mlp.experts.193", - "model.language_model.layers.32.mlp.experts.194", - "model.language_model.layers.32.mlp.experts.195", - "model.language_model.layers.32.mlp.experts.196", - "model.language_model.layers.32.mlp.experts.197", - "model.language_model.layers.32.mlp.experts.198", - "model.language_model.layers.32.mlp.experts.199", - "model.language_model.layers.32.mlp.experts.200", - "model.language_model.layers.32.mlp.experts.201", - "model.language_model.layers.32.mlp.experts.202", - "model.language_model.layers.32.mlp.experts.203", - "model.language_model.layers.32.mlp.experts.204", - "model.language_model.layers.32.mlp.experts.205", - "model.language_model.layers.32.mlp.experts.206", - "model.language_model.layers.32.mlp.experts.207", - "model.language_model.layers.32.mlp.experts.208", - "model.language_model.layers.32.mlp.experts.209", - "model.language_model.layers.32.mlp.experts.210", - "model.language_model.layers.32.mlp.experts.211", - "model.language_model.layers.32.mlp.experts.212", - "model.language_model.layers.32.mlp.experts.213", - "model.language_model.layers.32.mlp.experts.214", - "model.language_model.layers.32.mlp.experts.215", - "model.language_model.layers.32.mlp.experts.216", - "model.language_model.layers.32.mlp.experts.217", - "model.language_model.layers.32.mlp.experts.218", - "model.language_model.layers.32.mlp.experts.219", - "model.language_model.layers.32.mlp.experts.220", - "model.language_model.layers.32.mlp.experts.221", - "model.language_model.layers.32.mlp.experts.222", - "model.language_model.layers.32.mlp.experts.223", - "model.language_model.layers.32.mlp.experts.224", - "model.language_model.layers.32.mlp.experts.225", - "model.language_model.layers.32.mlp.experts.226", - "model.language_model.layers.32.mlp.experts.227", - "model.language_model.layers.32.mlp.experts.228", - "model.language_model.layers.32.mlp.experts.229", - "model.language_model.layers.32.mlp.experts.230", - "model.language_model.layers.32.mlp.experts.231", - "model.language_model.layers.32.mlp.experts.232", - "model.language_model.layers.32.mlp.experts.233", - "model.language_model.layers.32.mlp.experts.234", - "model.language_model.layers.32.mlp.experts.235", - "model.language_model.layers.32.mlp.experts.236", - "model.language_model.layers.32.mlp.experts.237", - "model.language_model.layers.32.mlp.experts.238", - "model.language_model.layers.32.mlp.experts.239", - "model.language_model.layers.32.mlp.experts.240", - "model.language_model.layers.32.mlp.experts.241", - "model.language_model.layers.32.mlp.experts.242", - "model.language_model.layers.32.mlp.experts.243", - "model.language_model.layers.32.mlp.experts.244", - "model.language_model.layers.32.mlp.experts.245", - "model.language_model.layers.32.mlp.experts.246", - "model.language_model.layers.32.mlp.experts.247", - "model.language_model.layers.32.mlp.experts.248", - "model.language_model.layers.32.mlp.experts.249", - "model.language_model.layers.32.mlp.experts.250", - "model.language_model.layers.32.mlp.experts.251", - "model.language_model.layers.32.mlp.experts.252", - "model.language_model.layers.32.mlp.experts.253", - "model.language_model.layers.32.mlp.experts.254", - "model.language_model.layers.32.mlp.experts.255", "model.language_model.layers.32.mlp.shared_expert_gate", - "model.language_model.layers.33.linear_attn", "model.language_model.layers.33.linear_attn.norm", "model.language_model.layers.33.linear_attn.in_proj_b", "model.language_model.layers.33.linear_attn.in_proj_a", "model.language_model.layers.33.mlp.gate", - "model.language_model.layers.33.mlp.experts.0", - "model.language_model.layers.33.mlp.experts.1", - "model.language_model.layers.33.mlp.experts.2", - "model.language_model.layers.33.mlp.experts.3", - "model.language_model.layers.33.mlp.experts.4", - "model.language_model.layers.33.mlp.experts.5", - "model.language_model.layers.33.mlp.experts.6", - "model.language_model.layers.33.mlp.experts.7", - "model.language_model.layers.33.mlp.experts.8", - "model.language_model.layers.33.mlp.experts.9", - "model.language_model.layers.33.mlp.experts.10", - "model.language_model.layers.33.mlp.experts.11", - "model.language_model.layers.33.mlp.experts.12", - "model.language_model.layers.33.mlp.experts.13", - "model.language_model.layers.33.mlp.experts.14", - "model.language_model.layers.33.mlp.experts.15", - "model.language_model.layers.33.mlp.experts.16", - "model.language_model.layers.33.mlp.experts.17", - "model.language_model.layers.33.mlp.experts.18", - "model.language_model.layers.33.mlp.experts.19", - "model.language_model.layers.33.mlp.experts.20", - "model.language_model.layers.33.mlp.experts.21", - "model.language_model.layers.33.mlp.experts.22", - "model.language_model.layers.33.mlp.experts.23", - "model.language_model.layers.33.mlp.experts.24", - "model.language_model.layers.33.mlp.experts.25", - "model.language_model.layers.33.mlp.experts.26", - "model.language_model.layers.33.mlp.experts.27", - "model.language_model.layers.33.mlp.experts.28", - "model.language_model.layers.33.mlp.experts.29", - "model.language_model.layers.33.mlp.experts.30", - "model.language_model.layers.33.mlp.experts.31", - "model.language_model.layers.33.mlp.experts.32", - "model.language_model.layers.33.mlp.experts.33", - "model.language_model.layers.33.mlp.experts.34", - "model.language_model.layers.33.mlp.experts.35", - "model.language_model.layers.33.mlp.experts.36", - "model.language_model.layers.33.mlp.experts.37", - "model.language_model.layers.33.mlp.experts.38", - "model.language_model.layers.33.mlp.experts.39", - "model.language_model.layers.33.mlp.experts.40", - "model.language_model.layers.33.mlp.experts.41", - "model.language_model.layers.33.mlp.experts.42", - "model.language_model.layers.33.mlp.experts.43", - "model.language_model.layers.33.mlp.experts.44", - "model.language_model.layers.33.mlp.experts.45", - "model.language_model.layers.33.mlp.experts.46", - "model.language_model.layers.33.mlp.experts.47", - "model.language_model.layers.33.mlp.experts.48", - "model.language_model.layers.33.mlp.experts.49", - "model.language_model.layers.33.mlp.experts.50", - "model.language_model.layers.33.mlp.experts.51", - "model.language_model.layers.33.mlp.experts.52", - "model.language_model.layers.33.mlp.experts.53", - "model.language_model.layers.33.mlp.experts.54", - "model.language_model.layers.33.mlp.experts.55", - "model.language_model.layers.33.mlp.experts.56", - "model.language_model.layers.33.mlp.experts.57", - "model.language_model.layers.33.mlp.experts.58", - "model.language_model.layers.33.mlp.experts.59", - "model.language_model.layers.33.mlp.experts.60", - "model.language_model.layers.33.mlp.experts.61", - "model.language_model.layers.33.mlp.experts.62", - "model.language_model.layers.33.mlp.experts.63", - "model.language_model.layers.33.mlp.experts.64", - "model.language_model.layers.33.mlp.experts.65", - "model.language_model.layers.33.mlp.experts.66", - "model.language_model.layers.33.mlp.experts.67", - "model.language_model.layers.33.mlp.experts.68", - "model.language_model.layers.33.mlp.experts.69", - "model.language_model.layers.33.mlp.experts.70", - "model.language_model.layers.33.mlp.experts.71", - "model.language_model.layers.33.mlp.experts.72", - "model.language_model.layers.33.mlp.experts.73", - "model.language_model.layers.33.mlp.experts.74", - "model.language_model.layers.33.mlp.experts.75", - "model.language_model.layers.33.mlp.experts.76", - "model.language_model.layers.33.mlp.experts.77", - "model.language_model.layers.33.mlp.experts.78", - "model.language_model.layers.33.mlp.experts.79", - "model.language_model.layers.33.mlp.experts.80", - "model.language_model.layers.33.mlp.experts.81", - "model.language_model.layers.33.mlp.experts.82", - "model.language_model.layers.33.mlp.experts.83", - "model.language_model.layers.33.mlp.experts.84", - "model.language_model.layers.33.mlp.experts.85", - "model.language_model.layers.33.mlp.experts.86", - "model.language_model.layers.33.mlp.experts.87", - "model.language_model.layers.33.mlp.experts.88", - "model.language_model.layers.33.mlp.experts.89", - "model.language_model.layers.33.mlp.experts.90", - "model.language_model.layers.33.mlp.experts.91", - "model.language_model.layers.33.mlp.experts.92", - "model.language_model.layers.33.mlp.experts.93", - "model.language_model.layers.33.mlp.experts.94", - "model.language_model.layers.33.mlp.experts.95", - "model.language_model.layers.33.mlp.experts.96", - "model.language_model.layers.33.mlp.experts.97", - "model.language_model.layers.33.mlp.experts.98", - "model.language_model.layers.33.mlp.experts.99", - "model.language_model.layers.33.mlp.experts.100", - "model.language_model.layers.33.mlp.experts.101", - "model.language_model.layers.33.mlp.experts.102", - "model.language_model.layers.33.mlp.experts.103", - "model.language_model.layers.33.mlp.experts.104", - "model.language_model.layers.33.mlp.experts.105", - "model.language_model.layers.33.mlp.experts.106", - "model.language_model.layers.33.mlp.experts.107", - "model.language_model.layers.33.mlp.experts.108", - "model.language_model.layers.33.mlp.experts.109", - "model.language_model.layers.33.mlp.experts.110", - "model.language_model.layers.33.mlp.experts.111", - "model.language_model.layers.33.mlp.experts.112", - "model.language_model.layers.33.mlp.experts.113", - "model.language_model.layers.33.mlp.experts.114", - "model.language_model.layers.33.mlp.experts.115", - "model.language_model.layers.33.mlp.experts.116", - "model.language_model.layers.33.mlp.experts.117", - "model.language_model.layers.33.mlp.experts.118", - "model.language_model.layers.33.mlp.experts.119", - "model.language_model.layers.33.mlp.experts.120", - "model.language_model.layers.33.mlp.experts.121", - "model.language_model.layers.33.mlp.experts.122", - "model.language_model.layers.33.mlp.experts.123", - "model.language_model.layers.33.mlp.experts.124", - "model.language_model.layers.33.mlp.experts.125", - "model.language_model.layers.33.mlp.experts.126", - "model.language_model.layers.33.mlp.experts.127", - "model.language_model.layers.33.mlp.experts.128", - "model.language_model.layers.33.mlp.experts.129", - "model.language_model.layers.33.mlp.experts.130", - "model.language_model.layers.33.mlp.experts.131", - "model.language_model.layers.33.mlp.experts.132", - "model.language_model.layers.33.mlp.experts.133", - "model.language_model.layers.33.mlp.experts.134", - "model.language_model.layers.33.mlp.experts.135", - "model.language_model.layers.33.mlp.experts.136", - "model.language_model.layers.33.mlp.experts.137", - "model.language_model.layers.33.mlp.experts.138", - "model.language_model.layers.33.mlp.experts.139", - "model.language_model.layers.33.mlp.experts.140", - "model.language_model.layers.33.mlp.experts.141", - "model.language_model.layers.33.mlp.experts.142", - "model.language_model.layers.33.mlp.experts.143", - "model.language_model.layers.33.mlp.experts.144", - "model.language_model.layers.33.mlp.experts.145", - "model.language_model.layers.33.mlp.experts.146", - "model.language_model.layers.33.mlp.experts.147", - "model.language_model.layers.33.mlp.experts.148", - "model.language_model.layers.33.mlp.experts.149", - "model.language_model.layers.33.mlp.experts.150", - "model.language_model.layers.33.mlp.experts.151", - "model.language_model.layers.33.mlp.experts.152", - "model.language_model.layers.33.mlp.experts.153", - "model.language_model.layers.33.mlp.experts.154", - "model.language_model.layers.33.mlp.experts.155", - "model.language_model.layers.33.mlp.experts.156", - "model.language_model.layers.33.mlp.experts.157", - "model.language_model.layers.33.mlp.experts.158", - "model.language_model.layers.33.mlp.experts.159", - "model.language_model.layers.33.mlp.experts.160", - "model.language_model.layers.33.mlp.experts.161", - "model.language_model.layers.33.mlp.experts.162", - "model.language_model.layers.33.mlp.experts.163", - "model.language_model.layers.33.mlp.experts.164", - "model.language_model.layers.33.mlp.experts.165", - "model.language_model.layers.33.mlp.experts.166", - "model.language_model.layers.33.mlp.experts.167", - "model.language_model.layers.33.mlp.experts.168", - "model.language_model.layers.33.mlp.experts.169", - "model.language_model.layers.33.mlp.experts.170", - "model.language_model.layers.33.mlp.experts.171", - "model.language_model.layers.33.mlp.experts.172", - "model.language_model.layers.33.mlp.experts.173", - "model.language_model.layers.33.mlp.experts.174", - "model.language_model.layers.33.mlp.experts.175", - "model.language_model.layers.33.mlp.experts.176", - "model.language_model.layers.33.mlp.experts.177", - "model.language_model.layers.33.mlp.experts.178", - "model.language_model.layers.33.mlp.experts.179", - "model.language_model.layers.33.mlp.experts.180", - "model.language_model.layers.33.mlp.experts.181", - "model.language_model.layers.33.mlp.experts.182", - "model.language_model.layers.33.mlp.experts.183", - "model.language_model.layers.33.mlp.experts.184", - "model.language_model.layers.33.mlp.experts.185", - "model.language_model.layers.33.mlp.experts.186", - "model.language_model.layers.33.mlp.experts.187", - "model.language_model.layers.33.mlp.experts.188", - "model.language_model.layers.33.mlp.experts.189", - "model.language_model.layers.33.mlp.experts.190", - "model.language_model.layers.33.mlp.experts.191", - "model.language_model.layers.33.mlp.experts.192", - "model.language_model.layers.33.mlp.experts.193", - "model.language_model.layers.33.mlp.experts.194", - "model.language_model.layers.33.mlp.experts.195", - "model.language_model.layers.33.mlp.experts.196", - "model.language_model.layers.33.mlp.experts.197", - "model.language_model.layers.33.mlp.experts.198", - "model.language_model.layers.33.mlp.experts.199", - "model.language_model.layers.33.mlp.experts.200", - "model.language_model.layers.33.mlp.experts.201", - "model.language_model.layers.33.mlp.experts.202", - "model.language_model.layers.33.mlp.experts.203", - "model.language_model.layers.33.mlp.experts.204", - "model.language_model.layers.33.mlp.experts.205", - "model.language_model.layers.33.mlp.experts.206", - "model.language_model.layers.33.mlp.experts.207", - "model.language_model.layers.33.mlp.experts.208", - "model.language_model.layers.33.mlp.experts.209", - "model.language_model.layers.33.mlp.experts.210", - "model.language_model.layers.33.mlp.experts.211", - "model.language_model.layers.33.mlp.experts.212", - "model.language_model.layers.33.mlp.experts.213", - "model.language_model.layers.33.mlp.experts.214", - "model.language_model.layers.33.mlp.experts.215", - "model.language_model.layers.33.mlp.experts.216", - "model.language_model.layers.33.mlp.experts.217", - "model.language_model.layers.33.mlp.experts.218", - "model.language_model.layers.33.mlp.experts.219", - "model.language_model.layers.33.mlp.experts.220", - "model.language_model.layers.33.mlp.experts.221", - "model.language_model.layers.33.mlp.experts.222", - "model.language_model.layers.33.mlp.experts.223", - "model.language_model.layers.33.mlp.experts.224", - "model.language_model.layers.33.mlp.experts.225", - "model.language_model.layers.33.mlp.experts.226", - "model.language_model.layers.33.mlp.experts.227", - "model.language_model.layers.33.mlp.experts.228", - "model.language_model.layers.33.mlp.experts.229", - "model.language_model.layers.33.mlp.experts.230", - "model.language_model.layers.33.mlp.experts.231", - "model.language_model.layers.33.mlp.experts.232", - "model.language_model.layers.33.mlp.experts.233", - "model.language_model.layers.33.mlp.experts.234", - "model.language_model.layers.33.mlp.experts.235", - "model.language_model.layers.33.mlp.experts.236", - "model.language_model.layers.33.mlp.experts.237", - "model.language_model.layers.33.mlp.experts.238", - "model.language_model.layers.33.mlp.experts.239", - "model.language_model.layers.33.mlp.experts.240", - "model.language_model.layers.33.mlp.experts.241", - "model.language_model.layers.33.mlp.experts.242", - "model.language_model.layers.33.mlp.experts.243", - "model.language_model.layers.33.mlp.experts.244", - "model.language_model.layers.33.mlp.experts.245", - "model.language_model.layers.33.mlp.experts.246", - "model.language_model.layers.33.mlp.experts.247", - "model.language_model.layers.33.mlp.experts.248", - "model.language_model.layers.33.mlp.experts.249", - "model.language_model.layers.33.mlp.experts.250", - "model.language_model.layers.33.mlp.experts.251", - "model.language_model.layers.33.mlp.experts.252", - "model.language_model.layers.33.mlp.experts.253", - "model.language_model.layers.33.mlp.experts.254", - "model.language_model.layers.33.mlp.experts.255", "model.language_model.layers.33.mlp.shared_expert_gate", - "model.language_model.layers.34.linear_attn", "model.language_model.layers.34.linear_attn.norm", "model.language_model.layers.34.linear_attn.in_proj_b", "model.language_model.layers.34.linear_attn.in_proj_a", "model.language_model.layers.34.mlp.gate", - "model.language_model.layers.34.mlp.experts.0", - "model.language_model.layers.34.mlp.experts.1", - "model.language_model.layers.34.mlp.experts.2", - "model.language_model.layers.34.mlp.experts.3", - "model.language_model.layers.34.mlp.experts.4", - "model.language_model.layers.34.mlp.experts.5", - "model.language_model.layers.34.mlp.experts.6", - "model.language_model.layers.34.mlp.experts.7", - "model.language_model.layers.34.mlp.experts.8", - "model.language_model.layers.34.mlp.experts.9", - "model.language_model.layers.34.mlp.experts.10", - "model.language_model.layers.34.mlp.experts.11", - "model.language_model.layers.34.mlp.experts.12", - "model.language_model.layers.34.mlp.experts.13", - "model.language_model.layers.34.mlp.experts.14", - "model.language_model.layers.34.mlp.experts.15", - "model.language_model.layers.34.mlp.experts.16", - "model.language_model.layers.34.mlp.experts.17", - "model.language_model.layers.34.mlp.experts.18", - "model.language_model.layers.34.mlp.experts.19", - "model.language_model.layers.34.mlp.experts.20", - "model.language_model.layers.34.mlp.experts.21", - "model.language_model.layers.34.mlp.experts.22", - "model.language_model.layers.34.mlp.experts.23", - "model.language_model.layers.34.mlp.experts.24", - "model.language_model.layers.34.mlp.experts.25", - "model.language_model.layers.34.mlp.experts.26", - "model.language_model.layers.34.mlp.experts.27", - "model.language_model.layers.34.mlp.experts.28", - "model.language_model.layers.34.mlp.experts.29", - "model.language_model.layers.34.mlp.experts.30", - "model.language_model.layers.34.mlp.experts.31", - "model.language_model.layers.34.mlp.experts.32", - "model.language_model.layers.34.mlp.experts.33", - "model.language_model.layers.34.mlp.experts.34", - "model.language_model.layers.34.mlp.experts.35", - "model.language_model.layers.34.mlp.experts.36", - "model.language_model.layers.34.mlp.experts.37", - "model.language_model.layers.34.mlp.experts.38", - "model.language_model.layers.34.mlp.experts.39", - "model.language_model.layers.34.mlp.experts.40", - "model.language_model.layers.34.mlp.experts.41", - "model.language_model.layers.34.mlp.experts.42", - "model.language_model.layers.34.mlp.experts.43", - "model.language_model.layers.34.mlp.experts.44", - "model.language_model.layers.34.mlp.experts.45", - "model.language_model.layers.34.mlp.experts.46", - "model.language_model.layers.34.mlp.experts.47", - "model.language_model.layers.34.mlp.experts.48", - "model.language_model.layers.34.mlp.experts.49", - "model.language_model.layers.34.mlp.experts.50", - "model.language_model.layers.34.mlp.experts.51", - "model.language_model.layers.34.mlp.experts.52", - "model.language_model.layers.34.mlp.experts.53", - "model.language_model.layers.34.mlp.experts.54", - "model.language_model.layers.34.mlp.experts.55", - "model.language_model.layers.34.mlp.experts.56", - "model.language_model.layers.34.mlp.experts.57", - "model.language_model.layers.34.mlp.experts.58", - "model.language_model.layers.34.mlp.experts.59", - "model.language_model.layers.34.mlp.experts.60", - "model.language_model.layers.34.mlp.experts.61", - "model.language_model.layers.34.mlp.experts.62", - "model.language_model.layers.34.mlp.experts.63", - "model.language_model.layers.34.mlp.experts.64", - "model.language_model.layers.34.mlp.experts.65", - "model.language_model.layers.34.mlp.experts.66", - "model.language_model.layers.34.mlp.experts.67", - "model.language_model.layers.34.mlp.experts.68", - "model.language_model.layers.34.mlp.experts.69", - "model.language_model.layers.34.mlp.experts.70", - "model.language_model.layers.34.mlp.experts.71", - "model.language_model.layers.34.mlp.experts.72", - "model.language_model.layers.34.mlp.experts.73", - "model.language_model.layers.34.mlp.experts.74", - "model.language_model.layers.34.mlp.experts.75", - "model.language_model.layers.34.mlp.experts.76", - "model.language_model.layers.34.mlp.experts.77", - "model.language_model.layers.34.mlp.experts.78", - "model.language_model.layers.34.mlp.experts.79", - "model.language_model.layers.34.mlp.experts.80", - "model.language_model.layers.34.mlp.experts.81", - "model.language_model.layers.34.mlp.experts.82", - "model.language_model.layers.34.mlp.experts.83", - "model.language_model.layers.34.mlp.experts.84", - "model.language_model.layers.34.mlp.experts.85", - "model.language_model.layers.34.mlp.experts.86", - "model.language_model.layers.34.mlp.experts.87", - "model.language_model.layers.34.mlp.experts.88", - "model.language_model.layers.34.mlp.experts.89", - "model.language_model.layers.34.mlp.experts.90", - "model.language_model.layers.34.mlp.experts.91", - "model.language_model.layers.34.mlp.experts.92", - "model.language_model.layers.34.mlp.experts.93", - "model.language_model.layers.34.mlp.experts.94", - "model.language_model.layers.34.mlp.experts.95", - "model.language_model.layers.34.mlp.experts.96", - "model.language_model.layers.34.mlp.experts.97", - "model.language_model.layers.34.mlp.experts.98", - "model.language_model.layers.34.mlp.experts.99", - "model.language_model.layers.34.mlp.experts.100", - "model.language_model.layers.34.mlp.experts.101", - "model.language_model.layers.34.mlp.experts.102", - "model.language_model.layers.34.mlp.experts.103", - "model.language_model.layers.34.mlp.experts.104", - "model.language_model.layers.34.mlp.experts.105", - "model.language_model.layers.34.mlp.experts.106", - "model.language_model.layers.34.mlp.experts.107", - "model.language_model.layers.34.mlp.experts.108", - "model.language_model.layers.34.mlp.experts.109", - "model.language_model.layers.34.mlp.experts.110", - "model.language_model.layers.34.mlp.experts.111", - "model.language_model.layers.34.mlp.experts.112", - "model.language_model.layers.34.mlp.experts.113", - "model.language_model.layers.34.mlp.experts.114", - "model.language_model.layers.34.mlp.experts.115", - "model.language_model.layers.34.mlp.experts.116", - "model.language_model.layers.34.mlp.experts.117", - "model.language_model.layers.34.mlp.experts.118", - "model.language_model.layers.34.mlp.experts.119", - "model.language_model.layers.34.mlp.experts.120", - "model.language_model.layers.34.mlp.experts.121", - "model.language_model.layers.34.mlp.experts.122", - "model.language_model.layers.34.mlp.experts.123", - "model.language_model.layers.34.mlp.experts.124", - "model.language_model.layers.34.mlp.experts.125", - "model.language_model.layers.34.mlp.experts.126", - "model.language_model.layers.34.mlp.experts.127", - "model.language_model.layers.34.mlp.experts.128", - "model.language_model.layers.34.mlp.experts.129", - "model.language_model.layers.34.mlp.experts.130", - "model.language_model.layers.34.mlp.experts.131", - "model.language_model.layers.34.mlp.experts.132", - "model.language_model.layers.34.mlp.experts.133", - "model.language_model.layers.34.mlp.experts.134", - "model.language_model.layers.34.mlp.experts.135", - "model.language_model.layers.34.mlp.experts.136", - "model.language_model.layers.34.mlp.experts.137", - "model.language_model.layers.34.mlp.experts.138", - "model.language_model.layers.34.mlp.experts.139", - "model.language_model.layers.34.mlp.experts.140", - "model.language_model.layers.34.mlp.experts.141", - "model.language_model.layers.34.mlp.experts.142", - "model.language_model.layers.34.mlp.experts.143", - "model.language_model.layers.34.mlp.experts.144", - "model.language_model.layers.34.mlp.experts.145", - "model.language_model.layers.34.mlp.experts.146", - "model.language_model.layers.34.mlp.experts.147", - "model.language_model.layers.34.mlp.experts.148", - "model.language_model.layers.34.mlp.experts.149", - "model.language_model.layers.34.mlp.experts.150", - "model.language_model.layers.34.mlp.experts.151", - "model.language_model.layers.34.mlp.experts.152", - "model.language_model.layers.34.mlp.experts.153", - "model.language_model.layers.34.mlp.experts.154", - "model.language_model.layers.34.mlp.experts.155", - "model.language_model.layers.34.mlp.experts.156", - "model.language_model.layers.34.mlp.experts.157", - "model.language_model.layers.34.mlp.experts.158", - "model.language_model.layers.34.mlp.experts.159", - "model.language_model.layers.34.mlp.experts.160", - "model.language_model.layers.34.mlp.experts.161", - "model.language_model.layers.34.mlp.experts.162", - "model.language_model.layers.34.mlp.experts.163", - "model.language_model.layers.34.mlp.experts.164", - "model.language_model.layers.34.mlp.experts.165", - "model.language_model.layers.34.mlp.experts.166", - "model.language_model.layers.34.mlp.experts.167", - "model.language_model.layers.34.mlp.experts.168", - "model.language_model.layers.34.mlp.experts.169", - "model.language_model.layers.34.mlp.experts.170", - "model.language_model.layers.34.mlp.experts.171", - "model.language_model.layers.34.mlp.experts.172", - "model.language_model.layers.34.mlp.experts.173", - "model.language_model.layers.34.mlp.experts.174", - "model.language_model.layers.34.mlp.experts.175", - "model.language_model.layers.34.mlp.experts.176", - "model.language_model.layers.34.mlp.experts.177", - "model.language_model.layers.34.mlp.experts.178", - "model.language_model.layers.34.mlp.experts.179", - "model.language_model.layers.34.mlp.experts.180", - "model.language_model.layers.34.mlp.experts.181", - "model.language_model.layers.34.mlp.experts.182", - "model.language_model.layers.34.mlp.experts.183", - "model.language_model.layers.34.mlp.experts.184", - "model.language_model.layers.34.mlp.experts.185", - "model.language_model.layers.34.mlp.experts.186", - "model.language_model.layers.34.mlp.experts.187", - "model.language_model.layers.34.mlp.experts.188", - "model.language_model.layers.34.mlp.experts.189", - "model.language_model.layers.34.mlp.experts.190", - "model.language_model.layers.34.mlp.experts.191", - "model.language_model.layers.34.mlp.experts.192", - "model.language_model.layers.34.mlp.experts.193", - "model.language_model.layers.34.mlp.experts.194", - "model.language_model.layers.34.mlp.experts.195", - "model.language_model.layers.34.mlp.experts.196", - "model.language_model.layers.34.mlp.experts.197", - "model.language_model.layers.34.mlp.experts.198", - "model.language_model.layers.34.mlp.experts.199", - "model.language_model.layers.34.mlp.experts.200", - "model.language_model.layers.34.mlp.experts.201", - "model.language_model.layers.34.mlp.experts.202", - "model.language_model.layers.34.mlp.experts.203", - "model.language_model.layers.34.mlp.experts.204", - "model.language_model.layers.34.mlp.experts.205", - "model.language_model.layers.34.mlp.experts.206", - "model.language_model.layers.34.mlp.experts.207", - "model.language_model.layers.34.mlp.experts.208", - "model.language_model.layers.34.mlp.experts.209", - "model.language_model.layers.34.mlp.experts.210", - "model.language_model.layers.34.mlp.experts.211", - "model.language_model.layers.34.mlp.experts.212", - "model.language_model.layers.34.mlp.experts.213", - "model.language_model.layers.34.mlp.experts.214", - "model.language_model.layers.34.mlp.experts.215", - "model.language_model.layers.34.mlp.experts.216", - "model.language_model.layers.34.mlp.experts.217", - "model.language_model.layers.34.mlp.experts.218", - "model.language_model.layers.34.mlp.experts.219", - "model.language_model.layers.34.mlp.experts.220", - "model.language_model.layers.34.mlp.experts.221", - "model.language_model.layers.34.mlp.experts.222", - "model.language_model.layers.34.mlp.experts.223", - "model.language_model.layers.34.mlp.experts.224", - "model.language_model.layers.34.mlp.experts.225", - "model.language_model.layers.34.mlp.experts.226", - "model.language_model.layers.34.mlp.experts.227", - "model.language_model.layers.34.mlp.experts.228", - "model.language_model.layers.34.mlp.experts.229", - "model.language_model.layers.34.mlp.experts.230", - "model.language_model.layers.34.mlp.experts.231", - "model.language_model.layers.34.mlp.experts.232", - "model.language_model.layers.34.mlp.experts.233", - "model.language_model.layers.34.mlp.experts.234", - "model.language_model.layers.34.mlp.experts.235", - "model.language_model.layers.34.mlp.experts.236", - "model.language_model.layers.34.mlp.experts.237", - "model.language_model.layers.34.mlp.experts.238", - "model.language_model.layers.34.mlp.experts.239", - "model.language_model.layers.34.mlp.experts.240", - "model.language_model.layers.34.mlp.experts.241", - "model.language_model.layers.34.mlp.experts.242", - "model.language_model.layers.34.mlp.experts.243", - "model.language_model.layers.34.mlp.experts.244", - "model.language_model.layers.34.mlp.experts.245", - "model.language_model.layers.34.mlp.experts.246", - "model.language_model.layers.34.mlp.experts.247", - "model.language_model.layers.34.mlp.experts.248", - "model.language_model.layers.34.mlp.experts.249", - "model.language_model.layers.34.mlp.experts.250", - "model.language_model.layers.34.mlp.experts.251", - "model.language_model.layers.34.mlp.experts.252", - "model.language_model.layers.34.mlp.experts.253", - "model.language_model.layers.34.mlp.experts.254", - "model.language_model.layers.34.mlp.experts.255", "model.language_model.layers.34.mlp.shared_expert_gate", "model.language_model.layers.35.mlp.gate", - "model.language_model.layers.35.mlp.experts.0", - "model.language_model.layers.35.mlp.experts.1", - "model.language_model.layers.35.mlp.experts.2", - "model.language_model.layers.35.mlp.experts.3", - "model.language_model.layers.35.mlp.experts.4", - "model.language_model.layers.35.mlp.experts.5", - "model.language_model.layers.35.mlp.experts.6", - "model.language_model.layers.35.mlp.experts.7", - "model.language_model.layers.35.mlp.experts.8", - "model.language_model.layers.35.mlp.experts.9", - "model.language_model.layers.35.mlp.experts.10", - "model.language_model.layers.35.mlp.experts.11", - "model.language_model.layers.35.mlp.experts.12", - "model.language_model.layers.35.mlp.experts.13", - "model.language_model.layers.35.mlp.experts.14", - "model.language_model.layers.35.mlp.experts.15", - "model.language_model.layers.35.mlp.experts.16", - "model.language_model.layers.35.mlp.experts.17", - "model.language_model.layers.35.mlp.experts.18", - "model.language_model.layers.35.mlp.experts.19", - "model.language_model.layers.35.mlp.experts.20", - "model.language_model.layers.35.mlp.experts.21", - "model.language_model.layers.35.mlp.experts.22", - "model.language_model.layers.35.mlp.experts.23", - "model.language_model.layers.35.mlp.experts.24", - "model.language_model.layers.35.mlp.experts.25", - "model.language_model.layers.35.mlp.experts.26", - "model.language_model.layers.35.mlp.experts.27", - "model.language_model.layers.35.mlp.experts.28", - "model.language_model.layers.35.mlp.experts.29", - "model.language_model.layers.35.mlp.experts.30", - "model.language_model.layers.35.mlp.experts.31", - "model.language_model.layers.35.mlp.experts.32", - "model.language_model.layers.35.mlp.experts.33", - "model.language_model.layers.35.mlp.experts.34", - "model.language_model.layers.35.mlp.experts.35", - "model.language_model.layers.35.mlp.experts.36", - "model.language_model.layers.35.mlp.experts.37", - "model.language_model.layers.35.mlp.experts.38", - "model.language_model.layers.35.mlp.experts.39", - "model.language_model.layers.35.mlp.experts.40", - "model.language_model.layers.35.mlp.experts.41", - "model.language_model.layers.35.mlp.experts.42", - "model.language_model.layers.35.mlp.experts.43", - "model.language_model.layers.35.mlp.experts.44", - "model.language_model.layers.35.mlp.experts.45", - "model.language_model.layers.35.mlp.experts.46", - "model.language_model.layers.35.mlp.experts.47", - "model.language_model.layers.35.mlp.experts.48", - "model.language_model.layers.35.mlp.experts.49", - "model.language_model.layers.35.mlp.experts.50", - "model.language_model.layers.35.mlp.experts.51", - "model.language_model.layers.35.mlp.experts.52", - "model.language_model.layers.35.mlp.experts.53", - "model.language_model.layers.35.mlp.experts.54", - "model.language_model.layers.35.mlp.experts.55", - "model.language_model.layers.35.mlp.experts.56", - "model.language_model.layers.35.mlp.experts.57", - "model.language_model.layers.35.mlp.experts.58", - "model.language_model.layers.35.mlp.experts.59", - "model.language_model.layers.35.mlp.experts.60", - "model.language_model.layers.35.mlp.experts.61", - "model.language_model.layers.35.mlp.experts.62", - "model.language_model.layers.35.mlp.experts.63", - "model.language_model.layers.35.mlp.experts.64", - "model.language_model.layers.35.mlp.experts.65", - "model.language_model.layers.35.mlp.experts.66", - "model.language_model.layers.35.mlp.experts.67", - "model.language_model.layers.35.mlp.experts.68", - "model.language_model.layers.35.mlp.experts.69", - "model.language_model.layers.35.mlp.experts.70", - "model.language_model.layers.35.mlp.experts.71", - "model.language_model.layers.35.mlp.experts.72", - "model.language_model.layers.35.mlp.experts.73", - "model.language_model.layers.35.mlp.experts.74", - "model.language_model.layers.35.mlp.experts.75", - "model.language_model.layers.35.mlp.experts.76", - "model.language_model.layers.35.mlp.experts.77", - "model.language_model.layers.35.mlp.experts.78", - "model.language_model.layers.35.mlp.experts.79", - "model.language_model.layers.35.mlp.experts.80", - "model.language_model.layers.35.mlp.experts.81", - "model.language_model.layers.35.mlp.experts.82", - "model.language_model.layers.35.mlp.experts.83", - "model.language_model.layers.35.mlp.experts.84", - "model.language_model.layers.35.mlp.experts.85", - "model.language_model.layers.35.mlp.experts.86", - "model.language_model.layers.35.mlp.experts.87", - "model.language_model.layers.35.mlp.experts.88", - "model.language_model.layers.35.mlp.experts.89", - "model.language_model.layers.35.mlp.experts.90", - "model.language_model.layers.35.mlp.experts.91", - "model.language_model.layers.35.mlp.experts.92", - "model.language_model.layers.35.mlp.experts.93", - "model.language_model.layers.35.mlp.experts.94", - "model.language_model.layers.35.mlp.experts.95", - "model.language_model.layers.35.mlp.experts.96", - "model.language_model.layers.35.mlp.experts.97", - "model.language_model.layers.35.mlp.experts.98", - "model.language_model.layers.35.mlp.experts.99", - "model.language_model.layers.35.mlp.experts.100", - "model.language_model.layers.35.mlp.experts.101", - "model.language_model.layers.35.mlp.experts.102", - "model.language_model.layers.35.mlp.experts.103", - "model.language_model.layers.35.mlp.experts.104", - "model.language_model.layers.35.mlp.experts.105", - "model.language_model.layers.35.mlp.experts.106", - "model.language_model.layers.35.mlp.experts.107", - "model.language_model.layers.35.mlp.experts.108", - "model.language_model.layers.35.mlp.experts.109", - "model.language_model.layers.35.mlp.experts.110", - "model.language_model.layers.35.mlp.experts.111", - "model.language_model.layers.35.mlp.experts.112", - "model.language_model.layers.35.mlp.experts.113", - "model.language_model.layers.35.mlp.experts.114", - "model.language_model.layers.35.mlp.experts.115", - "model.language_model.layers.35.mlp.experts.116", - "model.language_model.layers.35.mlp.experts.117", - "model.language_model.layers.35.mlp.experts.118", - "model.language_model.layers.35.mlp.experts.119", - "model.language_model.layers.35.mlp.experts.120", - "model.language_model.layers.35.mlp.experts.121", - "model.language_model.layers.35.mlp.experts.122", - "model.language_model.layers.35.mlp.experts.123", - "model.language_model.layers.35.mlp.experts.124", - "model.language_model.layers.35.mlp.experts.125", - "model.language_model.layers.35.mlp.experts.126", - "model.language_model.layers.35.mlp.experts.127", - "model.language_model.layers.35.mlp.experts.128", - "model.language_model.layers.35.mlp.experts.129", - "model.language_model.layers.35.mlp.experts.130", - "model.language_model.layers.35.mlp.experts.131", - "model.language_model.layers.35.mlp.experts.132", - "model.language_model.layers.35.mlp.experts.133", - "model.language_model.layers.35.mlp.experts.134", - "model.language_model.layers.35.mlp.experts.135", - "model.language_model.layers.35.mlp.experts.136", - "model.language_model.layers.35.mlp.experts.137", - "model.language_model.layers.35.mlp.experts.138", - "model.language_model.layers.35.mlp.experts.139", - "model.language_model.layers.35.mlp.experts.140", - "model.language_model.layers.35.mlp.experts.141", - "model.language_model.layers.35.mlp.experts.142", - "model.language_model.layers.35.mlp.experts.143", - "model.language_model.layers.35.mlp.experts.144", - "model.language_model.layers.35.mlp.experts.145", - "model.language_model.layers.35.mlp.experts.146", - "model.language_model.layers.35.mlp.experts.147", - "model.language_model.layers.35.mlp.experts.148", - "model.language_model.layers.35.mlp.experts.149", - "model.language_model.layers.35.mlp.experts.150", - "model.language_model.layers.35.mlp.experts.151", - "model.language_model.layers.35.mlp.experts.152", - "model.language_model.layers.35.mlp.experts.153", - "model.language_model.layers.35.mlp.experts.154", - "model.language_model.layers.35.mlp.experts.155", - "model.language_model.layers.35.mlp.experts.156", - "model.language_model.layers.35.mlp.experts.157", - "model.language_model.layers.35.mlp.experts.158", - "model.language_model.layers.35.mlp.experts.159", - "model.language_model.layers.35.mlp.experts.160", - "model.language_model.layers.35.mlp.experts.161", - "model.language_model.layers.35.mlp.experts.162", - "model.language_model.layers.35.mlp.experts.163", - "model.language_model.layers.35.mlp.experts.164", - "model.language_model.layers.35.mlp.experts.165", - "model.language_model.layers.35.mlp.experts.166", - "model.language_model.layers.35.mlp.experts.167", - "model.language_model.layers.35.mlp.experts.168", - "model.language_model.layers.35.mlp.experts.169", - "model.language_model.layers.35.mlp.experts.170", - "model.language_model.layers.35.mlp.experts.171", - "model.language_model.layers.35.mlp.experts.172", - "model.language_model.layers.35.mlp.experts.173", - "model.language_model.layers.35.mlp.experts.174", - "model.language_model.layers.35.mlp.experts.175", - "model.language_model.layers.35.mlp.experts.176", - "model.language_model.layers.35.mlp.experts.177", - "model.language_model.layers.35.mlp.experts.178", - "model.language_model.layers.35.mlp.experts.179", - "model.language_model.layers.35.mlp.experts.180", - "model.language_model.layers.35.mlp.experts.181", - "model.language_model.layers.35.mlp.experts.182", - "model.language_model.layers.35.mlp.experts.183", - "model.language_model.layers.35.mlp.experts.184", - "model.language_model.layers.35.mlp.experts.185", - "model.language_model.layers.35.mlp.experts.186", - "model.language_model.layers.35.mlp.experts.187", - "model.language_model.layers.35.mlp.experts.188", - "model.language_model.layers.35.mlp.experts.189", - "model.language_model.layers.35.mlp.experts.190", - "model.language_model.layers.35.mlp.experts.191", - "model.language_model.layers.35.mlp.experts.192", - "model.language_model.layers.35.mlp.experts.193", - "model.language_model.layers.35.mlp.experts.194", - "model.language_model.layers.35.mlp.experts.195", - "model.language_model.layers.35.mlp.experts.196", - "model.language_model.layers.35.mlp.experts.197", - "model.language_model.layers.35.mlp.experts.198", - "model.language_model.layers.35.mlp.experts.199", - "model.language_model.layers.35.mlp.experts.200", - "model.language_model.layers.35.mlp.experts.201", - "model.language_model.layers.35.mlp.experts.202", - "model.language_model.layers.35.mlp.experts.203", - "model.language_model.layers.35.mlp.experts.204", - "model.language_model.layers.35.mlp.experts.205", - "model.language_model.layers.35.mlp.experts.206", - "model.language_model.layers.35.mlp.experts.207", - "model.language_model.layers.35.mlp.experts.208", - "model.language_model.layers.35.mlp.experts.209", - "model.language_model.layers.35.mlp.experts.210", - "model.language_model.layers.35.mlp.experts.211", - "model.language_model.layers.35.mlp.experts.212", - "model.language_model.layers.35.mlp.experts.213", - "model.language_model.layers.35.mlp.experts.214", - "model.language_model.layers.35.mlp.experts.215", - "model.language_model.layers.35.mlp.experts.216", - "model.language_model.layers.35.mlp.experts.217", - "model.language_model.layers.35.mlp.experts.218", - "model.language_model.layers.35.mlp.experts.219", - "model.language_model.layers.35.mlp.experts.220", - "model.language_model.layers.35.mlp.experts.221", - "model.language_model.layers.35.mlp.experts.222", - "model.language_model.layers.35.mlp.experts.223", - "model.language_model.layers.35.mlp.experts.224", - "model.language_model.layers.35.mlp.experts.225", - "model.language_model.layers.35.mlp.experts.226", - "model.language_model.layers.35.mlp.experts.227", - "model.language_model.layers.35.mlp.experts.228", - "model.language_model.layers.35.mlp.experts.229", - "model.language_model.layers.35.mlp.experts.230", - "model.language_model.layers.35.mlp.experts.231", - "model.language_model.layers.35.mlp.experts.232", - "model.language_model.layers.35.mlp.experts.233", - "model.language_model.layers.35.mlp.experts.234", - "model.language_model.layers.35.mlp.experts.235", - "model.language_model.layers.35.mlp.experts.236", - "model.language_model.layers.35.mlp.experts.237", - "model.language_model.layers.35.mlp.experts.238", - "model.language_model.layers.35.mlp.experts.239", - "model.language_model.layers.35.mlp.experts.240", - "model.language_model.layers.35.mlp.experts.241", - "model.language_model.layers.35.mlp.experts.242", - "model.language_model.layers.35.mlp.experts.243", - "model.language_model.layers.35.mlp.experts.244", - "model.language_model.layers.35.mlp.experts.245", - "model.language_model.layers.35.mlp.experts.246", - "model.language_model.layers.35.mlp.experts.247", - "model.language_model.layers.35.mlp.experts.248", - "model.language_model.layers.35.mlp.experts.249", - "model.language_model.layers.35.mlp.experts.250", - "model.language_model.layers.35.mlp.experts.251", - "model.language_model.layers.35.mlp.experts.252", - "model.language_model.layers.35.mlp.experts.253", - "model.language_model.layers.35.mlp.experts.254", - "model.language_model.layers.35.mlp.experts.255", "model.language_model.layers.35.mlp.shared_expert_gate", - "model.language_model.layers.36.linear_attn", "model.language_model.layers.36.linear_attn.norm", "model.language_model.layers.36.linear_attn.in_proj_b", "model.language_model.layers.36.linear_attn.in_proj_a", "model.language_model.layers.36.mlp.gate", - "model.language_model.layers.36.mlp.experts.0", - "model.language_model.layers.36.mlp.experts.1", - "model.language_model.layers.36.mlp.experts.2", - "model.language_model.layers.36.mlp.experts.3", - "model.language_model.layers.36.mlp.experts.4", - "model.language_model.layers.36.mlp.experts.5", - "model.language_model.layers.36.mlp.experts.6", - "model.language_model.layers.36.mlp.experts.7", - "model.language_model.layers.36.mlp.experts.8", - "model.language_model.layers.36.mlp.experts.9", - "model.language_model.layers.36.mlp.experts.10", - "model.language_model.layers.36.mlp.experts.11", - "model.language_model.layers.36.mlp.experts.12", - "model.language_model.layers.36.mlp.experts.13", - "model.language_model.layers.36.mlp.experts.14", - "model.language_model.layers.36.mlp.experts.15", - "model.language_model.layers.36.mlp.experts.16", - "model.language_model.layers.36.mlp.experts.17", - "model.language_model.layers.36.mlp.experts.18", - "model.language_model.layers.36.mlp.experts.19", - "model.language_model.layers.36.mlp.experts.20", - "model.language_model.layers.36.mlp.experts.21", - "model.language_model.layers.36.mlp.experts.22", - "model.language_model.layers.36.mlp.experts.23", - "model.language_model.layers.36.mlp.experts.24", - "model.language_model.layers.36.mlp.experts.25", - "model.language_model.layers.36.mlp.experts.26", - "model.language_model.layers.36.mlp.experts.27", - "model.language_model.layers.36.mlp.experts.28", - "model.language_model.layers.36.mlp.experts.29", - "model.language_model.layers.36.mlp.experts.30", - "model.language_model.layers.36.mlp.experts.31", - "model.language_model.layers.36.mlp.experts.32", - "model.language_model.layers.36.mlp.experts.33", - "model.language_model.layers.36.mlp.experts.34", - "model.language_model.layers.36.mlp.experts.35", - "model.language_model.layers.36.mlp.experts.36", - "model.language_model.layers.36.mlp.experts.37", - "model.language_model.layers.36.mlp.experts.38", - "model.language_model.layers.36.mlp.experts.39", - "model.language_model.layers.36.mlp.experts.40", - "model.language_model.layers.36.mlp.experts.41", - "model.language_model.layers.36.mlp.experts.42", - "model.language_model.layers.36.mlp.experts.43", - "model.language_model.layers.36.mlp.experts.44", - "model.language_model.layers.36.mlp.experts.45", - "model.language_model.layers.36.mlp.experts.46", - "model.language_model.layers.36.mlp.experts.47", - "model.language_model.layers.36.mlp.experts.48", - "model.language_model.layers.36.mlp.experts.49", - "model.language_model.layers.36.mlp.experts.50", - "model.language_model.layers.36.mlp.experts.51", - "model.language_model.layers.36.mlp.experts.52", - "model.language_model.layers.36.mlp.experts.53", - "model.language_model.layers.36.mlp.experts.54", - "model.language_model.layers.36.mlp.experts.55", - "model.language_model.layers.36.mlp.experts.56", - "model.language_model.layers.36.mlp.experts.57", - "model.language_model.layers.36.mlp.experts.58", - "model.language_model.layers.36.mlp.experts.59", - "model.language_model.layers.36.mlp.experts.60", - "model.language_model.layers.36.mlp.experts.61", - "model.language_model.layers.36.mlp.experts.62", - "model.language_model.layers.36.mlp.experts.63", - "model.language_model.layers.36.mlp.experts.64", - "model.language_model.layers.36.mlp.experts.65", - "model.language_model.layers.36.mlp.experts.66", - "model.language_model.layers.36.mlp.experts.67", - "model.language_model.layers.36.mlp.experts.68", - "model.language_model.layers.36.mlp.experts.69", - "model.language_model.layers.36.mlp.experts.70", - "model.language_model.layers.36.mlp.experts.71", - "model.language_model.layers.36.mlp.experts.72", - "model.language_model.layers.36.mlp.experts.73", - "model.language_model.layers.36.mlp.experts.74", - "model.language_model.layers.36.mlp.experts.75", - "model.language_model.layers.36.mlp.experts.76", - "model.language_model.layers.36.mlp.experts.77", - "model.language_model.layers.36.mlp.experts.78", - "model.language_model.layers.36.mlp.experts.79", - "model.language_model.layers.36.mlp.experts.80", - "model.language_model.layers.36.mlp.experts.81", - "model.language_model.layers.36.mlp.experts.82", - "model.language_model.layers.36.mlp.experts.83", - "model.language_model.layers.36.mlp.experts.84", - "model.language_model.layers.36.mlp.experts.85", - "model.language_model.layers.36.mlp.experts.86", - "model.language_model.layers.36.mlp.experts.87", - "model.language_model.layers.36.mlp.experts.88", - "model.language_model.layers.36.mlp.experts.89", - "model.language_model.layers.36.mlp.experts.90", - "model.language_model.layers.36.mlp.experts.91", - "model.language_model.layers.36.mlp.experts.92", - "model.language_model.layers.36.mlp.experts.93", - "model.language_model.layers.36.mlp.experts.94", - "model.language_model.layers.36.mlp.experts.95", - "model.language_model.layers.36.mlp.experts.96", - "model.language_model.layers.36.mlp.experts.97", - "model.language_model.layers.36.mlp.experts.98", - "model.language_model.layers.36.mlp.experts.99", - "model.language_model.layers.36.mlp.experts.100", - "model.language_model.layers.36.mlp.experts.101", - "model.language_model.layers.36.mlp.experts.102", - "model.language_model.layers.36.mlp.experts.103", - "model.language_model.layers.36.mlp.experts.104", - "model.language_model.layers.36.mlp.experts.105", - "model.language_model.layers.36.mlp.experts.106", - "model.language_model.layers.36.mlp.experts.107", - "model.language_model.layers.36.mlp.experts.108", - "model.language_model.layers.36.mlp.experts.109", - "model.language_model.layers.36.mlp.experts.110", - "model.language_model.layers.36.mlp.experts.111", - "model.language_model.layers.36.mlp.experts.112", - "model.language_model.layers.36.mlp.experts.113", - "model.language_model.layers.36.mlp.experts.114", - "model.language_model.layers.36.mlp.experts.115", - "model.language_model.layers.36.mlp.experts.116", - "model.language_model.layers.36.mlp.experts.117", - "model.language_model.layers.36.mlp.experts.118", - "model.language_model.layers.36.mlp.experts.119", - "model.language_model.layers.36.mlp.experts.120", - "model.language_model.layers.36.mlp.experts.121", - "model.language_model.layers.36.mlp.experts.122", - "model.language_model.layers.36.mlp.experts.123", - "model.language_model.layers.36.mlp.experts.124", - "model.language_model.layers.36.mlp.experts.125", - "model.language_model.layers.36.mlp.experts.126", - "model.language_model.layers.36.mlp.experts.127", - "model.language_model.layers.36.mlp.experts.128", - "model.language_model.layers.36.mlp.experts.129", - "model.language_model.layers.36.mlp.experts.130", - "model.language_model.layers.36.mlp.experts.131", - "model.language_model.layers.36.mlp.experts.132", - "model.language_model.layers.36.mlp.experts.133", - "model.language_model.layers.36.mlp.experts.134", - "model.language_model.layers.36.mlp.experts.135", - "model.language_model.layers.36.mlp.experts.136", - "model.language_model.layers.36.mlp.experts.137", - "model.language_model.layers.36.mlp.experts.138", - "model.language_model.layers.36.mlp.experts.139", - "model.language_model.layers.36.mlp.experts.140", - "model.language_model.layers.36.mlp.experts.141", - "model.language_model.layers.36.mlp.experts.142", - "model.language_model.layers.36.mlp.experts.143", - "model.language_model.layers.36.mlp.experts.144", - "model.language_model.layers.36.mlp.experts.145", - "model.language_model.layers.36.mlp.experts.146", - "model.language_model.layers.36.mlp.experts.147", - "model.language_model.layers.36.mlp.experts.148", - "model.language_model.layers.36.mlp.experts.149", - "model.language_model.layers.36.mlp.experts.150", - "model.language_model.layers.36.mlp.experts.151", - "model.language_model.layers.36.mlp.experts.152", - "model.language_model.layers.36.mlp.experts.153", - "model.language_model.layers.36.mlp.experts.154", - "model.language_model.layers.36.mlp.experts.155", - "model.language_model.layers.36.mlp.experts.156", - "model.language_model.layers.36.mlp.experts.157", - "model.language_model.layers.36.mlp.experts.158", - "model.language_model.layers.36.mlp.experts.159", - "model.language_model.layers.36.mlp.experts.160", - "model.language_model.layers.36.mlp.experts.161", - "model.language_model.layers.36.mlp.experts.162", - "model.language_model.layers.36.mlp.experts.163", - "model.language_model.layers.36.mlp.experts.164", - "model.language_model.layers.36.mlp.experts.165", - "model.language_model.layers.36.mlp.experts.166", - "model.language_model.layers.36.mlp.experts.167", - "model.language_model.layers.36.mlp.experts.168", - "model.language_model.layers.36.mlp.experts.169", - "model.language_model.layers.36.mlp.experts.170", - "model.language_model.layers.36.mlp.experts.171", - "model.language_model.layers.36.mlp.experts.172", - "model.language_model.layers.36.mlp.experts.173", - "model.language_model.layers.36.mlp.experts.174", - "model.language_model.layers.36.mlp.experts.175", - "model.language_model.layers.36.mlp.experts.176", - "model.language_model.layers.36.mlp.experts.177", - "model.language_model.layers.36.mlp.experts.178", - "model.language_model.layers.36.mlp.experts.179", - "model.language_model.layers.36.mlp.experts.180", - "model.language_model.layers.36.mlp.experts.181", - "model.language_model.layers.36.mlp.experts.182", - "model.language_model.layers.36.mlp.experts.183", - "model.language_model.layers.36.mlp.experts.184", - "model.language_model.layers.36.mlp.experts.185", - "model.language_model.layers.36.mlp.experts.186", - "model.language_model.layers.36.mlp.experts.187", - "model.language_model.layers.36.mlp.experts.188", - "model.language_model.layers.36.mlp.experts.189", - "model.language_model.layers.36.mlp.experts.190", - "model.language_model.layers.36.mlp.experts.191", - "model.language_model.layers.36.mlp.experts.192", - "model.language_model.layers.36.mlp.experts.193", - "model.language_model.layers.36.mlp.experts.194", - "model.language_model.layers.36.mlp.experts.195", - "model.language_model.layers.36.mlp.experts.196", - "model.language_model.layers.36.mlp.experts.197", - "model.language_model.layers.36.mlp.experts.198", - "model.language_model.layers.36.mlp.experts.199", - "model.language_model.layers.36.mlp.experts.200", - "model.language_model.layers.36.mlp.experts.201", - "model.language_model.layers.36.mlp.experts.202", - "model.language_model.layers.36.mlp.experts.203", - "model.language_model.layers.36.mlp.experts.204", - "model.language_model.layers.36.mlp.experts.205", - "model.language_model.layers.36.mlp.experts.206", - "model.language_model.layers.36.mlp.experts.207", - "model.language_model.layers.36.mlp.experts.208", - "model.language_model.layers.36.mlp.experts.209", - "model.language_model.layers.36.mlp.experts.210", - "model.language_model.layers.36.mlp.experts.211", - "model.language_model.layers.36.mlp.experts.212", - "model.language_model.layers.36.mlp.experts.213", - "model.language_model.layers.36.mlp.experts.214", - "model.language_model.layers.36.mlp.experts.215", - "model.language_model.layers.36.mlp.experts.216", - "model.language_model.layers.36.mlp.experts.217", - "model.language_model.layers.36.mlp.experts.218", - "model.language_model.layers.36.mlp.experts.219", - "model.language_model.layers.36.mlp.experts.220", - "model.language_model.layers.36.mlp.experts.221", - "model.language_model.layers.36.mlp.experts.222", - "model.language_model.layers.36.mlp.experts.223", - "model.language_model.layers.36.mlp.experts.224", - "model.language_model.layers.36.mlp.experts.225", - "model.language_model.layers.36.mlp.experts.226", - "model.language_model.layers.36.mlp.experts.227", - "model.language_model.layers.36.mlp.experts.228", - "model.language_model.layers.36.mlp.experts.229", - "model.language_model.layers.36.mlp.experts.230", - "model.language_model.layers.36.mlp.experts.231", - "model.language_model.layers.36.mlp.experts.232", - "model.language_model.layers.36.mlp.experts.233", - "model.language_model.layers.36.mlp.experts.234", - "model.language_model.layers.36.mlp.experts.235", - "model.language_model.layers.36.mlp.experts.236", - "model.language_model.layers.36.mlp.experts.237", - "model.language_model.layers.36.mlp.experts.238", - "model.language_model.layers.36.mlp.experts.239", - "model.language_model.layers.36.mlp.experts.240", - "model.language_model.layers.36.mlp.experts.241", - "model.language_model.layers.36.mlp.experts.242", - "model.language_model.layers.36.mlp.experts.243", - "model.language_model.layers.36.mlp.experts.244", - "model.language_model.layers.36.mlp.experts.245", - "model.language_model.layers.36.mlp.experts.246", - "model.language_model.layers.36.mlp.experts.247", - "model.language_model.layers.36.mlp.experts.248", - "model.language_model.layers.36.mlp.experts.249", - "model.language_model.layers.36.mlp.experts.250", - "model.language_model.layers.36.mlp.experts.251", - "model.language_model.layers.36.mlp.experts.252", - "model.language_model.layers.36.mlp.experts.253", - "model.language_model.layers.36.mlp.experts.254", - "model.language_model.layers.36.mlp.experts.255", "model.language_model.layers.36.mlp.shared_expert_gate", - "model.language_model.layers.37.linear_attn", "model.language_model.layers.37.linear_attn.norm", "model.language_model.layers.37.linear_attn.in_proj_b", "model.language_model.layers.37.linear_attn.in_proj_a", "model.language_model.layers.37.mlp.gate", - "model.language_model.layers.37.mlp.experts.0", - "model.language_model.layers.37.mlp.experts.1", - "model.language_model.layers.37.mlp.experts.2", - "model.language_model.layers.37.mlp.experts.3", - "model.language_model.layers.37.mlp.experts.4", - "model.language_model.layers.37.mlp.experts.5", - "model.language_model.layers.37.mlp.experts.6", - "model.language_model.layers.37.mlp.experts.7", - "model.language_model.layers.37.mlp.experts.8", - "model.language_model.layers.37.mlp.experts.9", - "model.language_model.layers.37.mlp.experts.10", - "model.language_model.layers.37.mlp.experts.11", - "model.language_model.layers.37.mlp.experts.12", - "model.language_model.layers.37.mlp.experts.13", - "model.language_model.layers.37.mlp.experts.14", - "model.language_model.layers.37.mlp.experts.15", - "model.language_model.layers.37.mlp.experts.16", - "model.language_model.layers.37.mlp.experts.17", - "model.language_model.layers.37.mlp.experts.18", - "model.language_model.layers.37.mlp.experts.19", - "model.language_model.layers.37.mlp.experts.20", - "model.language_model.layers.37.mlp.experts.21", - "model.language_model.layers.37.mlp.experts.22", - "model.language_model.layers.37.mlp.experts.23", - "model.language_model.layers.37.mlp.experts.24", - "model.language_model.layers.37.mlp.experts.25", - "model.language_model.layers.37.mlp.experts.26", - "model.language_model.layers.37.mlp.experts.27", - "model.language_model.layers.37.mlp.experts.28", - "model.language_model.layers.37.mlp.experts.29", - "model.language_model.layers.37.mlp.experts.30", - "model.language_model.layers.37.mlp.experts.31", - "model.language_model.layers.37.mlp.experts.32", - "model.language_model.layers.37.mlp.experts.33", - "model.language_model.layers.37.mlp.experts.34", - "model.language_model.layers.37.mlp.experts.35", - "model.language_model.layers.37.mlp.experts.36", - "model.language_model.layers.37.mlp.experts.37", - "model.language_model.layers.37.mlp.experts.38", - "model.language_model.layers.37.mlp.experts.39", - "model.language_model.layers.37.mlp.experts.40", - "model.language_model.layers.37.mlp.experts.41", - "model.language_model.layers.37.mlp.experts.42", - "model.language_model.layers.37.mlp.experts.43", - "model.language_model.layers.37.mlp.experts.44", - "model.language_model.layers.37.mlp.experts.45", - "model.language_model.layers.37.mlp.experts.46", - "model.language_model.layers.37.mlp.experts.47", - "model.language_model.layers.37.mlp.experts.48", - "model.language_model.layers.37.mlp.experts.49", - "model.language_model.layers.37.mlp.experts.50", - "model.language_model.layers.37.mlp.experts.51", - "model.language_model.layers.37.mlp.experts.52", - "model.language_model.layers.37.mlp.experts.53", - "model.language_model.layers.37.mlp.experts.54", - "model.language_model.layers.37.mlp.experts.55", - "model.language_model.layers.37.mlp.experts.56", - "model.language_model.layers.37.mlp.experts.57", - "model.language_model.layers.37.mlp.experts.58", - "model.language_model.layers.37.mlp.experts.59", - "model.language_model.layers.37.mlp.experts.60", - "model.language_model.layers.37.mlp.experts.61", - "model.language_model.layers.37.mlp.experts.62", - "model.language_model.layers.37.mlp.experts.63", - "model.language_model.layers.37.mlp.experts.64", - "model.language_model.layers.37.mlp.experts.65", - "model.language_model.layers.37.mlp.experts.66", - "model.language_model.layers.37.mlp.experts.67", - "model.language_model.layers.37.mlp.experts.68", - "model.language_model.layers.37.mlp.experts.69", - "model.language_model.layers.37.mlp.experts.70", - "model.language_model.layers.37.mlp.experts.71", - "model.language_model.layers.37.mlp.experts.72", - "model.language_model.layers.37.mlp.experts.73", - "model.language_model.layers.37.mlp.experts.74", - "model.language_model.layers.37.mlp.experts.75", - "model.language_model.layers.37.mlp.experts.76", - "model.language_model.layers.37.mlp.experts.77", - "model.language_model.layers.37.mlp.experts.78", - "model.language_model.layers.37.mlp.experts.79", - "model.language_model.layers.37.mlp.experts.80", - "model.language_model.layers.37.mlp.experts.81", - "model.language_model.layers.37.mlp.experts.82", - "model.language_model.layers.37.mlp.experts.83", - "model.language_model.layers.37.mlp.experts.84", - "model.language_model.layers.37.mlp.experts.85", - "model.language_model.layers.37.mlp.experts.86", - "model.language_model.layers.37.mlp.experts.87", - "model.language_model.layers.37.mlp.experts.88", - "model.language_model.layers.37.mlp.experts.89", - "model.language_model.layers.37.mlp.experts.90", - "model.language_model.layers.37.mlp.experts.91", - "model.language_model.layers.37.mlp.experts.92", - "model.language_model.layers.37.mlp.experts.93", - "model.language_model.layers.37.mlp.experts.94", - "model.language_model.layers.37.mlp.experts.95", - "model.language_model.layers.37.mlp.experts.96", - "model.language_model.layers.37.mlp.experts.97", - "model.language_model.layers.37.mlp.experts.98", - "model.language_model.layers.37.mlp.experts.99", - "model.language_model.layers.37.mlp.experts.100", - "model.language_model.layers.37.mlp.experts.101", - "model.language_model.layers.37.mlp.experts.102", - "model.language_model.layers.37.mlp.experts.103", - "model.language_model.layers.37.mlp.experts.104", - "model.language_model.layers.37.mlp.experts.105", - "model.language_model.layers.37.mlp.experts.106", - "model.language_model.layers.37.mlp.experts.107", - "model.language_model.layers.37.mlp.experts.108", - "model.language_model.layers.37.mlp.experts.109", - "model.language_model.layers.37.mlp.experts.110", - "model.language_model.layers.37.mlp.experts.111", - "model.language_model.layers.37.mlp.experts.112", - "model.language_model.layers.37.mlp.experts.113", - "model.language_model.layers.37.mlp.experts.114", - "model.language_model.layers.37.mlp.experts.115", - "model.language_model.layers.37.mlp.experts.116", - "model.language_model.layers.37.mlp.experts.117", - "model.language_model.layers.37.mlp.experts.118", - "model.language_model.layers.37.mlp.experts.119", - "model.language_model.layers.37.mlp.experts.120", - "model.language_model.layers.37.mlp.experts.121", - "model.language_model.layers.37.mlp.experts.122", - "model.language_model.layers.37.mlp.experts.123", - "model.language_model.layers.37.mlp.experts.124", - "model.language_model.layers.37.mlp.experts.125", - "model.language_model.layers.37.mlp.experts.126", - "model.language_model.layers.37.mlp.experts.127", - "model.language_model.layers.37.mlp.experts.128", - "model.language_model.layers.37.mlp.experts.129", - "model.language_model.layers.37.mlp.experts.130", - "model.language_model.layers.37.mlp.experts.131", - "model.language_model.layers.37.mlp.experts.132", - "model.language_model.layers.37.mlp.experts.133", - "model.language_model.layers.37.mlp.experts.134", - "model.language_model.layers.37.mlp.experts.135", - "model.language_model.layers.37.mlp.experts.136", - "model.language_model.layers.37.mlp.experts.137", - "model.language_model.layers.37.mlp.experts.138", - "model.language_model.layers.37.mlp.experts.139", - "model.language_model.layers.37.mlp.experts.140", - "model.language_model.layers.37.mlp.experts.141", - "model.language_model.layers.37.mlp.experts.142", - "model.language_model.layers.37.mlp.experts.143", - "model.language_model.layers.37.mlp.experts.144", - "model.language_model.layers.37.mlp.experts.145", - "model.language_model.layers.37.mlp.experts.146", - "model.language_model.layers.37.mlp.experts.147", - "model.language_model.layers.37.mlp.experts.148", - "model.language_model.layers.37.mlp.experts.149", - "model.language_model.layers.37.mlp.experts.150", - "model.language_model.layers.37.mlp.experts.151", - "model.language_model.layers.37.mlp.experts.152", - "model.language_model.layers.37.mlp.experts.153", - "model.language_model.layers.37.mlp.experts.154", - "model.language_model.layers.37.mlp.experts.155", - "model.language_model.layers.37.mlp.experts.156", - "model.language_model.layers.37.mlp.experts.157", - "model.language_model.layers.37.mlp.experts.158", - "model.language_model.layers.37.mlp.experts.159", - "model.language_model.layers.37.mlp.experts.160", - "model.language_model.layers.37.mlp.experts.161", - "model.language_model.layers.37.mlp.experts.162", - "model.language_model.layers.37.mlp.experts.163", - "model.language_model.layers.37.mlp.experts.164", - "model.language_model.layers.37.mlp.experts.165", - "model.language_model.layers.37.mlp.experts.166", - "model.language_model.layers.37.mlp.experts.167", - "model.language_model.layers.37.mlp.experts.168", - "model.language_model.layers.37.mlp.experts.169", - "model.language_model.layers.37.mlp.experts.170", - "model.language_model.layers.37.mlp.experts.171", - "model.language_model.layers.37.mlp.experts.172", - "model.language_model.layers.37.mlp.experts.173", - "model.language_model.layers.37.mlp.experts.174", - "model.language_model.layers.37.mlp.experts.175", - "model.language_model.layers.37.mlp.experts.176", - "model.language_model.layers.37.mlp.experts.177", - "model.language_model.layers.37.mlp.experts.178", - "model.language_model.layers.37.mlp.experts.179", - "model.language_model.layers.37.mlp.experts.180", - "model.language_model.layers.37.mlp.experts.181", - "model.language_model.layers.37.mlp.experts.182", - "model.language_model.layers.37.mlp.experts.183", - "model.language_model.layers.37.mlp.experts.184", - "model.language_model.layers.37.mlp.experts.185", - "model.language_model.layers.37.mlp.experts.186", - "model.language_model.layers.37.mlp.experts.187", - "model.language_model.layers.37.mlp.experts.188", - "model.language_model.layers.37.mlp.experts.189", - "model.language_model.layers.37.mlp.experts.190", - "model.language_model.layers.37.mlp.experts.191", - "model.language_model.layers.37.mlp.experts.192", - "model.language_model.layers.37.mlp.experts.193", - "model.language_model.layers.37.mlp.experts.194", - "model.language_model.layers.37.mlp.experts.195", - "model.language_model.layers.37.mlp.experts.196", - "model.language_model.layers.37.mlp.experts.197", - "model.language_model.layers.37.mlp.experts.198", - "model.language_model.layers.37.mlp.experts.199", - "model.language_model.layers.37.mlp.experts.200", - "model.language_model.layers.37.mlp.experts.201", - "model.language_model.layers.37.mlp.experts.202", - "model.language_model.layers.37.mlp.experts.203", - "model.language_model.layers.37.mlp.experts.204", - "model.language_model.layers.37.mlp.experts.205", - "model.language_model.layers.37.mlp.experts.206", - "model.language_model.layers.37.mlp.experts.207", - "model.language_model.layers.37.mlp.experts.208", - "model.language_model.layers.37.mlp.experts.209", - "model.language_model.layers.37.mlp.experts.210", - "model.language_model.layers.37.mlp.experts.211", - "model.language_model.layers.37.mlp.experts.212", - "model.language_model.layers.37.mlp.experts.213", - "model.language_model.layers.37.mlp.experts.214", - "model.language_model.layers.37.mlp.experts.215", - "model.language_model.layers.37.mlp.experts.216", - "model.language_model.layers.37.mlp.experts.217", - "model.language_model.layers.37.mlp.experts.218", - "model.language_model.layers.37.mlp.experts.219", - "model.language_model.layers.37.mlp.experts.220", - "model.language_model.layers.37.mlp.experts.221", - "model.language_model.layers.37.mlp.experts.222", - "model.language_model.layers.37.mlp.experts.223", - "model.language_model.layers.37.mlp.experts.224", - "model.language_model.layers.37.mlp.experts.225", - "model.language_model.layers.37.mlp.experts.226", - "model.language_model.layers.37.mlp.experts.227", - "model.language_model.layers.37.mlp.experts.228", - "model.language_model.layers.37.mlp.experts.229", - "model.language_model.layers.37.mlp.experts.230", - "model.language_model.layers.37.mlp.experts.231", - "model.language_model.layers.37.mlp.experts.232", - "model.language_model.layers.37.mlp.experts.233", - "model.language_model.layers.37.mlp.experts.234", - "model.language_model.layers.37.mlp.experts.235", - "model.language_model.layers.37.mlp.experts.236", - "model.language_model.layers.37.mlp.experts.237", - "model.language_model.layers.37.mlp.experts.238", - "model.language_model.layers.37.mlp.experts.239", - "model.language_model.layers.37.mlp.experts.240", - "model.language_model.layers.37.mlp.experts.241", - "model.language_model.layers.37.mlp.experts.242", - "model.language_model.layers.37.mlp.experts.243", - "model.language_model.layers.37.mlp.experts.244", - "model.language_model.layers.37.mlp.experts.245", - "model.language_model.layers.37.mlp.experts.246", - "model.language_model.layers.37.mlp.experts.247", - "model.language_model.layers.37.mlp.experts.248", - "model.language_model.layers.37.mlp.experts.249", - "model.language_model.layers.37.mlp.experts.250", - "model.language_model.layers.37.mlp.experts.251", - "model.language_model.layers.37.mlp.experts.252", - "model.language_model.layers.37.mlp.experts.253", - "model.language_model.layers.37.mlp.experts.254", - "model.language_model.layers.37.mlp.experts.255", "model.language_model.layers.37.mlp.shared_expert_gate", - "model.language_model.layers.38.linear_attn", "model.language_model.layers.38.linear_attn.norm", "model.language_model.layers.38.linear_attn.in_proj_b", "model.language_model.layers.38.linear_attn.in_proj_a", "model.language_model.layers.38.mlp.gate", - "model.language_model.layers.38.mlp.experts.0", - "model.language_model.layers.38.mlp.experts.1", - "model.language_model.layers.38.mlp.experts.2", - "model.language_model.layers.38.mlp.experts.3", - "model.language_model.layers.38.mlp.experts.4", - "model.language_model.layers.38.mlp.experts.5", - "model.language_model.layers.38.mlp.experts.6", - "model.language_model.layers.38.mlp.experts.7", - "model.language_model.layers.38.mlp.experts.8", - "model.language_model.layers.38.mlp.experts.9", - "model.language_model.layers.38.mlp.experts.10", - "model.language_model.layers.38.mlp.experts.11", - "model.language_model.layers.38.mlp.experts.12", - "model.language_model.layers.38.mlp.experts.13", - "model.language_model.layers.38.mlp.experts.14", - "model.language_model.layers.38.mlp.experts.15", - "model.language_model.layers.38.mlp.experts.16", - "model.language_model.layers.38.mlp.experts.17", - "model.language_model.layers.38.mlp.experts.18", - "model.language_model.layers.38.mlp.experts.19", - "model.language_model.layers.38.mlp.experts.20", - "model.language_model.layers.38.mlp.experts.21", - "model.language_model.layers.38.mlp.experts.22", - "model.language_model.layers.38.mlp.experts.23", - "model.language_model.layers.38.mlp.experts.24", - "model.language_model.layers.38.mlp.experts.25", - "model.language_model.layers.38.mlp.experts.26", - "model.language_model.layers.38.mlp.experts.27", - "model.language_model.layers.38.mlp.experts.28", - "model.language_model.layers.38.mlp.experts.29", - "model.language_model.layers.38.mlp.experts.30", - "model.language_model.layers.38.mlp.experts.31", - "model.language_model.layers.38.mlp.experts.32", - "model.language_model.layers.38.mlp.experts.33", - "model.language_model.layers.38.mlp.experts.34", - "model.language_model.layers.38.mlp.experts.35", - "model.language_model.layers.38.mlp.experts.36", - "model.language_model.layers.38.mlp.experts.37", - "model.language_model.layers.38.mlp.experts.38", - "model.language_model.layers.38.mlp.experts.39", - "model.language_model.layers.38.mlp.experts.40", - "model.language_model.layers.38.mlp.experts.41", - "model.language_model.layers.38.mlp.experts.42", - "model.language_model.layers.38.mlp.experts.43", - "model.language_model.layers.38.mlp.experts.44", - "model.language_model.layers.38.mlp.experts.45", - "model.language_model.layers.38.mlp.experts.46", - "model.language_model.layers.38.mlp.experts.47", - "model.language_model.layers.38.mlp.experts.48", - "model.language_model.layers.38.mlp.experts.49", - "model.language_model.layers.38.mlp.experts.50", - "model.language_model.layers.38.mlp.experts.51", - "model.language_model.layers.38.mlp.experts.52", - "model.language_model.layers.38.mlp.experts.53", - "model.language_model.layers.38.mlp.experts.54", - "model.language_model.layers.38.mlp.experts.55", - "model.language_model.layers.38.mlp.experts.56", - "model.language_model.layers.38.mlp.experts.57", - "model.language_model.layers.38.mlp.experts.58", - "model.language_model.layers.38.mlp.experts.59", - "model.language_model.layers.38.mlp.experts.60", - "model.language_model.layers.38.mlp.experts.61", - "model.language_model.layers.38.mlp.experts.62", - "model.language_model.layers.38.mlp.experts.63", - "model.language_model.layers.38.mlp.experts.64", - "model.language_model.layers.38.mlp.experts.65", - "model.language_model.layers.38.mlp.experts.66", - "model.language_model.layers.38.mlp.experts.67", - "model.language_model.layers.38.mlp.experts.68", - "model.language_model.layers.38.mlp.experts.69", - "model.language_model.layers.38.mlp.experts.70", - "model.language_model.layers.38.mlp.experts.71", - "model.language_model.layers.38.mlp.experts.72", - "model.language_model.layers.38.mlp.experts.73", - "model.language_model.layers.38.mlp.experts.74", - "model.language_model.layers.38.mlp.experts.75", - "model.language_model.layers.38.mlp.experts.76", - "model.language_model.layers.38.mlp.experts.77", - "model.language_model.layers.38.mlp.experts.78", - "model.language_model.layers.38.mlp.experts.79", - "model.language_model.layers.38.mlp.experts.80", - "model.language_model.layers.38.mlp.experts.81", - "model.language_model.layers.38.mlp.experts.82", - "model.language_model.layers.38.mlp.experts.83", - "model.language_model.layers.38.mlp.experts.84", - "model.language_model.layers.38.mlp.experts.85", - "model.language_model.layers.38.mlp.experts.86", - "model.language_model.layers.38.mlp.experts.87", - "model.language_model.layers.38.mlp.experts.88", - "model.language_model.layers.38.mlp.experts.89", - "model.language_model.layers.38.mlp.experts.90", - "model.language_model.layers.38.mlp.experts.91", - "model.language_model.layers.38.mlp.experts.92", - "model.language_model.layers.38.mlp.experts.93", - "model.language_model.layers.38.mlp.experts.94", - "model.language_model.layers.38.mlp.experts.95", - "model.language_model.layers.38.mlp.experts.96", - "model.language_model.layers.38.mlp.experts.97", - "model.language_model.layers.38.mlp.experts.98", - "model.language_model.layers.38.mlp.experts.99", - "model.language_model.layers.38.mlp.experts.100", - "model.language_model.layers.38.mlp.experts.101", - "model.language_model.layers.38.mlp.experts.102", - "model.language_model.layers.38.mlp.experts.103", - "model.language_model.layers.38.mlp.experts.104", - "model.language_model.layers.38.mlp.experts.105", - "model.language_model.layers.38.mlp.experts.106", - "model.language_model.layers.38.mlp.experts.107", - "model.language_model.layers.38.mlp.experts.108", - "model.language_model.layers.38.mlp.experts.109", - "model.language_model.layers.38.mlp.experts.110", - "model.language_model.layers.38.mlp.experts.111", - "model.language_model.layers.38.mlp.experts.112", - "model.language_model.layers.38.mlp.experts.113", - "model.language_model.layers.38.mlp.experts.114", - "model.language_model.layers.38.mlp.experts.115", - "model.language_model.layers.38.mlp.experts.116", - "model.language_model.layers.38.mlp.experts.117", - "model.language_model.layers.38.mlp.experts.118", - "model.language_model.layers.38.mlp.experts.119", - "model.language_model.layers.38.mlp.experts.120", - "model.language_model.layers.38.mlp.experts.121", - "model.language_model.layers.38.mlp.experts.122", - "model.language_model.layers.38.mlp.experts.123", - "model.language_model.layers.38.mlp.experts.124", - "model.language_model.layers.38.mlp.experts.125", - "model.language_model.layers.38.mlp.experts.126", - "model.language_model.layers.38.mlp.experts.127", - "model.language_model.layers.38.mlp.experts.128", - "model.language_model.layers.38.mlp.experts.129", - "model.language_model.layers.38.mlp.experts.130", - "model.language_model.layers.38.mlp.experts.131", - "model.language_model.layers.38.mlp.experts.132", - "model.language_model.layers.38.mlp.experts.133", - "model.language_model.layers.38.mlp.experts.134", - "model.language_model.layers.38.mlp.experts.135", - "model.language_model.layers.38.mlp.experts.136", - "model.language_model.layers.38.mlp.experts.137", - "model.language_model.layers.38.mlp.experts.138", - "model.language_model.layers.38.mlp.experts.139", - "model.language_model.layers.38.mlp.experts.140", - "model.language_model.layers.38.mlp.experts.141", - "model.language_model.layers.38.mlp.experts.142", - "model.language_model.layers.38.mlp.experts.143", - "model.language_model.layers.38.mlp.experts.144", - "model.language_model.layers.38.mlp.experts.145", - "model.language_model.layers.38.mlp.experts.146", - "model.language_model.layers.38.mlp.experts.147", - "model.language_model.layers.38.mlp.experts.148", - "model.language_model.layers.38.mlp.experts.149", - "model.language_model.layers.38.mlp.experts.150", - "model.language_model.layers.38.mlp.experts.151", - "model.language_model.layers.38.mlp.experts.152", - "model.language_model.layers.38.mlp.experts.153", - "model.language_model.layers.38.mlp.experts.154", - "model.language_model.layers.38.mlp.experts.155", - "model.language_model.layers.38.mlp.experts.156", - "model.language_model.layers.38.mlp.experts.157", - "model.language_model.layers.38.mlp.experts.158", - "model.language_model.layers.38.mlp.experts.159", - "model.language_model.layers.38.mlp.experts.160", - "model.language_model.layers.38.mlp.experts.161", - "model.language_model.layers.38.mlp.experts.162", - "model.language_model.layers.38.mlp.experts.163", - "model.language_model.layers.38.mlp.experts.164", - "model.language_model.layers.38.mlp.experts.165", - "model.language_model.layers.38.mlp.experts.166", - "model.language_model.layers.38.mlp.experts.167", - "model.language_model.layers.38.mlp.experts.168", - "model.language_model.layers.38.mlp.experts.169", - "model.language_model.layers.38.mlp.experts.170", - "model.language_model.layers.38.mlp.experts.171", - "model.language_model.layers.38.mlp.experts.172", - "model.language_model.layers.38.mlp.experts.173", - "model.language_model.layers.38.mlp.experts.174", - "model.language_model.layers.38.mlp.experts.175", - "model.language_model.layers.38.mlp.experts.176", - "model.language_model.layers.38.mlp.experts.177", - "model.language_model.layers.38.mlp.experts.178", - "model.language_model.layers.38.mlp.experts.179", - "model.language_model.layers.38.mlp.experts.180", - "model.language_model.layers.38.mlp.experts.181", - "model.language_model.layers.38.mlp.experts.182", - "model.language_model.layers.38.mlp.experts.183", - "model.language_model.layers.38.mlp.experts.184", - "model.language_model.layers.38.mlp.experts.185", - "model.language_model.layers.38.mlp.experts.186", - "model.language_model.layers.38.mlp.experts.187", - "model.language_model.layers.38.mlp.experts.188", - "model.language_model.layers.38.mlp.experts.189", - "model.language_model.layers.38.mlp.experts.190", - "model.language_model.layers.38.mlp.experts.191", - "model.language_model.layers.38.mlp.experts.192", - "model.language_model.layers.38.mlp.experts.193", - "model.language_model.layers.38.mlp.experts.194", - "model.language_model.layers.38.mlp.experts.195", - "model.language_model.layers.38.mlp.experts.196", - "model.language_model.layers.38.mlp.experts.197", - "model.language_model.layers.38.mlp.experts.198", - "model.language_model.layers.38.mlp.experts.199", - "model.language_model.layers.38.mlp.experts.200", - "model.language_model.layers.38.mlp.experts.201", - "model.language_model.layers.38.mlp.experts.202", - "model.language_model.layers.38.mlp.experts.203", - "model.language_model.layers.38.mlp.experts.204", - "model.language_model.layers.38.mlp.experts.205", - "model.language_model.layers.38.mlp.experts.206", - "model.language_model.layers.38.mlp.experts.207", - "model.language_model.layers.38.mlp.experts.208", - "model.language_model.layers.38.mlp.experts.209", - "model.language_model.layers.38.mlp.experts.210", - "model.language_model.layers.38.mlp.experts.211", - "model.language_model.layers.38.mlp.experts.212", - "model.language_model.layers.38.mlp.experts.213", - "model.language_model.layers.38.mlp.experts.214", - "model.language_model.layers.38.mlp.experts.215", - "model.language_model.layers.38.mlp.experts.216", - "model.language_model.layers.38.mlp.experts.217", - "model.language_model.layers.38.mlp.experts.218", - "model.language_model.layers.38.mlp.experts.219", - "model.language_model.layers.38.mlp.experts.220", - "model.language_model.layers.38.mlp.experts.221", - "model.language_model.layers.38.mlp.experts.222", - "model.language_model.layers.38.mlp.experts.223", - "model.language_model.layers.38.mlp.experts.224", - "model.language_model.layers.38.mlp.experts.225", - "model.language_model.layers.38.mlp.experts.226", - "model.language_model.layers.38.mlp.experts.227", - "model.language_model.layers.38.mlp.experts.228", - "model.language_model.layers.38.mlp.experts.229", - "model.language_model.layers.38.mlp.experts.230", - "model.language_model.layers.38.mlp.experts.231", - "model.language_model.layers.38.mlp.experts.232", - "model.language_model.layers.38.mlp.experts.233", - "model.language_model.layers.38.mlp.experts.234", - "model.language_model.layers.38.mlp.experts.235", - "model.language_model.layers.38.mlp.experts.236", - "model.language_model.layers.38.mlp.experts.237", - "model.language_model.layers.38.mlp.experts.238", - "model.language_model.layers.38.mlp.experts.239", - "model.language_model.layers.38.mlp.experts.240", - "model.language_model.layers.38.mlp.experts.241", - "model.language_model.layers.38.mlp.experts.242", - "model.language_model.layers.38.mlp.experts.243", - "model.language_model.layers.38.mlp.experts.244", - "model.language_model.layers.38.mlp.experts.245", - "model.language_model.layers.38.mlp.experts.246", - "model.language_model.layers.38.mlp.experts.247", - "model.language_model.layers.38.mlp.experts.248", - "model.language_model.layers.38.mlp.experts.249", - "model.language_model.layers.38.mlp.experts.250", - "model.language_model.layers.38.mlp.experts.251", - "model.language_model.layers.38.mlp.experts.252", - "model.language_model.layers.38.mlp.experts.253", - "model.language_model.layers.38.mlp.experts.254", - "model.language_model.layers.38.mlp.experts.255", "model.language_model.layers.38.mlp.shared_expert_gate", "model.language_model.layers.39.mlp.gate", - "model.language_model.layers.39.mlp.experts.0", - "model.language_model.layers.39.mlp.experts.1", - "model.language_model.layers.39.mlp.experts.2", - "model.language_model.layers.39.mlp.experts.3", - "model.language_model.layers.39.mlp.experts.4", - "model.language_model.layers.39.mlp.experts.5", - "model.language_model.layers.39.mlp.experts.6", - "model.language_model.layers.39.mlp.experts.7", - "model.language_model.layers.39.mlp.experts.8", - "model.language_model.layers.39.mlp.experts.9", - "model.language_model.layers.39.mlp.experts.10", - "model.language_model.layers.39.mlp.experts.11", - "model.language_model.layers.39.mlp.experts.12", - "model.language_model.layers.39.mlp.experts.13", - "model.language_model.layers.39.mlp.experts.14", - "model.language_model.layers.39.mlp.experts.15", - "model.language_model.layers.39.mlp.experts.16", - "model.language_model.layers.39.mlp.experts.17", - "model.language_model.layers.39.mlp.experts.18", - "model.language_model.layers.39.mlp.experts.19", - "model.language_model.layers.39.mlp.experts.20", - "model.language_model.layers.39.mlp.experts.21", - "model.language_model.layers.39.mlp.experts.22", - "model.language_model.layers.39.mlp.experts.23", - "model.language_model.layers.39.mlp.experts.24", - "model.language_model.layers.39.mlp.experts.25", - "model.language_model.layers.39.mlp.experts.26", - "model.language_model.layers.39.mlp.experts.27", - "model.language_model.layers.39.mlp.experts.28", - "model.language_model.layers.39.mlp.experts.29", - "model.language_model.layers.39.mlp.experts.30", - "model.language_model.layers.39.mlp.experts.31", - "model.language_model.layers.39.mlp.experts.32", - "model.language_model.layers.39.mlp.experts.33", - "model.language_model.layers.39.mlp.experts.34", - "model.language_model.layers.39.mlp.experts.35", - "model.language_model.layers.39.mlp.experts.36", - "model.language_model.layers.39.mlp.experts.37", - "model.language_model.layers.39.mlp.experts.38", - "model.language_model.layers.39.mlp.experts.39", - "model.language_model.layers.39.mlp.experts.40", - "model.language_model.layers.39.mlp.experts.41", - "model.language_model.layers.39.mlp.experts.42", - "model.language_model.layers.39.mlp.experts.43", - "model.language_model.layers.39.mlp.experts.44", - "model.language_model.layers.39.mlp.experts.45", - "model.language_model.layers.39.mlp.experts.46", - "model.language_model.layers.39.mlp.experts.47", - "model.language_model.layers.39.mlp.experts.48", - "model.language_model.layers.39.mlp.experts.49", - "model.language_model.layers.39.mlp.experts.50", - "model.language_model.layers.39.mlp.experts.51", - "model.language_model.layers.39.mlp.experts.52", - "model.language_model.layers.39.mlp.experts.53", - "model.language_model.layers.39.mlp.experts.54", - "model.language_model.layers.39.mlp.experts.55", - "model.language_model.layers.39.mlp.experts.56", - "model.language_model.layers.39.mlp.experts.57", - "model.language_model.layers.39.mlp.experts.58", - "model.language_model.layers.39.mlp.experts.59", - "model.language_model.layers.39.mlp.experts.60", - "model.language_model.layers.39.mlp.experts.61", - "model.language_model.layers.39.mlp.experts.62", - "model.language_model.layers.39.mlp.experts.63", - "model.language_model.layers.39.mlp.experts.64", - "model.language_model.layers.39.mlp.experts.65", - "model.language_model.layers.39.mlp.experts.66", - "model.language_model.layers.39.mlp.experts.67", - "model.language_model.layers.39.mlp.experts.68", - "model.language_model.layers.39.mlp.experts.69", - "model.language_model.layers.39.mlp.experts.70", - "model.language_model.layers.39.mlp.experts.71", - "model.language_model.layers.39.mlp.experts.72", - "model.language_model.layers.39.mlp.experts.73", - "model.language_model.layers.39.mlp.experts.74", - "model.language_model.layers.39.mlp.experts.75", - "model.language_model.layers.39.mlp.experts.76", - "model.language_model.layers.39.mlp.experts.77", - "model.language_model.layers.39.mlp.experts.78", - "model.language_model.layers.39.mlp.experts.79", - "model.language_model.layers.39.mlp.experts.80", - "model.language_model.layers.39.mlp.experts.81", - "model.language_model.layers.39.mlp.experts.82", - "model.language_model.layers.39.mlp.experts.83", - "model.language_model.layers.39.mlp.experts.84", - "model.language_model.layers.39.mlp.experts.85", - "model.language_model.layers.39.mlp.experts.86", - "model.language_model.layers.39.mlp.experts.87", - "model.language_model.layers.39.mlp.experts.88", - "model.language_model.layers.39.mlp.experts.89", - "model.language_model.layers.39.mlp.experts.90", - "model.language_model.layers.39.mlp.experts.91", - "model.language_model.layers.39.mlp.experts.92", - "model.language_model.layers.39.mlp.experts.93", - "model.language_model.layers.39.mlp.experts.94", - "model.language_model.layers.39.mlp.experts.95", - "model.language_model.layers.39.mlp.experts.96", - "model.language_model.layers.39.mlp.experts.97", - "model.language_model.layers.39.mlp.experts.98", - "model.language_model.layers.39.mlp.experts.99", - "model.language_model.layers.39.mlp.experts.100", - "model.language_model.layers.39.mlp.experts.101", - "model.language_model.layers.39.mlp.experts.102", - "model.language_model.layers.39.mlp.experts.103", - "model.language_model.layers.39.mlp.experts.104", - "model.language_model.layers.39.mlp.experts.105", - "model.language_model.layers.39.mlp.experts.106", - "model.language_model.layers.39.mlp.experts.107", - "model.language_model.layers.39.mlp.experts.108", - "model.language_model.layers.39.mlp.experts.109", - "model.language_model.layers.39.mlp.experts.110", - "model.language_model.layers.39.mlp.experts.111", - "model.language_model.layers.39.mlp.experts.112", - "model.language_model.layers.39.mlp.experts.113", - "model.language_model.layers.39.mlp.experts.114", - "model.language_model.layers.39.mlp.experts.115", - "model.language_model.layers.39.mlp.experts.116", - "model.language_model.layers.39.mlp.experts.117", - "model.language_model.layers.39.mlp.experts.118", - "model.language_model.layers.39.mlp.experts.119", - "model.language_model.layers.39.mlp.experts.120", - "model.language_model.layers.39.mlp.experts.121", - "model.language_model.layers.39.mlp.experts.122", - "model.language_model.layers.39.mlp.experts.123", - "model.language_model.layers.39.mlp.experts.124", - "model.language_model.layers.39.mlp.experts.125", - "model.language_model.layers.39.mlp.experts.126", - "model.language_model.layers.39.mlp.experts.127", - "model.language_model.layers.39.mlp.experts.128", - "model.language_model.layers.39.mlp.experts.129", - "model.language_model.layers.39.mlp.experts.130", - "model.language_model.layers.39.mlp.experts.131", - "model.language_model.layers.39.mlp.experts.132", - "model.language_model.layers.39.mlp.experts.133", - "model.language_model.layers.39.mlp.experts.134", - "model.language_model.layers.39.mlp.experts.135", - "model.language_model.layers.39.mlp.experts.136", - "model.language_model.layers.39.mlp.experts.137", - "model.language_model.layers.39.mlp.experts.138", - "model.language_model.layers.39.mlp.experts.139", - "model.language_model.layers.39.mlp.experts.140", - "model.language_model.layers.39.mlp.experts.141", - "model.language_model.layers.39.mlp.experts.142", - "model.language_model.layers.39.mlp.experts.143", - "model.language_model.layers.39.mlp.experts.144", - "model.language_model.layers.39.mlp.experts.145", - "model.language_model.layers.39.mlp.experts.146", - "model.language_model.layers.39.mlp.experts.147", - "model.language_model.layers.39.mlp.experts.148", - "model.language_model.layers.39.mlp.experts.149", - "model.language_model.layers.39.mlp.experts.150", - "model.language_model.layers.39.mlp.experts.151", - "model.language_model.layers.39.mlp.experts.152", - "model.language_model.layers.39.mlp.experts.153", - "model.language_model.layers.39.mlp.experts.154", - "model.language_model.layers.39.mlp.experts.155", - "model.language_model.layers.39.mlp.experts.156", - "model.language_model.layers.39.mlp.experts.157", - "model.language_model.layers.39.mlp.experts.158", - "model.language_model.layers.39.mlp.experts.159", - "model.language_model.layers.39.mlp.experts.160", - "model.language_model.layers.39.mlp.experts.161", - "model.language_model.layers.39.mlp.experts.162", - "model.language_model.layers.39.mlp.experts.163", - "model.language_model.layers.39.mlp.experts.164", - "model.language_model.layers.39.mlp.experts.165", - "model.language_model.layers.39.mlp.experts.166", - "model.language_model.layers.39.mlp.experts.167", - "model.language_model.layers.39.mlp.experts.168", - "model.language_model.layers.39.mlp.experts.169", - "model.language_model.layers.39.mlp.experts.170", - "model.language_model.layers.39.mlp.experts.171", - "model.language_model.layers.39.mlp.experts.172", - "model.language_model.layers.39.mlp.experts.173", - "model.language_model.layers.39.mlp.experts.174", - "model.language_model.layers.39.mlp.experts.175", - "model.language_model.layers.39.mlp.experts.176", - "model.language_model.layers.39.mlp.experts.177", - "model.language_model.layers.39.mlp.experts.178", - "model.language_model.layers.39.mlp.experts.179", - "model.language_model.layers.39.mlp.experts.180", - "model.language_model.layers.39.mlp.experts.181", - "model.language_model.layers.39.mlp.experts.182", - "model.language_model.layers.39.mlp.experts.183", - "model.language_model.layers.39.mlp.experts.184", - "model.language_model.layers.39.mlp.experts.185", - "model.language_model.layers.39.mlp.experts.186", - "model.language_model.layers.39.mlp.experts.187", - "model.language_model.layers.39.mlp.experts.188", - "model.language_model.layers.39.mlp.experts.189", - "model.language_model.layers.39.mlp.experts.190", - "model.language_model.layers.39.mlp.experts.191", - "model.language_model.layers.39.mlp.experts.192", - "model.language_model.layers.39.mlp.experts.193", - "model.language_model.layers.39.mlp.experts.194", - "model.language_model.layers.39.mlp.experts.195", - "model.language_model.layers.39.mlp.experts.196", - "model.language_model.layers.39.mlp.experts.197", - "model.language_model.layers.39.mlp.experts.198", - "model.language_model.layers.39.mlp.experts.199", - "model.language_model.layers.39.mlp.experts.200", - "model.language_model.layers.39.mlp.experts.201", - "model.language_model.layers.39.mlp.experts.202", - "model.language_model.layers.39.mlp.experts.203", - "model.language_model.layers.39.mlp.experts.204", - "model.language_model.layers.39.mlp.experts.205", - "model.language_model.layers.39.mlp.experts.206", - "model.language_model.layers.39.mlp.experts.207", - "model.language_model.layers.39.mlp.experts.208", - "model.language_model.layers.39.mlp.experts.209", - "model.language_model.layers.39.mlp.experts.210", - "model.language_model.layers.39.mlp.experts.211", - "model.language_model.layers.39.mlp.experts.212", - "model.language_model.layers.39.mlp.experts.213", - "model.language_model.layers.39.mlp.experts.214", - "model.language_model.layers.39.mlp.experts.215", - "model.language_model.layers.39.mlp.experts.216", - "model.language_model.layers.39.mlp.experts.217", - "model.language_model.layers.39.mlp.experts.218", - "model.language_model.layers.39.mlp.experts.219", - "model.language_model.layers.39.mlp.experts.220", - "model.language_model.layers.39.mlp.experts.221", - "model.language_model.layers.39.mlp.experts.222", - "model.language_model.layers.39.mlp.experts.223", - "model.language_model.layers.39.mlp.experts.224", - "model.language_model.layers.39.mlp.experts.225", - "model.language_model.layers.39.mlp.experts.226", - "model.language_model.layers.39.mlp.experts.227", - "model.language_model.layers.39.mlp.experts.228", - "model.language_model.layers.39.mlp.experts.229", - "model.language_model.layers.39.mlp.experts.230", - "model.language_model.layers.39.mlp.experts.231", - "model.language_model.layers.39.mlp.experts.232", - "model.language_model.layers.39.mlp.experts.233", - "model.language_model.layers.39.mlp.experts.234", - "model.language_model.layers.39.mlp.experts.235", - "model.language_model.layers.39.mlp.experts.236", - "model.language_model.layers.39.mlp.experts.237", - "model.language_model.layers.39.mlp.experts.238", - "model.language_model.layers.39.mlp.experts.239", - "model.language_model.layers.39.mlp.experts.240", - "model.language_model.layers.39.mlp.experts.241", - "model.language_model.layers.39.mlp.experts.242", - "model.language_model.layers.39.mlp.experts.243", - "model.language_model.layers.39.mlp.experts.244", - "model.language_model.layers.39.mlp.experts.245", - "model.language_model.layers.39.mlp.experts.246", - "model.language_model.layers.39.mlp.experts.247", - "model.language_model.layers.39.mlp.experts.248", - "model.language_model.layers.39.mlp.experts.249", - "model.language_model.layers.39.mlp.experts.250", - "model.language_model.layers.39.mlp.experts.251", - "model.language_model.layers.39.mlp.experts.252", - "model.language_model.layers.39.mlp.experts.253", - "model.language_model.layers.39.mlp.experts.254", - "model.language_model.layers.39.mlp.experts.255", - "model.language_model.layers.39.mlp.shared_expert_gate" + "model.language_model.layers.39.mlp.shared_expert_gate", + "re:.*\\.mlp\\.experts\\.\\d+$", + "re:.*\\.linear_attn$" ], "kv_cache_scheme": { "actorder": null, @@ -10653,6 +385,25 @@ "type": "float", "zp_dtype": null }, + "packed_modules_mapping": { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj" + ], + "gate_up_proj": [ + "gate_proj", + "up_proj" + ], + "in_proj_qkvz": [ + "in_proj_qkv", + "in_proj_z" + ], + "in_proj_ba": [ + "in_proj_b", + "in_proj_a" + ] + }, "quant_method": "compressed-tensors", "quantization_status": "compressed", "sparsity_config": {}, @@ -10772,4 +523,4 @@ }, "vision_end_token_id": 248054, "vision_start_token_id": 248053 -} \ No newline at end of file +}