| {"cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9oZWFkLmxpbmVhci5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9oZWFkLmxpbmVhci5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2Rlcl9ub3JtLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2Rlcl9ub3JtLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC52YWx1ZS5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC52YWx1ZS5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC5rZXkuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC5rZXkua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC5vdXQuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC5vdXQua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC5xdWVyeS5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC5xdWVyeS5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk1scEJsb2NrXzAuRGVuc2VfMC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk1scEJsb2NrXzAuRGVuc2VfMC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk1scEJsb2NrXzAuRGVuc2VfMS5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk1scEJsb2NrXzAuRGVuc2VfMS5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLkxheWVyTm9ybV8wLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLkxheWVyTm9ybV8wLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLkxheWVyTm9ybV8xLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLkxheWVyTm9ybV8xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuZW1iZWRkaW5nLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuZW1iZWRkaW5nLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuaGVhZC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcuaGVhZC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5pbWcucG9zX2VtYmVkZGluZy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5sbG0uZW1iZWRkZXIuaW5wdXRfZW1iZWRkaW5nLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [\"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5sbG0uZmluYWxfbm9ybS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLm1scC5nYXRpbmdfZWluc3VtLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLm1scC5saW5lYXIudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLmF0dG4uYXR0bl92ZWNfZWluc3VtLncudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLmF0dG4ua3ZfZWluc3VtLncudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLmF0dG4ucV9laW5zdW0udy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLnByZV9hdHRlbnRpb25fbm9ybS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLnByZV9mZndfbm9ybS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLmFjdGlvbl9wcm9qLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLmFjdGlvbl9wcm9qLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLmNsc190b2tlbi52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLnN0YXRlX3Byb2ouYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMucV9uZXR3b3JrLnN0YXRlX3Byb2oua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfaGVhZC5saW5lYXIuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfaGVhZC5saW5lYXIua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJfbm9ybS5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJfbm9ybS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5MYXllck5vcm1fMC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5MYXllck5vcm1fMC5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5MYXllck5vcm1fMS5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5MYXllck5vcm1fMS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5NbHBCbG9ja18wLkRlbnNlXzAuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5NbHBCbG9ja18wLkRlbnNlXzAua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5NbHBCbG9ja18wLkRlbnNlXzEuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5NbHBCbG9ja18wLkRlbnNlXzEua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5NdWx0aUhlYWREb3RQcm9kdWN0QXR0ZW50aW9uXzAua2V5LmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5NdWx0aUhlYWREb3RQcm9kdWN0QXR0ZW50aW9uXzAua2V5Lmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5NdWx0aUhlYWREb3RQcm9kdWN0QXR0ZW50aW9uXzAub3V0LmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5NdWx0aUhlYWREb3RQcm9kdWN0QXR0ZW50aW9uXzAub3V0Lmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5NdWx0aUhlYWREb3RQcm9kdWN0QXR0ZW50aW9uXzAucXVlcnkuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5NdWx0aUhlYWREb3RQcm9kdWN0QXR0ZW50aW9uXzAucXVlcnkua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5NdWx0aUhlYWREb3RQcm9kdWN0QXR0ZW50aW9uXzAudmFsdWUuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLlRyYW5zZm9ybWVyLmVuY29kZXJibG9jay5NdWx0aUhlYWREb3RQcm9kdWN0QXR0ZW50aW9uXzAudmFsdWUua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLmVtYmVkZGluZy5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLmVtYmVkZGluZy5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLmhlYWQuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLmhlYWQua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEuaW1nLnBvc19lbWJlZGRpbmcudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEubGxtLmVtYmVkZGVyLmlucHV0X2VtYmVkZGluZy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [\"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEubGxtLmZpbmFsX25vcm0uc2NhbGUudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEubGxtLmxheWVycy5hdHRuLmF0dG5fdmVjX2VpbnN1bS53LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEubGxtLmxheWVycy5hdHRuLmt2X2VpbnN1bS53LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEubGxtLmxheWVycy5hdHRuLnFfZWluc3VtLncudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEubGxtLmxheWVycy5tbHAuZ2F0aW5nX2VpbnN1bS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEubGxtLmxheWVycy5tbHAubGluZWFyLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEubGxtLmxheWVycy5wcmVfYXR0ZW50aW9uX25vcm0uc2NhbGUudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5QYWxpR2VtbWEubGxtLmxheWVycy5wcmVfZmZ3X25vcm0uc2NhbGUudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5hY3Rpb25fcHJvai5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5hY3Rpb25fcHJvai5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5jbHNfdG9rZW4udmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5zdGF0ZV9wcm9qLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}","cGFyYW1zLmNyaXRpYy5wYXJhbXMudGFyZ2V0X3FfbmV0d29yay5zdGF0ZV9wcm9qLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 16], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 8}, {\"id\": 1}, {\"id\": 9}, {\"id\": 2}, {\"id\": 10}, {\"id\": 3}, {\"id\": 11}, {\"id\": 4}, {\"id\": 12}, {\"id\": 5}, {\"id\": 13}, {\"id\": 6}, {\"id\": 14}, {\"id\": 7}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 24}, {\"id\": 17}, {\"id\": 25}, {\"id\": 18}, {\"id\": 26}, {\"id\": 19}, {\"id\": 27}, {\"id\": 20}, {\"id\": 28}, {\"id\": 21}, {\"id\": 29}, {\"id\": 22}, {\"id\": 30}, {\"id\": 23}, {\"id\": 31}], [{\"id\": 32}, {\"id\": 40}, {\"id\": 33}, {\"id\": 41}, {\"id\": 34}, {\"id\": 42}, {\"id\": 35}, {\"id\": 43}, {\"id\": 36}, {\"id\": 44}, {\"id\": 37}, {\"id\": 45}, {\"id\": 38}, {\"id\": 46}, {\"id\": 39}, {\"id\": 47}], [{\"id\": 48}, {\"id\": 56}, {\"id\": 49}, {\"id\": 57}, {\"id\": 50}, {\"id\": 58}, {\"id\": 51}, {\"id\": 59}, {\"id\": 52}, {\"id\": 60}, {\"id\": 53}, {\"id\": 61}, {\"id\": 54}, {\"id\": 62}, {\"id\": 55}, {\"id\": 63}]]}}"} |