| {"cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2Rlcl9ub3JtLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2Rlcl9ub3JtLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC52YWx1ZS5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC52YWx1ZS5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC5rZXkuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC5rZXkua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC5vdXQuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC5vdXQua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC5xdWVyeS5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk11bHRpSGVhZERvdFByb2R1Y3RBdHRlbnRpb25fMC5xdWVyeS5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null, null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk1scEJsb2NrXzAuRGVuc2VfMC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk1scEJsb2NrXzAuRGVuc2VfMC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk1scEJsb2NrXzAuRGVuc2VfMS5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLk1scEJsb2NrXzAuRGVuc2VfMS5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLkxheWVyTm9ybV8wLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLkxheWVyTm9ybV8wLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLkxheWVyTm9ybV8xLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuVHJhbnNmb3JtZXIuZW5jb2RlcmJsb2NrLkxheWVyTm9ybV8xLnNjYWxlLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuZW1iZWRkaW5nLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuZW1iZWRkaW5nLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuaGVhZC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcuaGVhZC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5pbWcucG9zX2VtYmVkZGluZy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0uZW1iZWRkZXIuaW5wdXRfZW1iZWRkaW5nLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [\"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0uZmluYWxfbm9ybS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0uZmluYWxfbm9ybV8xLkRlbnNlXzAuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0uZmluYWxfbm9ybV8xLkRlbnNlXzAua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLm1scC5nYXRpbmdfZWluc3VtLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLm1scC5saW5lYXIudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLm1scF8xLmdhdGluZ19laW5zdW0udmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLm1scF8xLmxpbmVhci52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLmF0dG4uYXR0bl92ZWNfZWluc3VtLncudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLmF0dG4uYXR0bl92ZWNfZWluc3VtXzEudy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLmF0dG4ua3ZfZWluc3VtLncudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLmF0dG4ua3ZfZWluc3VtXzEudy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, null, \"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLmF0dG4ucV9laW5zdW0udy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLmF0dG4ucV9laW5zdW1fMS53LnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, \"fsdp\", null], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLnByZV9hdHRlbnRpb25fbm9ybS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLnByZV9hdHRlbnRpb25fbm9ybV8xLkRlbnNlXzAuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLnByZV9hdHRlbnRpb25fbm9ybV8xLkRlbnNlXzAua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLnByZV9mZndfbm9ybS5zY2FsZS52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLnByZV9mZndfbm9ybV8xLkRlbnNlXzAuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLlBhbGlHZW1tYS5sbG0ubGF5ZXJzLnByZV9mZndfbm9ybV8xLkRlbnNlXzAua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLm1ldGFkYXRhX21scF9pbi5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLm1ldGFkYXRhX21scF9pbi5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLm1ldGFkYXRhX21scF9vdXQuYmlhcy52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLm1ldGFkYXRhX21scF9vdXQua2VybmVsLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLmFjdGlvbl9pbl9wcm9qLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLmFjdGlvbl9pbl9wcm9qLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLmFjdGlvbl9vdXRfcHJvai5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLmFjdGlvbl9vdXRfcHJvai5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLnRpbWVfbWxwX291dC5iaWFzLnZhbHVl":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLnRpbWVfbWxwX291dC5rZXJuZWwudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLnRpbWVfbWxwX2luLmJpYXMudmFsdWU=":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}","cGFyYW1zLnRpbWVfbWxwX2luLmtlcm5lbC52YWx1ZQ==":"{\"sharding_type\": \"NamedSharding\", \"shape\": [4, 8], \"axis_names\": [\"batch\", \"fsdp\"], \"partition_spec\": [null, \"fsdp\"], \"device_mesh\": {\"mesh\": [[{\"id\": 0}, {\"id\": 1}, {\"id\": 2}, {\"id\": 3}, {\"id\": 4}, {\"id\": 5}, {\"id\": 6}, {\"id\": 7}], [{\"id\": 8}, {\"id\": 9}, {\"id\": 10}, {\"id\": 11}, {\"id\": 12}, {\"id\": 13}, {\"id\": 14}, {\"id\": 15}], [{\"id\": 16}, {\"id\": 17}, {\"id\": 18}, {\"id\": 19}, {\"id\": 20}, {\"id\": 21}, {\"id\": 22}, {\"id\": 23}], [{\"id\": 24}, {\"id\": 25}, {\"id\": 26}, {\"id\": 27}, {\"id\": 28}, {\"id\": 29}, {\"id\": 30}, {\"id\": 31}]]}}"} |